#include <ParquetFixedLengthArrayEncoder.h>

Inheritance diagram for foreign_storage::ParquetFixedLengthArrayEncoder:

Collaboration diagram for foreign_storage::ParquetFixedLengthArrayEncoder:

Public Member Functions
	ParquetFixedLengthArrayEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetArrayEncoder
	ParquetArrayEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	finalizeRowGroup ()

virtual void	disableMetadataStatsValidation () override

virtual void	initializeErrorTracking () override

virtual void	initializeColumnType (const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

RejectedRowIndices	getRejectedRowIndices () const

Protected Member Functions
void	processLastArray () override

Protected Member Functions inherited from foreign_storage::ParquetArrayEncoder
virtual void	appendArraysToBuffer ()

bool	isLastArrayNull () const

bool	isLastArrayEmpty () const

size_t	sizeOfLastArray () const

int8_t *	resizeArrayDataBytes (const size_t additional_num_elements)

virtual void	resetLastArrayMetadata ()

bool	isNewArray (const int16_t rep_level) const

int8_t *	encodedDataAtIndex (const size_t index)

void	updateMetadataForAppendedArrayItem (const int64_t encoded_index)

virtual void	appendArrayItem (const int64_t encoded_index)

virtual void	encodeAllValues (const int8_t *values, const int64_t values_read)

Private Member Functions
void	appendNullFixedLengthArray ()

void	setNullFixedLengthArraySentinel (int8_t *omnisci_data_bytes)

void	appendNullArrayOrCheckArraySize ()

void	throwEmptyArrayException (const size_t array_element_count, const std::string &omnisci_column_name)

void	throwWrongSizeArray (const size_t size_of_last_array, const size_t array_element_count, const std::string &omnisci_column_name)

Private Attributes
const ColumnDescriptor	column_desciptor_

size_t	array_element_count_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
size_t	omnisci_data_type_byte_size_

std::shared_ptr < ParquetScalarEncoder >	scalar_encoder_

std::vector< int8_t >	data_buffer_bytes_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Static Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
static const int16_t	non_null_def_level = 3

static const int16_t	item_null_def_level = 2

static const int16_t	empty_list_def_level = 1

static const int16_t	list_null_def_level = 0

Detailed Description

Definition at line 28 of file ParquetFixedLengthArrayEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetFixedLengthArrayEncoder::ParquetFixedLengthArrayEncoder	(	Data_Namespace::AbstractBuffer *	data_buffer,
		std::shared_ptr< ParquetScalarEncoder >	scalar_encoder,
		const ColumnDescriptor *	column_desciptor
	)

inline

Definition at line 30 of file ParquetFixedLengthArrayEncoder.h.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_size(), and foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

       : ParquetArrayEncoder(data_buffer, scalar_encoder, column_desciptor)
       , column_desciptor_(*column_desciptor)
       , array_element_count_(column_desciptor->columnType.get_size() /
                              omnisci_data_type_byte_size_) {
     CHECK(column_desciptor->columnType.get_size() % omnisci_data_type_byte_size_ == 0);
   }

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize ( )

inlineprivate

Definition at line 121 of file ParquetFixedLengthArrayEncoder.h.

References appendNullFixedLengthArray(), array_element_count_, CHECK, column_desciptor_, ColumnDescriptor::columnName, foreign_storage::ParquetArrayEncoder::isLastArrayEmpty(), foreign_storage::ParquetArrayEncoder::isLastArrayNull(), foreign_storage::ParquetArrayEncoder::sizeOfLastArray(), throwEmptyArrayException(), and throwWrongSizeArray().

Referenced by processLastArray().

                                          {
     auto size_of_last_array = sizeOfLastArray();
     if (isLastArrayNull()) {
       // append a null array sentinel
       CHECK(size_of_last_array == 0);
       appendNullFixedLengthArray();
     } else if (isLastArrayEmpty()) {
       throwEmptyArrayException(array_element_count_, column_desciptor_.columnName);
     } else {
       if (size_of_last_array != array_element_count_) {
         throwWrongSizeArray(
             size_of_last_array, array_element_count_, column_desciptor_.columnName);
       }
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray ( )

inlineprivate

Definition at line 58 of file ParquetFixedLengthArrayEncoder.h.

References array_element_count_, foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_, foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes(), foreign_storage::ParquetArrayEncoder::scalar_encoder_, and setNullFixedLengthArraySentinel().

Referenced by appendNullArrayOrCheckArraySize().

                                     {
     auto omnisci_data_ptr = resizeArrayDataBytes(array_element_count_);
     setNullFixedLengthArraySentinel(omnisci_data_ptr);
     for (size_t i = 1; i < array_element_count_; ++i) {
       scalar_encoder_->setNull(omnisci_data_ptr + i * omnisci_data_type_byte_size_);
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ChunkMetadata> foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 40 of file ParquetFixedLengthArrayEncoder.h.

References array_element_count_, foreign_storage::ParquetArrayEncoder::getRowGroupMetadata(), and foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

                                                {
     auto metadata = ParquetArrayEncoder::getRowGroupMetadata(
         group_metadata, parquet_column_index, column_type);
     metadata->numBytes =
         omnisci_data_type_byte_size_ * group_metadata->num_rows() * array_element_count_;
     return metadata;
   }

Here is the call graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::processLastArray ( )

inlineoverrideprotectedvirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 52 of file ParquetFixedLengthArrayEncoder.h.

References appendNullArrayOrCheckArraySize(), and foreign_storage::ParquetArrayEncoder::processLastArray().

                                    {
     ParquetArrayEncoder::processLastArray();
     appendNullArrayOrCheckArraySize();
   }

Here is the call graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::setNullFixedLengthArraySentinel ( int8_t * omnisci_data_bytes )

inlineprivate

Definition at line 66 of file ParquetFixedLengthArrayEncoder.h.

References column_desciptor_, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), inline_fixed_encoding_null_array_val(), kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, string_dict_to_int_type(), run_benchmark_import::type, and UNREACHABLE.

Referenced by appendNullFixedLengthArray().

                                                                    {
     auto ti = column_desciptor_.columnType.get_elem_type();
     SQLTypes type;
     if (ti.is_dict_encoded_string()) {
       type = string_dict_to_int_type(ti);
     } else {
       type = ti.get_type();
     }
     switch (type) {
       case kBOOLEAN:
         reinterpret_cast<bool*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kBIGINT:
       case kNUMERIC:
       case kDECIMAL:
         reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kINT:
         reinterpret_cast<int32_t*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kSMALLINT:
         reinterpret_cast<int16_t*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kTINYINT:
         reinterpret_cast<int8_t*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kFLOAT:
         reinterpret_cast<float*>(omnisci_data_bytes)[0] = NULL_ARRAY_FLOAT;
         break;
       case kDOUBLE:
         reinterpret_cast<double*>(omnisci_data_bytes)[0] = NULL_ARRAY_DOUBLE;
         break;
       case kTIME:
       case kTIMESTAMP:
       case kDATE:
         reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
             inline_fixed_encoding_null_array_val(ti);
         break;
       case kTEXT:
       case kVARCHAR:
       case kCHAR:
       case kPOINT:
       case kLINESTRING:
       case kPOLYGON:
       case kMULTIPOLYGON:
       default:
         UNREACHABLE();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::throwEmptyArrayException	(	const size_t	array_element_count,
		const std::string &	omnisci_column_name
	)

inlineprivate

Definition at line 137 of file ParquetFixedLengthArrayEncoder.h.

References to_string().

Referenced by appendNullArrayOrCheckArraySize().

                                                                       {
     throw ForeignStorageException(
         "Detected an empty array"
         " being loaded into"
         " HeavyDB column '" +
         omnisci_column_name +
         "' which has a fixed length array type,"
         " expecting " +
         std::to_string(array_element_count) + " elements.");
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::throwWrongSizeArray	(	const size_t	size_of_last_array,
		const size_t	array_element_count,
		const std::string &	omnisci_column_name
	)

inlineprivate

Definition at line 149 of file ParquetFixedLengthArrayEncoder.h.

References to_string().

Referenced by appendNullArrayOrCheckArraySize().

                                                                  {
     throw ForeignStorageException("Detected a row with " +
                                   std::to_string(size_of_last_array) +
                                   " elements being loaded into"
                                   " HeavyDB column '" +
                                   omnisci_column_name +
                                   "' which has a fixed length array type,"
                                   " expecting " +
                                   std::to_string(array_element_count) + " elements.");
   }