#include <ParquetArrayEncoder.h>

Inheritance diagram for foreign_storage::ParquetArrayEncoder:

Collaboration diagram for foreign_storage::ParquetArrayEncoder:

Public Member Functions
	ParquetArrayEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	finalizeRowGroup ()

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

virtual void	disableMetadataStatsValidation () override

virtual void	initializeErrorTracking () override

virtual void	initializeColumnType (const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

RejectedRowIndices	getRejectedRowIndices () const

Protected Member Functions
virtual void	processLastArray ()

virtual void	appendArraysToBuffer ()

bool	isLastArrayNull () const

bool	isLastArrayEmpty () const

size_t	sizeOfLastArray () const

int8_t *	resizeArrayDataBytes (const size_t additional_num_elements)

virtual void	resetLastArrayMetadata ()

bool	isNewArray (const int16_t rep_level) const

int8_t *	encodedDataAtIndex (const size_t index)

void	updateMetadataForAppendedArrayItem (const int64_t encoded_index)

virtual void	appendArrayItem (const int64_t encoded_index)

virtual void	encodeAllValues (const int8_t *values, const int64_t values_read)

Protected Attributes
size_t	omnisci_data_type_byte_size_

std::shared_ptr < ParquetScalarEncoder >	scalar_encoder_

std::vector< int8_t >	data_buffer_bytes_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Static Protected Attributes
static const int16_t	non_null_def_level = 3

static const int16_t	item_null_def_level = 2

static const int16_t	empty_list_def_level = 1

static const int16_t	list_null_def_level = 0

Private Member Functions
void	processArrayItem (const int16_t def_level, int64_t &encoded_index)

void	markArrayAsNull ()

void	markArrayAsEmpty ()

void	appendNullArrayItem ()

Private Attributes
std::vector< int8_t >	encode_buffer_

bool	has_assembly_started_

bool	is_null_array_

bool	is_empty_array_

size_t	num_elements_in_array_

size_t	num_array_assembled_

bool	is_invalid_array_

std::vector< bool >	is_valid_item_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Detailed Description

Definition at line 23 of file ParquetArrayEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetArrayEncoder::ParquetArrayEncoder	(	Data_Namespace::AbstractBuffer *	data_buffer,
		std::shared_ptr< ParquetScalarEncoder >	scalar_encoder,
		const ColumnDescriptor *	column_desciptor
	)

inline

Definition at line 25 of file ParquetArrayEncoder.h.

       : ParquetEncoder(data_buffer)
       , omnisci_data_type_byte_size_(
             column_desciptor->columnType.get_elem_type().get_size())
       , scalar_encoder_(scalar_encoder)
       , has_assembly_started_(false)
       , is_null_array_(false)
       , is_empty_array_(false)
       , num_elements_in_array_(0)
       , num_array_assembled_(0)
       , is_invalid_array_(false) {}

Member Function Documentation

virtual void foreign_storage::ParquetArrayEncoder::appendArrayItem ( const int64_t encoded_index )

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetArrayImportEncoder, and foreign_storage::ParquetArrayDetectEncoder.

Definition at line 171 of file ParquetArrayEncoder.h.

References encodedDataAtIndex(), resizeArrayDataBytes(), scalar_encoder_, and updateMetadataForAppendedArrayItem().

Referenced by foreign_storage::ParquetArrayImportEncoder::appendArrayItem(), and processArrayItem().

                                                             {
     auto omnisci_data_ptr = resizeArrayDataBytes(1);
     scalar_encoder_->copy(encodedDataAtIndex(encoded_index), omnisci_data_ptr);
     updateMetadataForAppendedArrayItem(encoded_index);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::appendArraysToBuffer ( )

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetArrayImportEncoder, foreign_storage::ParquetArrayDetectEncoder, and foreign_storage::ParquetVariableLengthArrayEncoder.

Definition at line 119 of file ParquetArrayEncoder.h.

References Data_Namespace::AbstractBuffer::append(), foreign_storage::ParquetEncoder::buffer_, and data_buffer_bytes_.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer(), and finalizeRowGroup().

                                       {
     buffer_->append(data_buffer_bytes_.data(), data_buffer_bytes_.size());
     data_buffer_bytes_.clear();
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Reimplemented in foreign_storage::ParquetVariableLengthArrayEncoder.

Definition at line 57 of file ParquetArrayEncoder.h.

References CHECK, encodeAllValues(), isNewArray(), processArrayItem(), processLastArray(), and resetLastArrayMetadata().

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendData(), appendDataTrackErrors(), and foreign_storage::ParquetArrayImportEncoder::validateAndAppendData().

                                            {
     CHECK(levels_read > 0);
 
     // encode all values in the temporary in-memory `encode_buffer_`, doing
     // this encoding as a batch rather than element-wise exposes opportunities
     // for performance optimization for certain scalar types
     encodeAllValues(values, values_read);
 
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (isNewArray(rep_levels[i])) {
         processLastArray();
         resetLastArrayMetadata();
       }
       processArrayItem(def_levels[i], j);
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 39 of file ParquetArrayEncoder.h.

References appendData(), CHECK, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, is_valid_item_, and scalar_encoder_.

                                                       {
     CHECK(is_error_tracking_enabled_);
     // validate all elements
     is_valid_item_.assign(values_read, true);
     for (int64_t j = 0; j < values_read; ++j) {
       try {
         scalar_encoder_->validateUsingEncodersColumnType(values, j);
       } catch (const std::runtime_error& error) {
         is_valid_item_[j] = false;
       }
     }
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

void foreign_storage::ParquetArrayEncoder::appendNullArrayItem ( )

inlineprivate

Definition at line 204 of file ParquetArrayEncoder.h.

References num_elements_in_array_, resizeArrayDataBytes(), and scalar_encoder_.

Referenced by processArrayItem().

                              {
     scalar_encoder_->setNull(resizeArrayDataBytes(1));
     num_elements_in_array_++;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::disableMetadataStatsValidation ( )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 95 of file ParquetArrayEncoder.h.

References foreign_storage::ParquetEncoder::disableMetadataStatsValidation(), and scalar_encoder_.

                                                          {
     ParquetEncoder::disableMetadataStatsValidation();
     scalar_encoder_->disableMetadataStatsValidation();
   }

Here is the call graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::encodeAllValues	(	const int8_t *	values,
		const int64_t	values_read
	)

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetArrayDetectEncoder.

Definition at line 177 of file ParquetArrayEncoder.h.

References encode_buffer_, omnisci_data_type_byte_size_, and scalar_encoder_.

Referenced by appendData(), and foreign_storage::ParquetArrayDetectEncoder::encodeAllValues().

                                                                                 {
     encode_buffer_.resize(values_read * omnisci_data_type_byte_size_);
     scalar_encoder_->encodeAndCopyContiguous(values, encode_buffer_.data(), values_read);
   }

Here is the caller graph for this function:

int8_t* foreign_storage::ParquetArrayEncoder::encodedDataAtIndex ( const size_t index )

inlineprotected

Definition at line 160 of file ParquetArrayEncoder.h.

References encode_buffer_, and omnisci_data_type_byte_size_.

Referenced by foreign_storage::ParquetArrayDetectEncoder::appendArrayItem(), and appendArrayItem().

                                                  {
     return encode_buffer_.data() + (index)*omnisci_data_type_byte_size_;
   }

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::finalizeRowGroup ( )

inline

Definition at line 78 of file ParquetArrayEncoder.h.

References appendArraysToBuffer(), has_assembly_started_, processLastArray(), and resetLastArrayMetadata().

                           {
     processLastArray();
     resetLastArrayMetadata();
     appendArraysToBuffer();
     has_assembly_started_ = false;
   }

Here is the call graph for this function:

std::shared_ptr<ChunkMetadata> foreign_storage::ParquetArrayEncoder::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Reimplemented in foreign_storage::ParquetFixedLengthArrayEncoder.

Definition at line 85 of file ParquetArrayEncoder.h.

References scalar_encoder_.

Referenced by foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata().

                                                {
     auto metadata = scalar_encoder_->getRowGroupMetadata(
         group_metadata, parquet_column_index, column_type);
     metadata->numBytes = 0;  // number of bytes is not known
     return metadata;
   }

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::initializeColumnType ( const SQLTypeInfo & column_type )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 105 of file ParquetArrayEncoder.h.

References SQLTypeInfo::get_elem_type(), foreign_storage::ParquetEncoder::initializeColumnType(), and scalar_encoder_.

                                                                              {
     ParquetEncoder::initializeColumnType(column_type);
     scalar_encoder_->initializeColumnType(column_type.get_elem_type());
   }

Here is the call graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::initializeErrorTracking ( )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 100 of file ParquetArrayEncoder.h.

References foreign_storage::ParquetEncoder::initializeErrorTracking(), and scalar_encoder_.

                                                   {
     ParquetEncoder::initializeErrorTracking();
     scalar_encoder_->initializeErrorTracking();
   }

Here is the call graph for this function:

bool foreign_storage::ParquetArrayEncoder::isLastArrayEmpty ( ) const

inlineprotected

Definition at line 126 of file ParquetArrayEncoder.h.

References is_empty_array_.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize(), foreign_storage::ParquetArrayImportEncoder::appendToArrayDatumBuffer(), and foreign_storage::ParquetArrayDetectEncoder::appendToDetectBuffer().

126 { return is_empty_array_; }

foreign_storage::ParquetArrayEncoder::is_empty_array_

bool is_empty_array_

Definition: ParquetArrayEncoder.h:212

Here is the caller graph for this function:

bool foreign_storage::ParquetArrayEncoder::isLastArrayNull ( ) const

inlineprotected

Definition at line 124 of file ParquetArrayEncoder.h.

References is_null_array_.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize(), foreign_storage::ParquetArrayImportEncoder::appendToArrayDatumBuffer(), foreign_storage::ParquetArrayDetectEncoder::appendToDetectBuffer(), and processLastArray().

124 { return is_null_array_; }

foreign_storage::ParquetArrayEncoder::is_null_array_

bool is_null_array_

Definition: ParquetArrayEncoder.h:211

Here is the caller graph for this function:

bool foreign_storage::ParquetArrayEncoder::isNewArray ( const int16_t rep_level ) const

inlineprotected

Definition at line 156 of file ParquetArrayEncoder.h.

References has_assembly_started_.

Referenced by appendData().

                                                  {
     return rep_level == 0 && has_assembly_started_;
   }

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::markArrayAsEmpty ( )

inlineprivate

Definition at line 202 of file ParquetArrayEncoder.h.

References is_empty_array_.

Referenced by processArrayItem().

202 { is_empty_array_ = true; }

foreign_storage::ParquetArrayEncoder::is_empty_array_

bool is_empty_array_

Definition: ParquetArrayEncoder.h:212

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::markArrayAsNull ( )

inlineprivate

Definition at line 200 of file ParquetArrayEncoder.h.

References is_null_array_.

Referenced by processArrayItem().

200 { is_null_array_ = true; }

foreign_storage::ParquetArrayEncoder::is_null_array_

bool is_null_array_

Definition: ParquetArrayEncoder.h:211

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::processArrayItem	(	const int16_t	def_level,
		int64_t &	encoded_index
	)

inlineprivate

Definition at line 183 of file ParquetArrayEncoder.h.

References appendArrayItem(), appendNullArrayItem(), empty_list_def_level, has_assembly_started_, item_null_def_level, list_null_def_level, markArrayAsEmpty(), markArrayAsNull(), non_null_def_level, and UNREACHABLE.

Referenced by appendData().

                                                                          {
     has_assembly_started_ = true;
     if (def_level == non_null_def_level) {
       // push back a scalar element to in-memory data buffer
       appendArrayItem(encoded_index++);
     } else if (def_level == item_null_def_level) {
       // push back a scalar null to in-memory data buffer
       appendNullArrayItem();
     } else if (def_level == list_null_def_level) {
       markArrayAsNull();
     } else if (def_level == empty_list_def_level) {
       markArrayAsEmpty();
     } else {
       UNREACHABLE();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::processLastArray ( )

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetArrayImportEncoder, foreign_storage::ParquetArrayDetectEncoder, foreign_storage::ParquetVariableLengthArrayEncoder, and foreign_storage::ParquetFixedLengthArrayEncoder.

Definition at line 111 of file ParquetArrayEncoder.h.

References foreign_storage::ParquetEncoder::column_type_, SQLTypeInfo::get_notnull(), foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, is_invalid_array_, isLastArrayNull(), and num_array_assembled_.

Referenced by appendData(), finalizeRowGroup(), foreign_storage::ParquetFixedLengthArrayEncoder::processLastArray(), foreign_storage::ParquetVariableLengthArrayEncoder::processLastArray(), and foreign_storage::ParquetArrayDetectEncoder::processLastArray().

                                   {
     if (is_error_tracking_enabled_ &&
         (is_invalid_array_ || (isLastArrayNull() && column_type_.get_notnull()))) {
       invalid_indices_.insert(num_array_assembled_);
     }
     ++num_array_assembled_;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::resetLastArrayMetadata ( )

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetArrayImportEncoder.

Definition at line 147 of file ParquetArrayEncoder.h.

References is_empty_array_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, is_invalid_array_, is_null_array_, and num_elements_in_array_.

Referenced by appendData(), finalizeRowGroup(), and foreign_storage::ParquetArrayImportEncoder::resetLastArrayMetadata().

                                         {
     is_empty_array_ = false;
     is_null_array_ = false;
     num_elements_in_array_ = 0;
     if (is_error_tracking_enabled_) {
       is_invalid_array_ = false;
     }
   }

Here is the caller graph for this function:

int8_t* foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes ( const size_t additional_num_elements )

inlineprotected

Definition at line 130 of file ParquetArrayEncoder.h.

References data_buffer_bytes_, and omnisci_data_type_byte_size_.

Referenced by appendArrayItem(), appendNullArrayItem(), and foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray().

                                                                      {
     auto current_data_byte_size = data_buffer_bytes_.size();
     data_buffer_bytes_.resize(current_data_byte_size +
                               additional_num_elements * omnisci_data_type_byte_size_);
     return data_buffer_bytes_.data() + current_data_byte_size;
   }

Here is the caller graph for this function:

size_t foreign_storage::ParquetArrayEncoder::sizeOfLastArray ( ) const

inlineprotected

Definition at line 128 of file ParquetArrayEncoder.h.

References num_elements_in_array_.

Referenced by foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize(), and foreign_storage::ParquetArrayImportEncoder::appendToArrayDatumBuffer().

128 { return num_elements_in_array_; }

foreign_storage::ParquetArrayEncoder::num_elements_in_array_

size_t num_elements_in_array_

Definition: ParquetArrayEncoder.h:213

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::updateMetadataForAppendedArrayItem ( const int64_t encoded_index )

inlineprotected

Definition at line 164 of file ParquetArrayEncoder.h.

References foreign_storage::ParquetEncoder::is_error_tracking_enabled_, is_invalid_array_, is_valid_item_, and num_elements_in_array_.

Referenced by foreign_storage::ParquetArrayDetectEncoder::appendArrayItem(), and appendArrayItem().

                                                                        {
     num_elements_in_array_++;
     if (is_error_tracking_enabled_ && !is_valid_item_[encoded_index]) {
       is_invalid_array_ = true;
     }
   }