#include <ParquetStringEncoder.h>

Inheritance diagram for foreign_storage::ParquetStringEncoder< V >:

Collaboration diagram for foreign_storage::ParquetStringEncoder< V >:

Public Member Functions
	ParquetStringEncoder (Data_Namespace::AbstractBuffer buffer, StringDictionary string_dictionary, ChunkMetadata *chunk_metadata)

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	encodeAndCopyContiguous (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes, const size_t num_elements) override

void	encodeAndCopy (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes) override

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::TypedParquetInPlaceEncoder< V, V >
	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer buffer, const ColumnDescriptor column_desciptor, const parquet::ColumnDescriptor *parquet_column_descriptor)

	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

void	validate (const int8_t *parquet_data, const int64_t j, const SQLTypeInfo &column_type) const override

std::string	integralTypeToString (const V &element) const

bool	isIntegralType (const SQLTypeInfo &type) const

std::string	elementToString (const V &element) const

std::string	encodedDataToString (const int8_t *bytes) const override

void	setDetectBufferConverterType ()

void	validateUsingEncodersColumnType (const int8_t *parquet_data, const int64_t j) const override

void	reserve (const size_t num_append_elements) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	encodeAndCopyContiguous (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes, const size_t num_elements) override

void	setNull (int8_t *omnisci_data_bytes) override

void	copy (const int8_t omnisci_data_bytes_source, int8_t omnisci_data_bytes_destination) override

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetInPlaceEncoder
	ParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

Public Member Functions inherited from foreign_storage::ParquetScalarEncoder
	ParquetScalarEncoder (Data_Namespace::AbstractBuffer *buffer)

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking ()

virtual void	initializeColumnType (const SQLTypeInfo &column_type)

Protected Member Functions
bool	encodingIsIdentityForSameTypes () const override

Protected Member Functions inherited from foreign_storage::TypedParquetInPlaceEncoder< V, V >
std::pair< V, V >	getUnencodedStats (std::shared_ptr< parquet::Statistics > stats) const

Private Member Functions
void	updateMetadataStats (int64_t values_read, int8_t *values)

Private Attributes
StringDictionary *	string_dictionary_

ChunkMetadata *	chunk_metadata_

std::vector< int8_t >	encode_buffer_

V	min_

V	max_

int64_t	current_batch_offset_

InvalidRowGroupIndices *	invalid_indices_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetInPlaceEncoder
const size_t	omnisci_data_type_byte_size_

const size_t	parquet_data_type_byte_size_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Detailed Description

template<typename V>
class foreign_storage::ParquetStringEncoder< V >

Definition at line 29 of file ParquetStringEncoder.h.

Constructor & Destructor Documentation

template<typename V >

foreign_storage::ParquetStringEncoder< V >::ParquetStringEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		StringDictionary *	string_dictionary,
		ChunkMetadata *	chunk_metadata
	)

inline

Definition at line 31 of file ParquetStringEncoder.h.

References foreign_storage::ParquetStringEncoder< V >::chunk_metadata_, ChunkMetadata::chunkStats, and ChunkStats::has_nulls.

       : TypedParquetInPlaceEncoder<V, V>(buffer, sizeof(V), sizeof(V))
       , string_dictionary_(string_dictionary)
       , chunk_metadata_(chunk_metadata)
       , encode_buffer_(LazyParquetChunkLoader::batch_reader_num_elements * sizeof(V))
       , min_(std::numeric_limits<V>::max())
       , max_(std::numeric_limits<V>::lowest())
       , current_batch_offset_(0)
       , invalid_indices_(nullptr) {
     if (chunk_metadata_) {
       chunk_metadata_->chunkStats.has_nulls = false;
     }
   }

Member Function Documentation

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Appends Parquet data to the buffer using an in-place algorithm. Any necessary transformation or validation of the data and decoding of nulls is part of appending the data. Each class inheriting from this abstract class must implement the functionality to copy, nullify and encode the data.

Parameters

def_levels	- an array containing the Dremel encoding definition levels
rep_levels	- an array containing the Dremel encoding repetition levels
values_read	- the number of non-null values read
levels_read	- the total number of values (non-null & null) that are read
values	- values that are read

Note that the Parquet format encodes nulls using Dremel encoding.

Reimplemented from foreign_storage::ParquetInPlaceEncoder.

Definition at line 96 of file ParquetStringEncoder.h.

References foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::appendData(), foreign_storage::ParquetStringEncoder< V >::chunk_metadata_, ChunkMetadata::chunkStats, foreign_storage::ParquetStringEncoder< V >::encode_buffer_, foreign_storage::ParquetStringEncoder< V >::encodeAndCopyContiguous(), and ChunkStats::has_nulls.

Referenced by foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors(), and foreign_storage::ParquetStringEncoder< V >::validateAndAppendData().

                                            {
     encodeAndCopyContiguous(values, encode_buffer_.data(), values_read);
     TypedParquetInPlaceEncoder<V, V>::appendData(
         def_levels, rep_levels, values_read, levels_read, encode_buffer_.data());
     chunk_metadata_->chunkStats.has_nulls =
         chunk_metadata_->chunkStats.has_nulls || (values_read < levels_read);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 71 of file ParquetStringEncoder.h.

References foreign_storage::ParquetStringEncoder< V >::appendData(), CHECK, CHECK_LT, foreign_storage::ParquetEncoder::column_type_, foreign_storage::ParquetEncoder::current_chunk_offset_, foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, and StringDictionary::MAX_STRLEN.

                                                       {
     CHECK(ParquetEncoder::is_error_tracking_enabled_);
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         CHECK_LT(j, values_read);
         auto& byte_array = parquet_data_ptr[j++];
         if (byte_array.len > StringDictionary::MAX_STRLEN) {
           ParquetEncoder::invalid_indices_.insert(ParquetEncoder::current_chunk_offset_ +
                                                   i);
         }
       } else if (ParquetEncoder::column_type_
                      .get_notnull()) {  // item is null for NOT NULL column
         ParquetEncoder::invalid_indices_.insert(ParquetEncoder::current_chunk_offset_ +
                                                 i);
       }
     }
     ParquetEncoder::current_chunk_offset_ += levels_read;
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::encodeAndCopy	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 130 of file ParquetStringEncoder.h.

References foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::copy().

                                                           {
     TypedParquetInPlaceEncoder<V, V>::copy(parquet_data_bytes, omnisci_data_bytes);
   }

Here is the call graph for this function:

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::encodeAndCopyContiguous	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes,
		const size_t	num_elements
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 108 of file ParquetStringEncoder.h.

References CHECK, StringDictionary::getOrAddBulk(), StringDictionary::MAX_STRLEN, foreign_storage::ParquetStringEncoder< V >::string_dictionary_, and foreign_storage::ParquetStringEncoder< V >::updateMetadataStats().

Referenced by foreign_storage::ParquetStringEncoder< V >::appendData().

                                                                    {
     CHECK(string_dictionary_);
     auto parquet_data_ptr =
         reinterpret_cast<const parquet::ByteArray*>(parquet_data_bytes);
     auto omnisci_data_ptr = reinterpret_cast<V*>(omnisci_data_bytes);
     std::vector<std::string_view> string_views;
     string_views.reserve(num_elements);
     for (size_t i = 0; i < num_elements; ++i) {
       auto& byte_array = parquet_data_ptr[i];
       if (byte_array.len <= StringDictionary::MAX_STRLEN) {
         string_views.emplace_back(reinterpret_cast<const char*>(byte_array.ptr),
                                   byte_array.len);
       } else {
         string_views.emplace_back(nullptr, 0);
       }
     }
     string_dictionary_->getOrAddBulk(string_views, omnisci_data_ptr);
     updateMetadataStats(num_elements, omnisci_data_bytes);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V >

bool foreign_storage::ParquetStringEncoder< V >::encodingIsIdentityForSameTypes ( ) const

inlineoverrideprotectedvirtual

Reimplemented from foreign_storage::TypedParquetInPlaceEncoder< V, V >.

Definition at line 151 of file ParquetStringEncoder.h.

151 { return true; }

template<typename V >

std::shared_ptr<ChunkMetadata> foreign_storage::ParquetStringEncoder< V >::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 135 of file ParquetStringEncoder.h.

References foreign_storage::ParquetEncoder::getRowGroupMetadata(), and foreign_storage::ParquetInPlaceEncoder::omnisci_data_type_byte_size_.

                                                {
     auto metadata = ParquetEncoder::getRowGroupMetadata(
         group_metadata, parquet_column_index, column_type);
     auto column_metadata = group_metadata->ColumnChunk(parquet_column_index);
     metadata->numBytes = ParquetInPlaceEncoder::omnisci_data_type_byte_size_ *
                          column_metadata->num_values();
 
     // Placeholder metadata is defined with has_nulls = false.
     metadata->chunkStats.has_nulls = false;
     return metadata;
   }

Here is the call graph for this function:

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::updateMetadataStats	(	int64_t	values_read,
		int8_t *	values
	)

inlineprivate

Definition at line 154 of file ParquetStringEncoder.h.

References foreign_storage::ParquetStringEncoder< V >::chunk_metadata_, ChunkMetadata::chunkStats, ChunkMetadata::fillChunkStats(), ChunkStats::has_nulls, foreign_storage::ParquetStringEncoder< V >::max_, and foreign_storage::ParquetStringEncoder< V >::min_.

Referenced by foreign_storage::ParquetStringEncoder< V >::encodeAndCopyContiguous().

                                                                 {
     if (!chunk_metadata_) {
       return;
     }
     V* data_ptr = reinterpret_cast<V*>(values);
     for (int64_t i = 0; i < values_read; ++i) {
       min_ = std::min<V>(data_ptr[i], min_);
       max_ = std::max<V>(data_ptr[i], max_);
     }
     chunk_metadata_->fillChunkStats(min_, max_, chunk_metadata_->chunkStats.has_nulls);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V >

void foreign_storage::ParquetStringEncoder< V >::validateAndAppendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values,
		const SQLTypeInfo &	column_type,
		InvalidRowGroupIndices &	invalid_indices
	)

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 50 of file ParquetStringEncoder.h.

References foreign_storage::ParquetStringEncoder< V >::appendData(), CHECK_LT, foreign_storage::ParquetStringEncoder< V >::current_batch_offset_, and StringDictionary::MAX_STRLEN.

                                                                                {
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         CHECK_LT(j, values_read);
         auto& byte_array = parquet_data_ptr[j++];
         if (byte_array.len > StringDictionary::MAX_STRLEN) {
           invalid_indices.insert(current_batch_offset_ + i);
         }
       }
     }
     current_batch_offset_ += levels_read;
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }