#include <ParquetDetectStringEncoder.h>

Inheritance diagram for foreign_storage::ParquetDetectStringEncoder:

Collaboration diagram for foreign_storage::ParquetDetectStringEncoder:

Public Member Functions
	ParquetDetectStringEncoder (Data_Namespace::AbstractBuffer *buffer)

void	setNull (int8_t *omnisci_data_bytes) override

void	copy (const int8_t omnisci_data_bytes_source, int8_t omnisci_data_bytes_destination) override

void	encodeAndCopy (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes) override

void	encodeAndCopyContiguous (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes, const size_t num_elements) override

void	validate (const int8_t *parquet_data, const int64_t j, const SQLTypeInfo &column_type) const override

void	validateUsingEncodersColumnType (const int8_t *parquet_data, const int64_t j) const override

std::string	encodedDataToString (const int8_t *bytes) const override

void	eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

Public Member Functions inherited from foreign_storage::ParquetScalarEncoder
	ParquetScalarEncoder (Data_Namespace::AbstractBuffer *buffer)

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

virtual std::shared_ptr < ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking ()

virtual void	initializeColumnType (const SQLTypeInfo &column_type)

Public Attributes
TypedParquetDetectBuffer *	detect_buffer_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Detailed Description

Definition at line 29 of file ParquetDetectStringEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetDetectStringEncoder::ParquetDetectStringEncoder ( Data_Namespace::AbstractBuffer * buffer )

inline

Definition at line 31 of file ParquetDetectStringEncoder.h.

References CHECK, and detect_buffer_.

       : ParquetScalarEncoder(buffer)
       , detect_buffer_(dynamic_cast<TypedParquetDetectBuffer*>(buffer_)) {
     CHECK(detect_buffer_);
   }

Member Function Documentation

void foreign_storage::ParquetDetectStringEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 89 of file ParquetDetectStringEncoder.h.

References foreign_storage::TypedParquetDetectBuffer::appendValue(), CHECK, foreign_storage::ParquetEncoder::current_chunk_offset_, detect_buffer_, foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, and StringDictionary::MAX_STRLEN.

Referenced by appendDataTrackErrors().

                                            {
     CHECK(levels_read > 0);
 
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
 
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         CHECK(j < values_read);
         auto& byte_array = parquet_data_ptr[j++];
         if (is_error_tracking_enabled_ && byte_array.len > StringDictionary::MAX_STRLEN) {
           ParquetEncoder::invalid_indices_.insert(ParquetEncoder::current_chunk_offset_ +
                                                   i);
           detect_buffer_->appendValue({});  // add empty string
         } else {
           auto string_value =
               std::string{reinterpret_cast<const char*>(byte_array.ptr), byte_array.len};
           detect_buffer_->appendValue(string_value);
         }
       } else {
         detect_buffer_->appendValue("NULL");
       }
     }
     if (is_error_tracking_enabled_) {
       ParquetEncoder::current_chunk_offset_ += levels_read;
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetDetectStringEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 120 of file ParquetDetectStringEncoder.h.

References appendData(), CHECK, and foreign_storage::ParquetEncoder::is_error_tracking_enabled_.

                                                       {
     CHECK(is_error_tracking_enabled_);
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

void foreign_storage::ParquetDetectStringEncoder::copy	(	const int8_t *	omnisci_data_bytes_source,
		int8_t *	omnisci_data_bytes_destination
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 38 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                              {
     UNREACHABLE();
   }

void foreign_storage::ParquetDetectStringEncoder::encodeAndCopy	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 43 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                           {
     UNREACHABLE();
   }

void foreign_storage::ParquetDetectStringEncoder::encodeAndCopyContiguous	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes,
		const size_t	num_elements
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 48 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                                    {
     UNREACHABLE();
   }

std::string foreign_storage::ParquetDetectStringEncoder::encodedDataToString ( const int8_t * bytes ) const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 69 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                                     {
     UNREACHABLE();
     return {};
   }

void foreign_storage::ParquetDetectStringEncoder::eraseInvalidIndicesInBuffer ( const InvalidRowGroupIndices & invalid_indices )

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 74 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                               {
     UNREACHABLE();
   }

void foreign_storage::ParquetDetectStringEncoder::setNull ( int8_t * omnisci_data_bytes )

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 37 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

37 { UNREACHABLE(); }

UNREACHABLE

#define UNREACHABLE()

Definition: Logger.h:338

void foreign_storage::ParquetDetectStringEncoder::validate	(	const int8_t *	parquet_data,
		const int64_t	j,
		const SQLTypeInfo &	column_type
	)		const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 54 of file ParquetDetectStringEncoder.h.

References StringDictionary::MAX_STRLEN.

Referenced by validateUsingEncodersColumnType().

                                                                {
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(parquet_data);
     auto& byte_array = parquet_data_ptr[j];
     if (byte_array.len > StringDictionary::MAX_STRLEN) {
       throw ForeignStorageException("String exceeeds max length allowed in dictionary");
     }
   }

Here is the caller graph for this function:

void foreign_storage::ParquetDetectStringEncoder::validateAndAppendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values,
		const SQLTypeInfo &	column_type,
		InvalidRowGroupIndices &	invalid_indices
	)

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 79 of file ParquetDetectStringEncoder.h.

References UNREACHABLE.

                                                                                {
     UNREACHABLE();
   }

void foreign_storage::ParquetDetectStringEncoder::validateUsingEncodersColumnType	(	const int8_t *	parquet_data,
		const int64_t	j
	)		const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 64 of file ParquetDetectStringEncoder.h.

References foreign_storage::ParquetEncoder::column_type_, and validate().

                                                                        {
     validate(parquet_data, j, column_type_);
   }

Here is the call graph for this function:

Member Data Documentation

TypedParquetDetectBuffer* foreign_storage::ParquetDetectStringEncoder::detect_buffer_

Definition at line 129 of file ParquetDetectStringEncoder.h.

Referenced by appendData(), and ParquetDetectStringEncoder().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetDetectStringEncoder.h

Public Member Functions

Public Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation