#include <ParquetEncoder.h>

Inheritance diagram for foreign_storage::ParquetEncoder:

Collaboration diagram for foreign_storage::ParquetEncoder:

Public Member Functions
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

virtual void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values)=0

virtual void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values)=0

virtual std::shared_ptr < ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking ()

virtual void	initializeColumnType (const SQLTypeInfo &column_type)

Static Protected Member Functions
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Detailed Description

Definition at line 30 of file ParquetEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetEncoder::ParquetEncoder ( Data_Namespace::AbstractBuffer * buffer )

inline

Definition at line 32 of file ParquetEncoder.h.

       : buffer_(buffer)
       , is_error_tracking_enabled_(false)
       , current_chunk_offset_(0)
       , validate_metadata_stats_(true) {}

virtual foreign_storage::ParquetEncoder::~ParquetEncoder ( )

virtualdefault

Member Function Documentation

virtual void foreign_storage::ParquetEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

pure virtual

virtual void foreign_storage::ParquetEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

pure virtual

Implemented in foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >, foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t, NullType >, foreign_storage::TypedParquetInPlaceEncoder< V, V >, foreign_storage::ParquetGeospatialImportEncoder, foreign_storage::ParquetDetectStringEncoder, foreign_storage::ParquetStringNoneEncoder, foreign_storage::ParquetGeospatialEncoder, foreign_storage::ParquetStringEncoder< V >, foreign_storage::ParquetStringImportEncoder, and foreign_storage::ParquetArrayEncoder.

static std::shared_ptr<ChunkMetadata> foreign_storage::ParquetEncoder::createMetadata ( const SQLTypeInfo & column_type )

inlinestaticprotected

Definition at line 96 of file ParquetEncoder.h.

References SQLTypeInfo::get_elem_type(), Data_Namespace::AbstractBuffer::initEncoder(), and SQLTypeInfo::is_array().

Referenced by getRowGroupMetadata(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata().

                                                                                      {
     auto metadata = std::make_shared<ChunkMetadata>();
     ForeignStorageBuffer buffer;
     buffer.initEncoder(column_type.is_array() ? column_type.get_elem_type()
                                               : column_type);
     auto encoder = buffer.getEncoder();
     encoder->getMetadata(metadata);
     metadata->sqlType = column_type;
     return metadata;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetEncoder::disableMetadataStatsValidation ( )

inlinevirtual

Reimplemented in foreign_storage::ParquetArrayEncoder.

Definition at line 76 of file ParquetEncoder.h.

References validate_metadata_stats_.

Referenced by foreign_storage::ParquetArrayEncoder::disableMetadataStatsValidation().

76 { validate_metadata_stats_ = false; }

foreign_storage::ParquetEncoder::validate_metadata_stats_

bool validate_metadata_stats_

Definition: ParquetEncoder.h:94

Here is the caller graph for this function:

RejectedRowIndices foreign_storage::ParquetEncoder::getRejectedRowIndices ( ) const

inline

Definition at line 74 of file ParquetEncoder.h.

References invalid_indices_.

74 { return invalid_indices_; }

foreign_storage::ParquetEncoder::invalid_indices_

RejectedRowIndices invalid_indices_

Definition: ParquetEncoder.h:89

virtual std::shared_ptr<ChunkMetadata> foreign_storage::ParquetEncoder::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlinevirtual

Reimplemented in foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >, foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t, NullType >, foreign_storage::TypedParquetInPlaceEncoder< V, V >, foreign_storage::ParquetStringEncoder< V >, foreign_storage::ParquetArrayEncoder, and foreign_storage::ParquetFixedLengthArrayEncoder.

Definition at line 51 of file ParquetEncoder.h.

References createMetadata(), report::stats, foreign_storage::validate_and_get_column_metadata_statistics(), validate_metadata_stats_, and validateNullCount().

Referenced by foreign_storage::ParquetStringEncoder< V >::getRowGroupMetadata().

                                       {
     int64_t null_count{0};
     auto metadata = createMetadata(column_type);
 
     if (validate_metadata_stats_ && group_metadata->num_rows() > 0) {
       // update statistics
       auto column_metadata = group_metadata->ColumnChunk(parquet_column_index);
       auto stats = validate_and_get_column_metadata_statistics(column_metadata.get());
       null_count = stats->null_count();
       validateNullCount(group_metadata->schema()->Column(parquet_column_index)->name(),
                         null_count,
                         column_type);
     }
     metadata->chunkStats.has_nulls = null_count > 0;
 
     // update sizing
     metadata->numElements = group_metadata->num_rows();
     return metadata;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetEncoder::initializeColumnType ( const SQLTypeInfo & column_type )

inlinevirtual

Reimplemented in foreign_storage::ParquetArrayEncoder.

Definition at line 80 of file ParquetEncoder.h.

References column_type_.

Referenced by foreign_storage::ParquetArrayEncoder::initializeColumnType().

                                                                     {
     column_type_ = column_type;
   }

Here is the caller graph for this function:

virtual void foreign_storage::ParquetEncoder::initializeErrorTracking ( )

inlinevirtual

Reimplemented in foreign_storage::ParquetArrayEncoder.

Definition at line 78 of file ParquetEncoder.h.

References is_error_tracking_enabled_.

Referenced by foreign_storage::ParquetArrayEncoder::initializeErrorTracking().

78 { is_error_tracking_enabled_ = true; }

foreign_storage::ParquetEncoder::is_error_tracking_enabled_

bool is_error_tracking_enabled_

Definition: ParquetEncoder.h:88

Here is the caller graph for this function:

static void foreign_storage::ParquetEncoder::throwNotNullViolation ( const std::string & parquet_column_name )

inlinestaticprotected

Definition at line 107 of file ParquetEncoder.h.

Referenced by validateNullCount().

                                                                           {
     std::stringstream error_message;
     error_message << "A null value was detected in Parquet column '"
                   << parquet_column_name << "' but HeavyDB column is set to not null";
     throw std::runtime_error(error_message.str());
   }

Here is the caller graph for this function:

static void foreign_storage::ParquetEncoder::validateNullCount	(	const std::string &	parquet_column_name,
		int64_t	null_count,
		const SQLTypeInfo &	column_type
	)

inlinestaticprotected

Definition at line 114 of file ParquetEncoder.h.

References SQLTypeInfo::get_notnull(), and throwNotNullViolation().

Referenced by getRowGroupMetadata(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata().

                                                                 {
     bool has_nulls = null_count > 0;
     if (has_nulls && column_type.get_notnull()) {
       throwNotNullViolation(parquet_column_name);
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Data_Namespace::AbstractBuffer* foreign_storage::ParquetEncoder::buffer_

protected

Definition at line 85 of file ParquetEncoder.h.

SQLTypeInfo foreign_storage::ParquetEncoder::column_type_

protected

Definition at line 91 of file ParquetEncoder.h.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), foreign_storage::ParquetGeospatialEncoder::appendData(), foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::elementToString(), initializeColumnType(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::integralTypeToString(), foreign_storage::ParquetArrayEncoder::processLastArray(), foreign_storage::ParquetDetectStringEncoder::validateUsingEncodersColumnType(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::validateUsingEncodersColumnType().

size_t foreign_storage::ParquetEncoder::current_chunk_offset_

protected

Definition at line 90 of file ParquetEncoder.h.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), foreign_storage::ParquetGeospatialEncoder::appendData(), foreign_storage::ParquetDetectStringEncoder::appendData(), foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors().

RejectedRowIndices foreign_storage::ParquetEncoder::invalid_indices_

protected

Definition at line 89 of file ParquetEncoder.h.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), foreign_storage::ParquetGeospatialEncoder::appendData(), foreign_storage::ParquetDetectStringEncoder::appendData(), foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors(), getRejectedRowIndices(), and foreign_storage::ParquetArrayEncoder::processLastArray().

bool foreign_storage::ParquetEncoder::is_error_tracking_enabled_

protected

Definition at line 88 of file ParquetEncoder.h.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), foreign_storage::ParquetGeospatialEncoder::appendData(), foreign_storage::ParquetDetectStringEncoder::appendData(), foreign_storage::ParquetArrayEncoder::appendDataTrackErrors(), foreign_storage::ParquetStringEncoder< V >::appendDataTrackErrors(), foreign_storage::ParquetGeospatialEncoder::appendDataTrackErrors(), foreign_storage::ParquetStringNoneEncoder::appendDataTrackErrors(), foreign_storage::ParquetDetectStringEncoder::appendDataTrackErrors(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors(), initializeErrorTracking(), foreign_storage::ParquetArrayEncoder::processLastArray(), foreign_storage::ParquetArrayEncoder::resetLastArrayMetadata(), and foreign_storage::ParquetArrayEncoder::updateMetadataForAppendedArrayItem().

bool foreign_storage::ParquetEncoder::validate_metadata_stats_

protected

Definition at line 94 of file ParquetEncoder.h.

Referenced by disableMetadataStatsValidation(), getRowGroupMetadata(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetEncoder.h

Public Member Functions

Static Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation