OmniSciDB
a5dc49c757
|
#include <Encoder.h>
Public Member Functions | |
Encoder (Data_Namespace::AbstractBuffer *buffer) | |
virtual | ~Encoder () |
virtual size_t | getNumElemsForBytesEncodedDataAtIndices (const int8_t *index_data, const std::vector< size_t > &selected_idx, const size_t byte_limit)=0 |
virtual std::shared_ptr < ChunkMetadata > | appendEncodedDataAtIndices (const int8_t *index_data, int8_t *data, const std::vector< size_t > &selected_idx)=0 |
virtual std::shared_ptr < ChunkMetadata > | appendEncodedData (const int8_t *index_data, int8_t *data, const size_t start_idx, const size_t num_elements)=0 |
virtual std::shared_ptr < ChunkMetadata > | appendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1)=0 |
virtual void | getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata) |
virtual std::shared_ptr < ChunkMetadata > | getMetadata (const SQLTypeInfo &ti)=0 |
virtual void | updateStats (const int64_t val, const bool is_null)=0 |
virtual void | updateStats (const double val, const bool is_null)=0 |
virtual void | updateStats (const int8_t *const src_data, const size_t num_elements)=0 |
virtual void | updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements) |
virtual void | updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements)=0 |
virtual void | updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements)=0 |
virtual void | reduceStats (const Encoder &)=0 |
virtual void | copyMetadata (const Encoder *copyFromEncoder)=0 |
virtual void | writeMetadata (FILE *f)=0 |
virtual void | readMetadata (FILE *f)=0 |
virtual bool | resetChunkStats (const ChunkStats &) |
: Reset chunk level stats (min, max, nulls) using new values from the argument. More... | |
virtual void | resetChunkStats ()=0 |
size_t | getNumElems () const |
void | setNumElems (const size_t num_elems) |
Static Public Member Functions | |
static Encoder * | Create (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType) |
Protected Attributes | |
size_t | num_elems_ |
Data_Namespace::AbstractBuffer * | buffer_ |
DecimalOverflowValidator | decimal_overflow_validator_ |
DateDaysOverflowValidator | date_days_overflow_validator_ |
Encoder::Encoder | ( | Data_Namespace::AbstractBuffer * | buffer | ) |
Definition at line 225 of file Encoder.cpp.
|
pure virtual |
Append data to the chunk buffer backing this encoder.
src_data | Source data for the append |
num_elems_to_append | Number of elements to append |
ti | SQL Type Info for the column TODO(adb): used? |
replicating | Pass one value and fill the chunk with it |
offset | Write data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to num_elems_to_append . |
Implemented in FixedLengthArrayNoneEncoder, ArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.
Referenced by Chunk_NS::Chunk::appendData().
|
pure virtual |
Append encoded data to the chunk buffer backing this encoder.
index_data | - (optional) the index data of data to append |
data | - the data to append |
start_idx | - the position to start encoding from in the data array |
num_elements | - the number of elements to encode from the data array |
NOTE: index_data
must be non-null for varlen encoder types.
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, StringNoneEncoder, and DateDaysEncoder< T, V >.
Referenced by Chunk_NS::Chunk::appendEncodedData().
|
pure virtual |
Append selected encoded data to the chunk buffer backing this encoder.
index_data | - (optional) the index data of data to append |
data | - the data to append |
selected_idx | - which indices in the encoded data to append |
NOTE: index_data
must be non-null for varlen encoder types.
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, StringNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, and DateDaysEncoder< T, V >.
Referenced by Chunk_NS::Chunk::appendEncodedDataAtIndices().
|
pure virtual |
|
static |
Definition at line 26 of file Encoder.cpp.
References CHECK, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.
Referenced by Data_Namespace::AbstractBuffer::initEncoder(), and synthesize_metadata().
|
virtual |
Reimplemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, StringNoneEncoder, and DateDaysEncoder< T, V >.
Definition at line 231 of file Encoder.cpp.
References buffer_, Data_Namespace::AbstractBuffer::getSqlType(), num_elems_, and Data_Namespace::AbstractBuffer::size().
Referenced by foreign_storage::ForeignStorageCache::cacheMetadataVec(), foreign_storage::get_placeholder_metadata(), foreign_storage::Csv::get_placeholder_metadata(), DateDaysEncoder< T, V >::getMetadata(), StringNoneEncoder::getMetadata(), FixedLengthEncoder< T, V >::getMetadata(), NoneEncoder< T >::getMetadata(), FixedLengthArrayNoneEncoder::getMetadata(), and ArrayNoneEncoder::getMetadata().
|
pure virtual |
|
inline |
Definition at line 284 of file Encoder.h.
References num_elems_.
Referenced by Fragmenter_Namespace::compute_row_indices_of_shards(), StringNoneEncoder::copyMetadata(), DateDaysEncoder< T, V >::copyMetadata(), FixedLengthEncoder< T, V >::copyMetadata(), NoneEncoder< T >::copyMetadata(), FixedLengthArrayNoneEncoder::copyMetadata(), and ArrayNoneEncoder::copyMetadata().
|
pure virtual |
Compute the maximum number of variable length encoded elements given a byte limit
index_data | - (optional) index data for the encoded type |
selected_idx | - which indices in the encoded data to consider |
byte_limit | - byte limit that must be respected |
NOTE: optional parameters above may be ignored by the implementation, but may or may not be required depending on the encoder type backing the implementation.
Implemented in ArrayNoneEncoder, StringNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, and DateDaysEncoder< T, V >.
Referenced by Chunk_NS::Chunk::getNumElemsForBytesEncodedDataAtIndices().
|
pure virtual |
|
pure virtual |
|
inlinevirtual |
: Reset chunk level stats (min, max, nulls) using new values from the argument.
Reimplemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, NoneEncoder< T >, DateDaysEncoder< T, V >, and StringNoneEncoder.
Definition at line 274 of file Encoder.h.
References UNREACHABLE.
Referenced by foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().
|
pure virtual |
Resets chunk metadata stats to their default values.
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, NoneEncoder< T >, DateDaysEncoder< T, V >, and StringNoneEncoder.
|
inline |
Definition at line 285 of file Encoder.h.
References num_elems_.
Referenced by DBHandler::insert_chunks(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().
|
pure virtual |
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.
Referenced by foreign_storage::update_stats().
|
pure virtual |
|
pure virtual |
Update statistics for data without appending.
src_data | - the data with which to update statistics |
num_elements | - the number of elements to scan in the data |
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.
|
pure virtual |
Update statistics for string data without appending.
src_data | - the string data with which to update statistics |
start_idx | - the offset into src_data to start the update |
num_elements | - the number of elements to scan in the string data |
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, NoneEncoder< T >, DateDaysEncoder< T, V >, and StringNoneEncoder.
|
pure virtual |
Update statistics for array data without appending.
src_data | - the array data with which to update statistics |
start_idx | - the offset into src_data to start the update |
num_elements | - the number of elements to scan in the array data |
Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, NoneEncoder< T >, DateDaysEncoder< T, V >, and StringNoneEncoder.
|
inlinevirtual |
Update statistics for encoded data without appending.
dst_data | - the data with which to update statistics |
num_elements | - the number of elements to scan in the data |
Reimplemented in FixedLengthEncoder< T, V >, and NoneEncoder< T >.
Definition at line 236 of file Encoder.h.
References UNREACHABLE.
|
pure virtual |
|
protected |
Definition at line 290 of file Encoder.h.
Referenced by StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), DateDaysEncoder< T, V >::appendEncodedOrUnencodedData(), FixedLengthEncoder< T, V >::appendEncodedOrUnencodedData(), NoneEncoder< T >::appendValidatedOrNonValidatedData(), getMetadata(), FixedLengthArrayNoneEncoder::is_null(), FixedLengthArrayNoneEncoder::is_null_ignore_not_null(), FixedLengthArrayNoneEncoder::resetChunkStats(), ArrayNoneEncoder::resetChunkStats(), FixedLengthArrayNoneEncoder::update_elem_stats(), and ArrayNoneEncoder::update_elem_stats().
|
protected |
Definition at line 293 of file Encoder.h.
Referenced by DateDaysEncoder< T, V >::encodeDataAndUpdateStats().
|
protected |
Definition at line 292 of file Encoder.h.
Referenced by FixedLengthEncoder< T, V >::encodeDataAndUpdateStats(), FixedLengthArrayNoneEncoder::update_elem_stats(), NoneEncoder< T >::updateStatsEncoded(), FixedLengthEncoder< T, V >::updateStatsEncoded(), and NoneEncoder< T >::validateDataAndUpdateStats().
|
protected |
Definition at line 288 of file Encoder.h.
Referenced by StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), DateDaysEncoder< T, V >::appendEncodedOrUnencodedData(), FixedLengthEncoder< T, V >::appendEncodedOrUnencodedData(), NoneEncoder< T >::appendValidatedOrNonValidatedData(), StringNoneEncoder::copyMetadata(), DateDaysEncoder< T, V >::copyMetadata(), FixedLengthEncoder< T, V >::copyMetadata(), NoneEncoder< T >::copyMetadata(), FixedLengthArrayNoneEncoder::copyMetadata(), ArrayNoneEncoder::copyMetadata(), getMetadata(), getNumElems(), StringNoneEncoder::readMetadata(), DateDaysEncoder< T, V >::readMetadata(), NoneEncoder< T >::readMetadata(), FixedLengthEncoder< T, V >::readMetadata(), FixedLengthArrayNoneEncoder::readMetadata(), ArrayNoneEncoder::readMetadata(), setNumElems(), StringNoneEncoder::writeMetadata(), DateDaysEncoder< T, V >::writeMetadata(), NoneEncoder< T >::writeMetadata(), FixedLengthEncoder< T, V >::writeMetadata(), FixedLengthArrayNoneEncoder::writeMetadata(), and ArrayNoneEncoder::writeMetadata().