#include <StringNoneEncoder.h>

Inheritance diagram for StringNoneEncoder:

Collaboration diagram for StringNoneEncoder:

Public Member Functions
	StringNoneEncoder (AbstractBuffer *buffer)

size_t	getNumElemsForBytesInsertData (const std::vector< std::string > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)

size_t	getNumElemsForBytesEncodedDataAtIndices (const int8_t *index_data, const std::vector< size_t > &selected_idx, const size_t byte_limit) override

std::shared_ptr< ChunkMetadata >	appendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1) override

std::shared_ptr< ChunkMetadata >	appendEncodedDataAtIndices (const int8_t index_data, int8_t data, const std::vector< size_t > &selected_idx) override

std::shared_ptr< ChunkMetadata >	appendEncodedData (const int8_t index_data, int8_t data, const size_t start_idx, const size_t num_elements) override

template<typename StringType >
std::shared_ptr< ChunkMetadata >	appendData (const StringType *srcData, const int start_idx, const size_t numAppendElems, const bool replicating=false)

template<typename StringType >
std::shared_ptr< ChunkMetadata >	appendData (const std::vector< StringType > *srcData, const int start_idx, const size_t numAppendElems, const bool replicating=false)

void	getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata) override

std::shared_ptr< ChunkMetadata >	getMetadata (const SQLTypeInfo &ti) override

void	updateStats (const int64_t, const bool) override

void	updateStats (const double, const bool) override

void	updateStats (const int8_t *const src_data, const size_t num_elements) override

void	updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override

void	updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override

void	reduceStats (const Encoder &) override

void	writeMetadata (FILE *f) override

void	readMetadata (FILE *f) override

void	copyMetadata (const Encoder *copyFromEncoder) override

AbstractBuffer *	getIndexBuf () const

void	setIndexBuffer (AbstractBuffer *buf)

bool	resetChunkStats (const ChunkStats &stats) override
	: Reset chunk level stats (min, max, nulls) using new values from the argument. More...

void	resetChunkStats () override

Public Member Functions inherited from Encoder
	Encoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~Encoder ()

virtual void	updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements)

size_t	getNumElems () const

void	setNumElems (const size_t num_elems)

Static Public Member Functions
static std::string_view	getStringAtIndex (const int8_t index_data, const int8_t data, size_t index)

Static Public Member Functions inherited from Encoder
static Encoder *	Create (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)

Private Member Functions
template<typename StringType >
void	update_elem_stats (const StringType &elem)

Static Private Member Functions
static std::pair < StringOffsetT, StringOffsetT >	getStringOffsets (const int8_t *index_data, size_t index)

static size_t	getStringSizeAtIndex (const int8_t *index_data, size_t index)

Private Attributes
AbstractBuffer *	index_buf

StringOffsetT	last_offset

bool	has_nulls

Additional Inherited Members
Protected Attributes inherited from Encoder
size_t	num_elems_

Data_Namespace::AbstractBuffer *	buffer_

DecimalOverflowValidator	decimal_overflow_validator_

DateDaysOverflowValidator	date_days_overflow_validator_

Detailed Description

Definition at line 36 of file StringNoneEncoder.h.

Constructor & Destructor Documentation

StringNoneEncoder::StringNoneEncoder ( AbstractBuffer * buffer )

inline

Definition at line 38 of file StringNoneEncoder.h.

39 : Encoder(buffer), index_buf(nullptr), last_offset(-1), has_nulls(false) {}

StringNoneEncoder::has_nulls

bool has_nulls

Definition: StringNoneEncoder.h:150

StringNoneEncoder::index_buf

AbstractBuffer * index_buf

Definition: StringNoneEncoder.h:148

Encoder::Encoder

Encoder(Data_Namespace::AbstractBuffer *buffer)

Definition: Encoder.cpp:225

StringNoneEncoder::last_offset

StringOffsetT last_offset

Definition: StringNoneEncoder.h:149

Member Function Documentation

std::shared_ptr<ChunkMetadata> StringNoneEncoder::appendData	(	int8_t *&	src_data,
		const size_t	num_elems_to_append,
		const SQLTypeInfo &	ti,
		const bool	replicating = `false`,
		const int64_t	offset = `-1`
	)

inlineoverridevirtual

Append data to the chunk buffer backing this encoder.

Parameters

src_data	Source data for the append
num_elems_to_append	Number of elements to append
ti	SQL Type Info for the column TODO(adb): used?
replicating	Pass one value and fill the chunk with it
offset	Write data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to `num_elems_to_append`.

Implements Encoder.

Definition at line 51 of file StringNoneEncoder.h.

References UNREACHABLE.

Referenced by foreign_storage::GeospatialEncoder::appendBaseDataAndUpdateMetadata(), appendData(), Chunk_NS::Chunk::appendData(), appendEncodedData(), appendEncodedDataAtIndices(), and data_conversion::StringViewToStringNoneEncoder::encodeAndAppendData().

                                                                                 {
     UNREACHABLE();  // should never be called for strings
     return nullptr;
   }

Here is the caller graph for this function:

template<typename StringType >

std::shared_ptr< ChunkMetadata > StringNoneEncoder::appendData	(	const StringType *	srcData,
		const int	start_idx,
		const size_t	numAppendElems,
		const bool	replicating = `false`
	)

Definition at line 102 of file StringNoneEncoder.cpp.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, CHECK, CHECK_GE, gpu_enabled::copy(), Data_Namespace::CPU_LEVEL, run_benchmark_import::dest, getMetadata(), index_buf, Data_Namespace::AbstractBuffer::isDirty(), last_offset, MAX_INPUT_BUF_SIZE, anonymous_namespace{Utm.h}::n, Encoder::num_elems_, Data_Namespace::AbstractBuffer::read(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setDirty(), Data_Namespace::AbstractBuffer::size(), and update_elem_stats().

                                                                                      {
   CHECK(index_buf);  // index_buf must be set before this.
   size_t append_index_size = numAppendElems * sizeof(StringOffsetT);
   if (num_elems_ == 0) {
     append_index_size += sizeof(StringOffsetT);  // plus one for the initial offset of 0.
   }
   index_buf->reserve(index_buf->size() + append_index_size);
   StringOffsetT offset = 0;
   if (num_elems_ == 0) {
     index_buf->append((int8_t*)&offset,
                       sizeof(StringOffsetT));  // write the inital 0 offset
     last_offset = 0;
   } else {
     // always need to read a valid last offset from buffer/disk
     // b/c now due to vacuum "last offset" may go backward and if
     // index chunk was not reloaded last_offset would go way off!
     index_buf->read((int8_t*)&last_offset,
                     sizeof(StringOffsetT),
                     index_buf->size() - sizeof(StringOffsetT),
                     Data_Namespace::CPU_LEVEL);
     CHECK_GE(last_offset, 0);
   }
   size_t append_data_size = 0;
   for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
     size_t len = (srcData)[replicating ? 0 : n].length();
     append_data_size += len;
   }
   buffer_->reserve(buffer_->size() + append_data_size);
 
   size_t inbuf_size =
       std::min(std::max(append_index_size, append_data_size), (size_t)MAX_INPUT_BUF_SIZE);
   auto inbuf = std::make_unique<int8_t[]>(inbuf_size);
   for (size_t num_appended = 0; num_appended < numAppendElems;) {
     StringOffsetT* p = reinterpret_cast<StringOffsetT*>(inbuf.get());
     size_t i;
     for (i = 0; num_appended < numAppendElems && i < inbuf_size / sizeof(StringOffsetT);
          i++, num_appended++) {
       p[i] = last_offset + (srcData)[replicating ? 0 : num_appended + start_idx].length();
       last_offset = p[i];
     }
     index_buf->append(inbuf.get(), i * sizeof(StringOffsetT));
   }
 
   for (size_t num_appended = 0; num_appended < numAppendElems;) {
     size_t size = 0;
     for (int i = start_idx + num_appended;
          num_appended < numAppendElems && size < inbuf_size;
          i++, num_appended++) {
       size_t len = (srcData)[replicating ? 0 : i].length();
       if (len > inbuf_size) {
         // for large strings, append on its own
         if (size > 0) {
           buffer_->append(inbuf.get(), size);
         }
         size = 0;
         buffer_->append((int8_t*)(srcData)[replicating ? 0 : i].data(), len);
         num_appended++;
         break;
       } else if (size + len > inbuf_size) {
         break;
       }
       char* dest = reinterpret_cast<char*>(inbuf.get()) + size;
       if (len > 0) {
         (srcData)[replicating ? 0 : i].copy(dest, len);
         size += len;
       }
       update_elem_stats((srcData)[replicating ? 0 : i]);
     }
     if (size > 0) {
       buffer_->append(inbuf.get(), size);
     }
   }
   // make sure buffer_ is flushed even if no new data is appended to it
   // (e.g. empty strings) because the metadata needs to be flushed.
   if (!buffer_->isDirty()) {
     buffer_->setDirty();
   }
 
   num_elems_ += numAppendElems;
   auto chunk_metadata = std::make_shared<ChunkMetadata>();
   getMetadata(chunk_metadata);
   return chunk_metadata;
 }

Here is the call graph for this function:

template<typename StringType >

std::shared_ptr< ChunkMetadata > StringNoneEncoder::appendData	(	const std::vector< StringType > *	srcData,
		const int	start_idx,
		const size_t	numAppendElems,
		const bool	replicating = `false`
	)

Definition at line 93 of file StringNoneEncoder.cpp.

References appendData().

                             {
   return appendData(srcData->data(), start_idx, numAppendElems, replicating);
 }

Here is the call graph for this function:

std::shared_ptr< ChunkMetadata > StringNoneEncoder::appendEncodedData	(	const int8_t *	index_data,
		int8_t *	data,
		const size_t	start_idx,
		const size_t	num_elements
	)

overridevirtual

Append encoded data to the chunk buffer backing this encoder.

Parameters

index_data	- (optional) the index data of data to append
data	- the data to append
start_idx	- the position to start encoding from in the `data` array
num_elements	- the number of elements to encode from the `data` array

Returns: updated chunk metadata for the chunk buffer backing this encoder

NOTE: index_data must be non-null for varlen encoder types.

Implements Encoder.

Definition at line 78 of file StringNoneEncoder.cpp.

References appendData(), and getStringAtIndex().

                                {
   std::vector<std::string_view> data_subset;
   data_subset.reserve(num_elements);
   for (size_t count = 0; count < num_elements; ++count) {
     auto current_index = start_idx + count;
     data_subset.emplace_back(getStringAtIndex(index_data, data, current_index));
   }
   return appendData(&data_subset, 0, num_elements, false);
 }

Here is the call graph for this function:

std::shared_ptr< ChunkMetadata > StringNoneEncoder::appendEncodedDataAtIndices	(	const int8_t *	index_data,
		int8_t *	data,
		const std::vector< size_t > &	selected_idx
	)

overridevirtual

Append selected encoded data to the chunk buffer backing this encoder.

Parameters

index_data	- (optional) the index data of data to append
data	- the data to append
selected_idx	- which indices in the encoded data to append

Returns: updated chunk metadata for the chunk buffer backing this encoder

NOTE: index_data must be non-null for varlen encoder types.

Implements Encoder.

Definition at line 66 of file StringNoneEncoder.cpp.

References appendData(), and getStringAtIndex().

                                            {
   std::vector<std::string_view> data_subset;
   data_subset.reserve(selected_idx.size());
   for (const auto& offset_index : selected_idx) {
     data_subset.emplace_back(getStringAtIndex(index_data, data, offset_index));
   }
   return appendData(&data_subset, 0, selected_idx.size(), false);
 }

Here is the call graph for this function:

void StringNoneEncoder::copyMetadata ( const Encoder * copyFromEncoder )

inlineoverridevirtual

Implements Encoder.

Definition at line 119 of file StringNoneEncoder.h.

References Encoder::getNumElems(), has_nulls, and Encoder::num_elems_.

                                                              {
     num_elems_ = copyFromEncoder->getNumElems();
     has_nulls = static_cast<const StringNoneEncoder*>(copyFromEncoder)->has_nulls;
   }

Here is the call graph for this function:

AbstractBuffer* StringNoneEncoder::getIndexBuf ( ) const

inline

Definition at line 124 of file StringNoneEncoder.h.

References index_buf.

124 { return index_buf; }

StringNoneEncoder::index_buf

AbstractBuffer * index_buf

Definition: StringNoneEncoder.h:148

void StringNoneEncoder::getMetadata ( const std::shared_ptr< ChunkMetadata > & chunkMetadata )

overridevirtual

Reimplemented from Encoder.

Definition at line 262 of file StringNoneEncoder.cpp.

References Encoder::getMetadata(), and has_nulls.

Referenced by appendData().

                                                                                      {
   Encoder::getMetadata(chunkMetadata);  // call on parent class
   chunkMetadata->chunkStats.min.stringval = nullptr;
   chunkMetadata->chunkStats.max.stringval = nullptr;
   chunkMetadata->chunkStats.has_nulls = has_nulls;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > StringNoneEncoder::getMetadata ( const SQLTypeInfo & ti )

overridevirtual

Implements Encoder.

Definition at line 270 of file StringNoneEncoder.cpp.

References has_nulls, ChunkStats::min, and Datum::stringval.

                                                                                  {
   auto chunk_stats = ChunkStats{};
   chunk_stats.min.stringval = nullptr;
   chunk_stats.max.stringval = nullptr;
   chunk_stats.has_nulls = has_nulls;
   return std::make_shared<ChunkMetadata>(ti, 0, 0, chunk_stats);
 }

size_t StringNoneEncoder::getNumElemsForBytesEncodedDataAtIndices	(	const int8_t *	index_data,
		const std::vector< size_t > &	selected_idx,
		const size_t	byte_limit
	)

overridevirtual

Compute the maximum number of variable length encoded elements given a byte limit

Parameters

index_data	- (optional) index data for the encoded type
selected_idx	- which indices in the encoded data to consider
byte_limit	- byte limit that must be respected

Returns: the number of elements

NOTE: optional parameters above may be ignored by the implementation, but may or may not be required depending on the encoder type backing the implementation.

Implements Encoder.

Definition at line 49 of file StringNoneEncoder.cpp.

References getStringSizeAtIndex().

                              {
   size_t num_elements = 0;
   size_t data_size = 0;
   for (const auto& offset_index : selected_idx) {
     auto element_size = getStringSizeAtIndex(index_data, offset_index);
     if (data_size + element_size > byte_limit) {
       break;
     }
     data_size += element_size;
     num_elements++;
   }
   return num_elements;
 }

Here is the call graph for this function:

size_t StringNoneEncoder::getNumElemsForBytesInsertData	(	const std::vector< std::string > *	srcData,
		const int	start_idx,
		const size_t	numAppendElems,
		const size_t	byteLimit,
		const bool	replicating = `false`
	)

Definition at line 31 of file StringNoneEncoder.cpp.

References anonymous_namespace{Utm.h}::n.

Referenced by Chunk_NS::Chunk::getNumElemsForBytesInsertData().

                             {
   size_t dataSize = 0;
   size_t n = start_idx;
   for (; n < start_idx + numAppendElems; n++) {
     size_t len = (*srcData)[replicating ? 0 : n].length();
     if (dataSize + len > byteLimit) {
       break;
     }
     dataSize += len;
   }
   return n - start_idx;
 }

Here is the caller graph for this function:

std::string_view StringNoneEncoder::getStringAtIndex	(	const int8_t *	index_data,
		const int8_t *	data,
		size_t	index
	)

static

Definition at line 225 of file StringNoneEncoder.cpp.

References getStringOffsets(), and last_offset.

Referenced by appendEncodedData(), appendEncodedDataAtIndices(), and data_conversion::StringViewSource::getSourceData().

                                                                    {
   auto [offset, last_offset] = getStringOffsets(index_data, index);
   size_t string_byte_size = offset - last_offset;
   auto current_data = reinterpret_cast<const char*>(data + last_offset);
   return std::string_view{current_data, string_byte_size};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair< StringOffsetT, StringOffsetT > StringNoneEncoder::getStringOffsets	(	const int8_t *	index_data,
		size_t	index
	)

staticprivate

Definition at line 207 of file StringNoneEncoder.cpp.

References CHECK, and last_offset.

Referenced by getStringAtIndex(), and getStringSizeAtIndex().

                   {
   auto string_offsets = reinterpret_cast<const StringOffsetT*>(index_data);
   auto current_index = index + 1;
   auto offset = string_offsets[current_index];
   CHECK(offset >= 0);
   int64_t last_offset = string_offsets[current_index - 1];
   CHECK(last_offset >= 0 && last_offset <= offset);
   return {offset, last_offset};
 }

Here is the caller graph for this function:

size_t StringNoneEncoder::getStringSizeAtIndex	(	const int8_t *	index_data,
		size_t	index
	)

staticprivate

Definition at line 219 of file StringNoneEncoder.cpp.

References getStringOffsets(), and last_offset.

Referenced by getNumElemsForBytesEncodedDataAtIndices().

                                                                                      {
   auto [offset, last_offset] = getStringOffsets(index_data, index);
   size_t string_byte_size = offset - last_offset;
   return string_byte_size;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void StringNoneEncoder::readMetadata ( FILE * f )

inlineoverridevirtual

Implements Encoder.

Definition at line 113 of file StringNoneEncoder.h.

References CHECK_NE, has_nulls, and Encoder::num_elems_.

                                       {
     // assumes pointer is already in right place
     CHECK_NE(fread((int8_t*)&num_elems_, sizeof(size_t), size_t(1), f), size_t(0));
     CHECK_NE(fread((int8_t*)&has_nulls, sizeof(bool), size_t(1), f), size_t(0));
   }

void StringNoneEncoder::reduceStats ( const Encoder & )

inlineoverridevirtual

Implements Encoder.

Definition at line 105 of file StringNoneEncoder.h.

References CHECK.

105 { CHECK(false); }

CHECK

#define CHECK(condition)

Definition: Logger.h:291

bool StringNoneEncoder::resetChunkStats ( const ChunkStats & )

inlineoverridevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns: : True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented from Encoder.

Definition at line 127 of file StringNoneEncoder.h.

References ChunkStats::has_nulls, and has_nulls.

                                                          {
     if (has_nulls == stats.has_nulls) {
       return false;
     }
     has_nulls = stats.has_nulls;
     return true;
   }

void StringNoneEncoder::resetChunkStats ( )

inlineoverridevirtual

Resets chunk metadata stats to their default values.

Implements Encoder.

Definition at line 135 of file StringNoneEncoder.h.

References has_nulls.

135 { has_nulls = false; }

StringNoneEncoder::has_nulls

bool has_nulls

Definition: StringNoneEncoder.h:150

void StringNoneEncoder::setIndexBuffer ( AbstractBuffer * buf )

inline

Definition at line 125 of file StringNoneEncoder.h.

References index_buf.

Referenced by Chunk_NS::Chunk::initEncoder().

125 { index_buf = buf; }

StringNoneEncoder::index_buf

AbstractBuffer * index_buf

Definition: StringNoneEncoder.h:148

Here is the caller graph for this function:

template<typename StringType >

void StringNoneEncoder::update_elem_stats ( const StringType & elem )

private

Definition at line 201 of file StringNoneEncoder.cpp.

References has_nulls.

Referenced by appendData(), and updateStats().

                                                                 {
   if (!has_nulls && elem.empty()) {
     has_nulls = true;
   }
 }

Here is the caller graph for this function:

void StringNoneEncoder::updateStats	(	const int64_t	,
		const bool
	)

inlineoverridevirtual

Implements Encoder.

Definition at line 87 of file StringNoneEncoder.h.

References CHECK.

87 { CHECK(false); }

CHECK

#define CHECK(condition)

Definition: Logger.h:291

void StringNoneEncoder::updateStats	(	const double	,
		const bool
	)

inlineoverridevirtual

Implements Encoder.

Definition at line 89 of file StringNoneEncoder.h.

References CHECK.

89 { CHECK(false); }

CHECK

#define CHECK(condition)

Definition: Logger.h:291

void StringNoneEncoder::updateStats	(	const int8_t *const	src_data,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for data without appending.

Parameters

src_data	- the data with which to update statistics
num_elements	- the number of elements to scan in the data

Implements Encoder.

Definition at line 91 of file StringNoneEncoder.h.

References UNREACHABLE.

                                                                                      {
     UNREACHABLE();
   }

void StringNoneEncoder::updateStats	(	const std::vector< std::string > *const	src_data,
		const size_t	start_idx,
		const size_t	num_elements
	)

overridevirtual

Update statistics for string data without appending.

Parameters

src_data	- the string data with which to update statistics
start_idx	- the offset into `src_data` to start the update
num_elements	- the number of elements to scan in the string data

Implements Encoder.

Definition at line 189 of file StringNoneEncoder.cpp.

References has_nulls, anonymous_namespace{Utm.h}::n, and update_elem_stats().

                                                                {
   for (size_t n = start_idx; n < start_idx + num_elements; n++) {
     update_elem_stats((*src_data)[n]);
     if (has_nulls) {
       break;
     }
   }
 }

Here is the call graph for this function:

void StringNoneEncoder::updateStats	(	const std::vector< ArrayDatum > *const	src_data,
		const size_t	start_idx,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for array data without appending.

Parameters

src_data	- the array data with which to update statistics
start_idx	- the offset into `src_data` to start the update
num_elements	- the number of elements to scan in the array data

Implements Encoder.

Definition at line 99 of file StringNoneEncoder.h.

References UNREACHABLE.

                                                        {
     UNREACHABLE();
   }

void StringNoneEncoder::writeMetadata ( FILE * f )

inlineoverridevirtual

Implements Encoder.

Definition at line 107 of file StringNoneEncoder.h.

References has_nulls, and Encoder::num_elems_.

                                        {
     // assumes pointer is already in right place
     fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
     fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
   }

Member Data Documentation

bool StringNoneEncoder::has_nulls

private

Definition at line 150 of file StringNoneEncoder.h.

Referenced by copyMetadata(), getMetadata(), heavydb.thrift.ttypes.TColumnRange::read(), readMetadata(), resetChunkStats(), update_elem_stats(), updateStats(), heavydb.thrift.ttypes.TColumnRange::write(), and writeMetadata().

AbstractBuffer* StringNoneEncoder::index_buf

private

Definition at line 148 of file StringNoneEncoder.h.

Referenced by appendData(), getIndexBuf(), and setIndexBuffer().

StringOffsetT StringNoneEncoder::last_offset

private

Definition at line 149 of file StringNoneEncoder.h.

Referenced by appendData(), getStringAtIndex(), getStringOffsets(), and getStringSizeAtIndex().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/StringNoneEncoder.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/StringNoneEncoder.cpp

Public Member Functions

Static Public Member Functions

Private Member Functions

Static Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation