#include <TargetValueConvertersImpl.h>

Inheritance diagram for DictionaryValueConverter< TARGET_TYPE >:

Collaboration diagram for DictionaryValueConverter< TARGET_TYPE >:

Public Types
using	ElementsDataColumnPtr = typename NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr

using	ElementsBufferColumnPtr = std::unique_ptr< std::vector< int32_t >>

Public Types inherited from NumericValueConverter< int64_t, TARGET_TYPE >
using	ColumnDataPtr = std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE >>

using	ElementsBufferColumnPtr = ColumnDataPtr

using	CasterFunc = std::function< TARGET_TYPE(int64_t, bool, TARGET_TYPE)>

Public Member Functions
	DictionaryValueConverter (const shared::StringDictKey &source_dict_key, const ColumnDescriptor targetDescriptor, const Catalog_Namespace::Catalog &target_cat, size_t num_rows, TARGET_TYPE nullValue, int64_t nullCheckValue, bool doNullCheck, StringDictionaryProxy literals_dict, StringDictionaryProxy *source_dict_proxy)

	~DictionaryValueConverter () override

ElementsBufferColumnPtr	allocateColumnarBuffer (size_t num_rows)

void	convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)

void	convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)

void	convertToColumnarFormat (size_t row, const TargetValue *value) override

int32_t	convertTransientStringIdToPermanentId (int32_t &transient_string_id)

NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr	processBuffer (ElementsBufferColumnPtr buffer)

void	processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)

void	finalizeDataBlocksForInsertData () override

void	addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override

Public Member Functions inherited from NumericValueConverter< int64_t, TARGET_TYPE >
	NumericValueConverter (const ColumnDescriptor *cd, size_t num_rows, TARGET_TYPE nullValue, int64_tnullCheckValue, bool doNullCheck)

	~NumericValueConverter () override

void	setValueCaster (CasterFunc caster)

void	populateFixedArrayNullSentinel (size_t num_rows)

void	allocateColumnarData (size_t num_rows) override

ElementsBufferColumnPtr	allocateColumnarBuffer (size_t num_rows)

void	convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnData, const ScalarTargetValue *scalarValue)

void	convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)

void	convertToColumnarFormat (size_t row, const TargetValue *value) override

void	processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)

void	addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override

Public Member Functions inherited from TargetValueConverter
	TargetValueConverter (const ColumnDescriptor *cd)

virtual	~TargetValueConverter ()

Public Attributes
ElementsBufferColumnPtr	column_buffer_

const DictDescriptor *	target_dict_desc_

const DictDescriptor *	source_dict_desc_

StringDictionaryProxy *	source_dict_proxy_

const int32_t	buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()

const StringDictionaryProxy *	literals_dict_

std::unordered_map< int32_t, int32_t >	literals_lookup_

bool	use_literals_

Public Attributes inherited from NumericValueConverter< int64_t, TARGET_TYPE >
ColumnDataPtr	column_data_

TARGET_TYPE	null_value_

int64_t	null_check_value_

bool	do_null_check_

TARGET_TYPE	fixed_array_null_value_

CasterFunc	checked_caster_

boost_variant_accessor< int64_t >	SOURCE_TYPE_ACCESSOR

Public Attributes inherited from TargetValueConverter
const ColumnDescriptor *	column_descriptor_

boost_variant_accessor < ScalarTargetValue >	SCALAR_TARGET_VALUE_ACCESSOR

boost_variant_accessor < ArrayTargetValue >	ARRAY_TARGET_VALUE_ACCESSOR

boost_variant_accessor < GeoTargetValue >	GEO_TARGET_VALUE_ACCESSOR

boost_variant_accessor < NullableString >	NULLABLE_STRING_ACCESSOR

boost_variant_accessor < std::string >	STRING_ACCESSOR

Detailed Description

template<typename TARGET_TYPE>
struct DictionaryValueConverter< TARGET_TYPE >

Definition at line 160 of file TargetValueConvertersImpl.h.

Member Typedef Documentation

template<typename TARGET_TYPE >

using DictionaryValueConverter< TARGET_TYPE >::ElementsBufferColumnPtr = std::unique_ptr<std::vector<int32_t>>

Definition at line 164 of file TargetValueConvertersImpl.h.

template<typename TARGET_TYPE >

using DictionaryValueConverter< TARGET_TYPE >::ElementsDataColumnPtr = typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr

Definition at line 162 of file TargetValueConvertersImpl.h.

Constructor & Destructor Documentation

template<typename TARGET_TYPE >

DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter	(	const shared::StringDictKey &	source_dict_key,
		const ColumnDescriptor *	targetDescriptor,
		const Catalog_Namespace::Catalog &	target_cat,
		size_t	num_rows,
		TARGET_TYPE	nullValue,
		int64_t	nullCheckValue,
		bool	doNullCheck,
		StringDictionaryProxy *	literals_dict,
		StringDictionaryProxy *	source_dict_proxy
	)

inline

Definition at line 180 of file TargetValueConvertersImpl.h.

       : NumericValueConverter<int64_t, TARGET_TYPE>(targetDescriptor,
                                                     num_rows,
                                                     nullValue,
                                                     nullCheckValue,
                                                     doNullCheck) {
     literals_dict_ = literals_dict;
     const auto& target_dict_key = targetDescriptor->columnType.getStringDictKey();
     CHECK_EQ(target_dict_key.db_id, target_cat.getDatabaseId());
     target_dict_desc_ = target_cat.getMetadataForDict(target_dict_key.dict_id, true);
 
     source_dict_desc_ = nullptr;
     source_dict_proxy_ = source_dict_proxy;
 
     use_literals_ = 0 == source_dict_key.dict_id;
     if (!use_literals_) {
       const auto source_cat =
           Catalog_Namespace::SysCatalog::instance().getCatalog(source_dict_key.db_id);
       CHECK(source_cat);
       source_dict_desc_ =
           source_cat->getMetadataForDict(std::abs(source_dict_key.dict_id), true);
       CHECK(source_dict_desc_);
     } else {
       if (literals_dict) {
         auto const& transient_vecmap = literals_dict->getTransientVector();
         for (unsigned index = 0; index < transient_vecmap.size(); ++index) {
           auto const old_id = StringDictionaryProxy::transientIndexToId(index);
           std::string const& str = *transient_vecmap[index];
           auto const new_id = target_dict_desc_->stringDict->getOrAdd(str);
           literals_lookup_[old_id] = new_id;
         }
       }
 
       literals_lookup_[buffer_null_sentinal_] = buffer_null_sentinal_;
     }
 
     CHECK(target_dict_desc_);
 
     if (num_rows) {
       column_buffer_ = allocateColumnarBuffer(num_rows);
     }
   }

Here is the call graph for this function:

template<typename TARGET_TYPE >

DictionaryValueConverter< TARGET_TYPE >::~DictionaryValueConverter ( )

inlineoverride

Definition at line 231 of file TargetValueConvertersImpl.h.

231 {}

Member Function Documentation

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData ( Fragmenter_Namespace::InsertData & insertData )

inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 407 of file TargetValueConvertersImpl.h.

References NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, TargetValueConverter::column_descriptor_, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DataBlockPtr::numbersPtr.

                                                                                       {
     finalizeDataBlocksForInsertData();
     DataBlockPtr dataBlock;
     dataBlock.numbersPtr = reinterpret_cast<int8_t*>(this->column_data_.get());
     insertData.data.push_back(dataBlock);
     insertData.columnIds.push_back(this->column_descriptor_->columnId);
   }

Here is the call graph for this function:

template<typename TARGET_TYPE >

ElementsBufferColumnPtr DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer ( size_t num_rows )

inline

Definition at line 233 of file TargetValueConvertersImpl.h.

References CHECK.

Referenced by DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter().

                                                                   {
     CHECK(num_rows > 0);
     return std::make_unique<std::vector<int32_t>>(num_rows);
   }

Here is the caller graph for this function:

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat	(	size_t	row,
		typename ElementsBufferColumnPtr::pointer	columnBuffer,
		const ScalarTargetValue *	scalarValue
	)

inline

Definition at line 238 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, NumericValueConverter< int64_t, TARGET_TYPE >::do_null_check_, NumericValueConverter< int64_t, TARGET_TYPE >::null_check_value_, and NumericValueConverter< int64_t, TARGET_TYPE >::SOURCE_TYPE_ACCESSOR.

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

                                             {
     auto db_p = checked_get<int64_t>(row, scalarValue, this->SOURCE_TYPE_ACCESSOR);
     auto val = *db_p;
 
     if (this->do_null_check_ && this->null_check_value_ == val) {
       (*columnBuffer)[row] = this->buffer_null_sentinal_;
     } else {
       (*columnBuffer)[row] = (int32_t)val;
     }
   }

Here is the caller graph for this function:

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat	(	size_t	row,
		const ScalarTargetValue *	scalarValue
	)

inline

Definition at line 252 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, and DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat().

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

                                                                                  {
     convertElementToColumnarFormat(row, this->column_buffer_.get(), scalarValue);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat	(	size_t	row,
		const TargetValue *	value
	)

inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 256 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat(), and TargetValueConverter::SCALAR_TARGET_VALUE_ACCESSOR.

                                                                               {
     auto scalarValue =
         checked_get<ScalarTargetValue>(row, value, this->SCALAR_TARGET_VALUE_ACCESSOR);
 
     convertToColumnarFormat(row, scalarValue);
   }

Here is the call graph for this function:

template<typename TARGET_TYPE >

int32_t DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId ( int32_t & transient_string_id )

inline

Definition at line 263 of file TargetValueConvertersImpl.h.

References StringDictionaryProxy::getOrAdd(), StringDictionaryProxy::getString(), and DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::processBuffer().

                                                                                      {
     if (source_dict_proxy_) {
       auto str = source_dict_proxy_->getString(transient_string_id);
       return source_dict_proxy_->getOrAdd(str);
     } else {
       throw std::runtime_error("Unexpected negative source ID");
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData ( )

inlineoverridevirtual

Reimplemented from TargetValueConverter.

Definition at line 400 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, and DictionaryValueConverter< TARGET_TYPE >::processBuffer().

Referenced by DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData().

                                                   {
     if (column_buffer_) {
       this->column_data_ = processBuffer(std::move(column_buffer_));
       column_buffer_ = nullptr;
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename TARGET_TYPE >

void DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer	(	std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &	array_buffer,
		std::unique_ptr< std::vector< ArrayDatum >>::pointer	arrayData
	)

inline

Definition at line 353 of file TargetValueConvertersImpl.h.

References is_null(), StringDictionary::populate_string_array_ids(), DictionaryValueConverter< TARGET_TYPE >::processBuffer(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

                                                              {
     if (use_literals_) {
       for (size_t row = 0; row < array_buffer->size(); row++) {
         auto& element = (array_buffer->at(row));
         bool is_null = false;
         if (element.second) {
           typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr data =
               processBuffer(std::move(element.second));
           int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(data.release());
           (*arrayData)[row] =
               ArrayDatum(element.first * sizeof(TARGET_TYPE), arrayDataPtr, is_null);
         }
       }
     } else {
       std::vector<std::vector<int32_t>> srcArrayIds(array_buffer->size());
       std::vector<std::vector<int32_t>> destArrayIds(0);
 
       for (size_t row = 0; row < array_buffer->size(); row++) {
         auto& element = (array_buffer->at(row));
         if (element.second) {
           srcArrayIds[row] = *(element.second.get());
         }
       }
 
       StringDictionary::populate_string_array_ids(destArrayIds,
                                                   target_dict_desc_->stringDict.get(),
                                                   srcArrayIds,
                                                   source_dict_desc_->stringDict.get());
 
       for (size_t row = 0; row < array_buffer->size(); row++) {
         auto& element = (array_buffer->at(row));
         bool is_null = false;
         if (element.second) {
           *(element.second.get()) = destArrayIds[row];
           int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(&(element.second->at(0)));
           (*arrayData)[row] = ArrayDatum(element.first * sizeof(TARGET_TYPE),
                                          arrayDataPtr,
                                          is_null,
                                          DoNothingDeleter());
         }
       }
     }
   }

Here is the call graph for this function:

template<typename TARGET_TYPE >

NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr DictionaryValueConverter< TARGET_TYPE >::processBuffer ( ElementsBufferColumnPtr buffer )

inline

Definition at line 272 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, checked_malloc(), DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId(), DictDescriptor::dictRef, StringDictionaryProxy::getTransientVector(), DictionaryValueConverter< TARGET_TYPE >::literals_lookup_, NumericValueConverter< int64_t, TARGET_TYPE >::null_value_, StringDictionary::populate_string_ids(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, to_string(), and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer().

                                       {
     typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr data =
         typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr(
             reinterpret_cast<TARGET_TYPE*>(
                 checked_malloc(buffer->size() * sizeof(TARGET_TYPE))));
 
     std::vector<int32_t>* bufferPtr =
         reinterpret_cast<std::vector<int32_t>*>(buffer.get());
     TARGET_TYPE* columnDataPtr = reinterpret_cast<TARGET_TYPE*>(data.get());
     if (use_literals_) {
       for (size_t i = 0; i < bufferPtr->size(); i++) {
         auto id = literals_lookup_[(*bufferPtr)[i]];
         if (id == buffer_null_sentinal_) {
           columnDataPtr[i] = this->null_value_;
         } else {
           columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
         }
       }
     } else {
       if (source_dict_desc_->dictRef == target_dict_desc_->dictRef) {
         // special case, where source and target dict are the same
         // mostly in update
         for (size_t i = 0; i < bufferPtr->size(); i++) {
           auto src_id = (*bufferPtr)[i];
           if (src_id == buffer_null_sentinal_) {
             columnDataPtr[i] = this->null_value_;
           } else if (src_id < 0) {
             columnDataPtr[i] = convertTransientStringIdToPermanentId(src_id);
           } else {
             columnDataPtr[i] = static_cast<TARGET_TYPE>(src_id);
           }
         }
 
       } else {
         std::vector<int32_t> dest_ids;
         dest_ids.resize(bufferPtr->size());
 
         if (source_dict_proxy_) {
           StringDictionary::populate_string_ids(dest_ids,
                                                 target_dict_desc_->stringDict.get(),
                                                 *bufferPtr,
                                                 source_dict_desc_->stringDict.get(),
                                                 source_dict_proxy_->getTransientVector());
         } else {
           StringDictionary::populate_string_ids(dest_ids,
                                                 target_dict_desc_->stringDict.get(),
                                                 *bufferPtr,
                                                 source_dict_desc_->stringDict.get());
         }
 
         // fixup NULL sentinel
         for (size_t i = 0; i < dest_ids.size(); i++) {
           auto id = dest_ids[i];
           if (id == buffer_null_sentinal_) {
             columnDataPtr[i] = this->null_value_;
           } else {
             if (std::is_signed<TARGET_TYPE>::value) {
               if (id < 0) {
                 throw std::runtime_error(
                     "Maximum number of unique strings (" +
                     std::to_string(std::numeric_limits<TARGET_TYPE>::max()) +
                     ") reached in target dictionary");
               }
             } else {
               if (id >= std::numeric_limits<TARGET_TYPE>::max()) {
                 throw std::runtime_error(
                     "Maximum number of unique strings (" +
                     std::to_string(std::numeric_limits<TARGET_TYPE>::max()) +
                     ") reached in target column's dict encoding");
               }
             }
             columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
           }
         }
       }
     }
 
     return data;
   }