Namespaces
	anonymous_namespace{InsertOrderFragmenter.cpp}

	anonymous_namespace{UpdelStorage.cpp}

Classes
class	RowDataProvider

struct	UpdateValuesStats

struct	ChunkUpdateStats

class	AbstractFragmenter

struct	InsertChunks

struct	InsertData
	The data to be inserted using the fragment manager. More...

class	FragmentInfo
	Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(rows) currently stored by that fragment. More...

class	TableInfo

struct	ShardDataOwner

struct	BlockWithColumnId

class	InsertDataLoader

class	LocalInsertConnector

class	InsertOrderFragmenter
	The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...

class	SortedOrderFragmenter

struct	ChunkToInsertDataConverter

struct	ScalarChunkConverter

struct	FixedLenArrayChunkConverter

struct	ArrayChunkConverter

struct	StringChunkConverter

struct	DateChunkConverter

Enumerations
enum	FragmenterType { INSERT_ORDER = 0 }

Functions
template<typename SRC >
std::vector< std::vector < size_t > >	compute_row_indices_of_shards (size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)

template<typename T >
size_t	indexOf (std::vector< T > &vec, T val)

bool	isStringVectorData (const ColumnDescriptor *cd)

bool	isDatumVectorData (const ColumnDescriptor *cd)

size_t	size_of_raw_column (const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)

std::vector< std::vector < size_t > >	compute_row_indices_of_shards (const Catalog_Namespace::Catalog &cat, size_t leaf_count, const InsertChunks &insert_chunks)

std::vector< std::vector < size_t > >	computeRowIndicesOfShards (const Catalog_Namespace::Catalog &cat, size_t leafCount, InsertData &insert_data)

template<typename T >
void	copyColumnDataOfShard (const std::vector< size_t > &rowIndices, T src, T dst)

BlockWithColumnId	copyColumnDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, const std::vector< size_t > &rowIndices, const ColumnDescriptor *pCol, size_t columnIndex, DataBlockPtr dataBlock, bool is_default)

std::pair< std::list < std::unique_ptr < foreign_storage::ForeignStorageBuffer > >, InsertChunks >	copy_data_of_shard (const Catalog_Namespace::Catalog &cat, const InsertChunks &insert_chunks, int shardTableIndex, const std::vector< size_t > &rowIndices)

InsertData	copyDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)

template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, T *buffer)

template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, std::vector< T > &buffer)

void	shuffleByIndexes (const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)

template<typename T >
void	sortIndexesImpl (std::vector< size_t > &indexes, const T *buffer)

void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< std::string > &buffer)

void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< ArrayDatum > &buffer)

void	sortIndexes (const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)

void	wait_cleanup_threads (std::vector< std::future< void >> &threads)

bool	is_integral (const SQLTypeInfo &t)

static int	get_chunks (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)

template<typename T >
static void	set_chunk_stats (const SQLTypeInfo &col_type, int8_t *data_addr, bool &has_null, T &min, T &max)

static void	set_chunk_metadata (const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)

size_t	get_null_padding (bool is_varlen_array, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)

std::set< size_t >	get_var_len_null_array_indexes (const SQLTypeInfo sql_type_info, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)

StringOffsetT	get_buffer_offset (bool is_varlen_array, const StringOffsetT *index_array, size_t index)

Enumeration Type Documentation

enum Fragmenter_Namespace::FragmenterType

stores the type of a child class of AbstractTableFragmenter

Enumerator
INSERT_ORDER

Definition at line 45 of file Fragmenter.h.

                     {
   INSERT_ORDER = 0  // these values persist in catalog.  make explicit
 };

Function Documentation

template<typename SRC >

std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards	(	size_t	shard_count,
		size_t	leaf_count,
		size_t	row_count,
		SRC *	src,
		bool	duplicated_key_value
	)

Definition at line 35 of file InsertDataLoader.cpp.

References SHARD_FOR_KEY.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and Fragmenter_Namespace::InsertDataLoader::insertChunks().

                                {
   const auto n_shard_tables = shard_count * leaf_count;
   std::vector<std::vector<size_t>> row_indices_of_shards(n_shard_tables);
   if (!duplicated_key_value) {
     for (size_t row = 0; row < row_count; row++) {
       // expecting unsigned data
       // thus, no need for double remainder
       auto shard_id = (std::is_unsigned<SRC>::value)
                           ? src[row] % n_shard_tables
                           : SHARD_FOR_KEY(src[row], n_shard_tables);
       row_indices_of_shards[shard_id].push_back(row);
     }
   } else {
     auto shard_id = (std::is_unsigned<SRC>::value)
                         ? src[0] % n_shard_tables
                         : SHARD_FOR_KEY(src[0], n_shard_tables);
     row_indices_of_shards[shard_id].reserve(row_count);
     for (size_t row = 0; row < row_count; row++) {
       row_indices_of_shards[shard_id].push_back(row);
     }
   }
 
   return row_indices_of_shards;
 }

Here is the caller graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards	(	const Catalog_Namespace::Catalog &	cat,
		size_t	leaf_count,
		const InsertChunks &	insert_chunks
	)

Definition at line 124 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertChunks::chunks, compute_row_indices_of_shards(), Chunk_NS::Chunk::getBuffer(), Data_Namespace::AbstractBuffer::getEncoder(), Catalog_Namespace::Catalog::getMetadataForTable(), Encoder::getNumElems(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), isDatumVectorData(), isStringVectorData(), size_of_raw_column(), Fragmenter_Namespace::InsertChunks::table_id, and UNREACHABLE.

                                        {
   const auto* td = cat.getMetadataForTable(insert_chunks.table_id);
   const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
   auto find_it = insert_chunks.chunks.find(shard_cd->columnId);
   CHECK(find_it != insert_chunks.chunks.end());
   Chunk_NS::Chunk& shard_chunk = *find_it->second;
   auto row_count = shard_chunk.getBuffer()->getEncoder()->getNumElems();
   auto shard_count = td->nShards;
 
   CHECK(!isStringVectorData(shard_cd));
   CHECK(!isDatumVectorData(shard_cd));
 
   auto memory_ptr = shard_chunk.getBuffer()->getMemoryPtr();
   CHECK(memory_ptr);
   switch (size_of_raw_column(cat, shard_cd, false)) {
     case 1:
       return compute_row_indices_of_shards(shard_count,
                                            leaf_count,
                                            row_count,
                                            reinterpret_cast<uint8_t*>(memory_ptr),
                                            false);
     case 2:
       return compute_row_indices_of_shards(shard_count,
                                            leaf_count,
                                            row_count,
                                            reinterpret_cast<uint16_t*>(memory_ptr),
                                            false);
     case 4:
       return compute_row_indices_of_shards(shard_count,
                                            leaf_count,
                                            row_count,
                                            reinterpret_cast<uint32_t*>(memory_ptr),
                                            false);
     case 8:
       return compute_row_indices_of_shards(shard_count,
                                            leaf_count,
                                            row_count,
                                            reinterpret_cast<uint64_t*>(memory_ptr),
                                            false);
     default:
       UNREACHABLE() << "unexpected data element size of column";
   }
   return {};
 }

Here is the call graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards	(	const Catalog_Namespace::Catalog &	cat,
		size_t	leafCount,
		InsertData &	insert_data
	)

Definition at line 172 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertData::columnIds, compute_row_indices_of_shards(), Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), indexOf(), Fragmenter_Namespace::InsertData::is_default, isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::InsertData::numRows, size_of_raw_column(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

                              {
   const auto* td = cat.getMetadataForTable(insert_data.tableId);
   const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
   auto shardDataBlockIndex = indexOf(insert_data.columnIds, shard_cd->columnId);
   DataBlockPtr& shardDataBlock = insert_data.data[shardDataBlockIndex];
   auto rowCount = insert_data.numRows;
   auto shardCount = td->nShards;
 
   CHECK(!isStringVectorData(shard_cd));
   CHECK(!isDatumVectorData(shard_cd));
 
   CHECK(insert_data.is_default.size() == insert_data.columnIds.size());
   bool is_default = insert_data.is_default[shardDataBlockIndex];
   switch (size_of_raw_column(cat, shard_cd)) {
     case 1:
       return compute_row_indices_of_shards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint8_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 2:
       return compute_row_indices_of_shards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint16_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 4:
       return compute_row_indices_of_shards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint32_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 8:
       return compute_row_indices_of_shards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint64_t*>(shardDataBlock.numbersPtr),
           is_default);
   }
   throw std::runtime_error("Unexpected data block element size");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair<std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer> >, InsertChunks> Fragmenter_Namespace::copy_data_of_shard	(	const Catalog_Namespace::Catalog &	cat,
		const InsertChunks &	insert_chunks,
		int	shardTableIndex,
		const std::vector< size_t > &	rowIndices
	)

Definition at line 298 of file InsertDataLoader.cpp.

References CHECK_EQ, Fragmenter_Namespace::InsertChunks::chunks, Fragmenter_Namespace::InsertChunks::db_id, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), Fragmenter_Namespace::InsertChunks::table_id, and Fragmenter_Namespace::InsertChunks::valid_row_indices.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertChunks().

                                                         {
   const auto* table = cat.getMetadataForTable(insert_chunks.table_id);
   const auto* physical_table = cat.getPhysicalTablesDescriptors(table)[shardTableIndex];
 
   InsertChunks insert_chunks_for_shard{
       physical_table->tableId, insert_chunks.db_id, {}, {}};
 
   std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer>> buffers;
 
   for (const auto& [column_id, chunk] : insert_chunks.chunks) {
     auto column = chunk->getColumnDesc();
     insert_chunks_for_shard.chunks[column_id] = std::make_shared<Chunk_NS::Chunk>(column);
     auto& chunk_for_shard = *insert_chunks_for_shard.chunks[column_id];
     chunk_for_shard.setBuffer(
         buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
             .get());
     if (column->columnType.is_varlen_indeed()) {  // requires an index buffer
       chunk_for_shard.setIndexBuffer(
           buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
               .get());
     }
     chunk_for_shard.initEncoder();
     chunk_for_shard.appendEncodedDataAtIndices(*chunk, rowIndices);
     CHECK_EQ(chunk_for_shard.getBuffer()->getEncoder()->getNumElems(), rowIndices.size());
   }
 
   // mark which row indices are valid for import
   auto row_count = rowIndices.size();
   insert_chunks_for_shard.valid_row_indices.reserve(row_count);
   for (size_t irow = 0; irow < row_count; ++irow) {
     auto row_index = rowIndices[irow];
     if (std::binary_search(insert_chunks.valid_row_indices.begin(),
                            insert_chunks.valid_row_indices.end(),
                            row_index)) {
       insert_chunks_for_shard.valid_row_indices.emplace_back(irow);
     }
   }
 
   return {std::move(buffers), insert_chunks_for_shard};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::copyColumnDataOfShard	(	const std::vector< size_t > &	rowIndices,
		T *	src,
		T *	dst
	)

Definition at line 222 of file InsertDataLoader.cpp.

Referenced by copyColumnDataOfShard(), and copyDataOfShard().

                                                                                 {
   for (size_t row = 0; row < rowIndices.size(); row++) {
     auto srcRowIndex = rowIndices[row];
     dst[row] = src[srcRowIndex];
   }
 }

Here is the caller graph for this function:

BlockWithColumnId Fragmenter_Namespace::copyColumnDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		const std::vector< size_t > &	rowIndices,
		const ColumnDescriptor *	pCol,
		size_t	columnIndex,
		DataBlockPtr	dataBlock,
		bool	is_default
	)

Definition at line 235 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, DataBlockPtr::arraysPtr, ColumnDescriptor::columnId, copyColumnDataOfShard(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::ShardDataOwner::rawData, report::rows, size_of_raw_column(), Fragmenter_Namespace::ShardDataOwner::stringData, and DataBlockPtr::stringsPtr.

                                                          {
   DataBlockPtr ret;
   std::vector<size_t> single_row_idx({0ul});
   const std::vector<size_t>& rows = is_default ? single_row_idx : rowIndices;
   if (isStringVectorData(pCol)) {
     auto& data = dataOwner.stringData[columnIndex];
     data.resize(rows.size());
     copyColumnDataOfShard(rows, &(*(dataBlock.stringsPtr))[0], &data[0]);
     ret.stringsPtr = &data;
 
   } else if (isDatumVectorData(pCol)) {
     auto& data = dataOwner.arrayData[columnIndex];
     data.resize(rows.size());
     copyColumnDataOfShard(rows, &(*(dataBlock.arraysPtr))[0], &data[0]);
     ret.arraysPtr = &data;
 
   } else {
     auto rawArrayElementSize = size_of_raw_column(cat, pCol);
     auto& data = dataOwner.rawData[columnIndex];
     data.resize(rows.size() * rawArrayElementSize);
 
     switch (rawArrayElementSize) {
       case 1: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint8_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint8_t*>(&data[0]));
         break;
       }
       case 2: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint16_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint16_t*>(&data[0]));
         break;
       }
       case 4: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint32_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint32_t*>(&data[0]));
         break;
       }
       case 8: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint64_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint64_t*>(&data[0]));
         break;
       }
       default:
         throw std::runtime_error("Unexpected data block element size");
     }
 
     ret.numbersPtr = reinterpret_cast<int8_t*>(&data[0]);
   }
 
   return {pCol->columnId, ret, is_default};
 }

Here is the call graph for this function:

InsertData Fragmenter_Namespace::copyDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		InsertData &	insert_data,
		int	shardTableIndex,
		const std::vector< size_t > &	rowIndices
	)

Definition at line 342 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, threading_serial::async(), cat(), Fragmenter_Namespace::InsertData::columnIds, copyColumnDataOfShard(), Fragmenter_Namespace::InsertData::data, Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), indexOf(), Fragmenter_Namespace::InsertData::is_default, Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::ShardDataOwner::rawData, Fragmenter_Namespace::ShardDataOwner::stringData, and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

                                                                 {
   const auto* td = cat.getMetadataForTable(insert_data.tableId);
   const auto* ptd = cat.getPhysicalTablesDescriptors(td)[shardTableIndex];
 
   InsertData shardData;
   shardData.databaseId = insert_data.databaseId;
   shardData.tableId = ptd->tableId;
   shardData.numRows = rowIndices.size();
 
   std::vector<const ColumnDescriptor*> pCols;
   std::vector<int> lCols;
 
   {
     auto logicalColumns = cat.getAllColumnMetadataForTable(td->tableId, true, true, true);
     for (const auto& cd : logicalColumns) {
       lCols.push_back(cd->columnId);
     }
 
     auto physicalColumns =
         cat.getAllColumnMetadataForTable(ptd->tableId, true, true, true);
     for (const auto& cd : physicalColumns) {
       pCols.push_back(cd);
     }
   }
 
   for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
     dataOwner.arrayData.emplace_back();
     dataOwner.rawData.emplace_back();
     dataOwner.stringData.emplace_back();
   }
 
   auto copycat = [&cat, &dataOwner, &rowIndices, &lCols, &pCols, &insert_data](int col) {
     const auto lColId = insert_data.columnIds[col];
     const auto pCol = pCols[indexOf(lCols, lColId)];
     return copyColumnDataOfShard(cat,
                                  dataOwner,
                                  rowIndices,
                                  pCol,
                                  col,
                                  insert_data.data[col],
                                  insert_data.is_default[col]);
   };
 
   std::vector<std::future<BlockWithColumnId>> worker_threads;
   for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
     worker_threads.push_back(std::async(std::launch::async, copycat, col));
   }
 
   for (auto& child : worker_threads) {
     child.wait();
   }
 
   for (auto& child : worker_threads) {
     auto shardColumnData = child.get();
     shardData.columnIds.push_back(shardColumnData.columnId);
     shardData.data.push_back(shardColumnData.block);
     shardData.is_default.push_back(shardColumnData.is_default);
   }
 
   return shardData;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

StringOffsetT Fragmenter_Namespace::get_buffer_offset	(	bool	is_varlen_array,
		const StringOffsetT *	index_array,
		size_t	index
	)

Definition at line 1182 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                               {
   auto offset = index_array[index];
   if (offset < 0) {
     // Variable length arrays encode null arrays as negative offsets
     CHECK(is_varlen_array);
     offset = -offset;
   }
   return offset;
 }

Here is the caller graph for this function:

static int Fragmenter_Namespace::get_chunks	(	const Catalog_Namespace::Catalog *	catalog,
		const TableDescriptor *	td,
		const FragmentInfo &	fragment,
		const Data_Namespace::MemoryLevel	memory_level,
		std::vector< std::shared_ptr< Chunk_NS::Chunk >> &	chunks
	)

static

Definition at line 75 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

                                                                        {
   for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
     if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
       ++nc;
       if (!cd->isVirtualCol) {
         auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
         CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
         ChunkKey chunk_key{
             catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
         auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                                &catalog->getDataMgr(),
                                                chunk_key,
                                                memory_level,
                                                0,
                                                chunk_meta_it->second->numBytes,
                                                chunk_meta_it->second->numElements);
         chunks.push_back(chunk);
       }
     }
   }
   return chunks.size();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t Fragmenter_Namespace::get_null_padding	(	bool	is_varlen_array,
		const std::vector< uint64_t > &	frag_offsets,
		const StringOffsetT *	index_array,
		size_t	fragment_row_count
	)

Definition at line 1119 of file UpdelStorage.cpp.

References CHECK_GT, CHECK_LT, and ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                                    {
   if (is_varlen_array) {
     size_t first_non_deleted_row_index{0};
     for (auto deleted_offset : frag_offsets) {
       if (first_non_deleted_row_index < deleted_offset) {
         break;
       } else {
         first_non_deleted_row_index++;
       }
     }
     CHECK_LT(first_non_deleted_row_index, fragment_row_count);
     if (first_non_deleted_row_index == 0) {
       // If the first row in the fragment is not deleted, then the first offset in the
       // index buffer/array already contains expected padding.
       return index_array[0];
     } else {
       // If the first non-deleted element is a null array (indentified by a negative
       // offset), get a padding value for the chunk buffer.
       if (index_array[first_non_deleted_row_index + 1] < 0) {
         size_t first_non_zero_offset{0};
         for (size_t i = 0; i <= first_non_deleted_row_index; i++) {
           if (index_array[i] != 0) {
             first_non_zero_offset = index_array[i];
             break;
           }
         }
         CHECK_GT(first_non_zero_offset, static_cast<size_t>(0));
         return std::min(ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE,
                         first_non_zero_offset);
       } else {
         return 0;
       }
     }
   } else {
     return 0;
   }
 }

Here is the caller graph for this function:

std::set<size_t> Fragmenter_Namespace::get_var_len_null_array_indexes	(	const SQLTypeInfo	sql_type_info,
		const std::vector< uint64_t > &	frag_offsets,
		const StringOffsetT *	index_array,
		size_t	fragment_row_count
	)

Definition at line 1161 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), and SQLTypeInfo::is_varlen_array().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                                                            {
   std::set<size_t> null_array_indexes;
   if (sql_type_info.is_varlen_array() && !sql_type_info.get_notnull()) {
     size_t frag_offset_index{0};
     size_t vacuum_offset{0};
     for (size_t i = 0; i < fragment_row_count; i++) {
       if (frag_offset_index < frag_offsets.size() &&
           i == frag_offsets[frag_offset_index]) {
         frag_offset_index++;
         vacuum_offset++;
       } else if (index_array[i + 1] < 0) {
         null_array_indexes.emplace(i - vacuum_offset);
       }
     }
   }
   return null_array_indexes;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

size_t Fragmenter_Namespace::indexOf	(	std::vector< T > &	vec,
		T	val
	)

Definition at line 66 of file InsertDataLoader.cpp.

References CHECK.

Referenced by org.apache.calcite.sql2rel.SqlToRelConverter::collectInsertTargets(), computeRowIndicesOfShards(), copyDataOfShard(), org.apache.calcite.sql.validate.SqlValidatorImpl::expandExprFromJoin(), and com.mapd.utility.db_vendors.PostGis_types::get_wkt().

                                          {
   typename std::vector<T>::iterator it = std::find(vec.begin(), vec.end(), val);
   CHECK(it != vec.end());
   return std::distance(vec.begin(), it);
 }

Here is the caller graph for this function:

bool Fragmenter_Namespace::is_integral ( const SQLTypeInfo & t )

inline

Definition at line 49 of file UpdelStorage.cpp.

References SQLTypeInfo::is_boolean(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), and SQLTypeInfo::is_timeinterval().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata().

                                               {
   return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Fragmenter_Namespace::isDatumVectorData ( const ColumnDescriptor * cd )

Definition at line 78 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_array().

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                    {
   return cd->columnType.is_array();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Fragmenter_Namespace::isStringVectorData ( const ColumnDescriptor * cd )

Definition at line 72 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), and kENCODING_NONE.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                     {
   return (cd->columnType.is_geometry()) ||
          (cd->columnType.is_string() &&
           cd->columnType.get_compression() == kENCODING_NONE);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

static void Fragmenter_Namespace::set_chunk_metadata	(	const Catalog_Namespace::Catalog *	catalog,
		FragmentInfo &	fragment,
		const std::shared_ptr< Chunk_NS::Chunk > &	chunk,
		const size_t	nrows_to_keep,
		UpdelRoll &	updel_roll
	)

static

Definition at line 1065 of file UpdelStorage.cpp.

References UpdelRoll::addDirtyChunk(), Fragmenter_Namespace::FragmentInfo::fragmentId, UpdelRoll::getChunkMetadata(), and Catalog_Namespace::Catalog::getMetadataForTable().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                                       {
   auto cd = chunk->getColumnDesc();
   auto td = catalog->getMetadataForTable(cd->tableId);
   auto data_buffer = chunk->getBuffer();
   auto chunkMetadata =
       updel_roll.getChunkMetadata({td, &fragment}, cd->columnId, fragment);
   chunkMetadata->numElements = nrows_to_keep;
   chunkMetadata->numBytes = data_buffer->size();
   updel_roll.addDirtyChunk(chunk, fragment.fragmentId);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

static void Fragmenter_Namespace::set_chunk_stats	(	const SQLTypeInfo &	col_type,
		int8_t *	data_addr,
		bool &	has_null,
		T &	min,
		T &	max
	)

static

Definition at line 1050 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), is_null(), anonymous_namespace{TypedDataAccessors.h}::set_minmax(), and heavydb.dtypes::T.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                     {
   T v;
   const auto can_be_null = !col_type.get_notnull();
   const auto is_null = get_scalar<T>(data_addr, col_type, v);
   if (is_null) {
     has_null = has_null || (can_be_null && is_null);
   } else {
     set_minmax(min, max, v);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::shuffleByIndexes	(	const ColumnDescriptor *	cd,
		const std::vector< size_t > &	indexes,
		DataBlockPtr &	data
	)

Definition at line 45 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, shuffleByIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                           {
   const auto& ti = cd->columnType;
   switch (ti.get_type()) {
     case kBOOLEAN:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kTINYINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kSMALLINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
       break;
     case kINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
       break;
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kFLOAT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
       break;
     case kDOUBLE:
       shuffleByIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
       break;
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (ti.is_varlen()) {
         shuffleByIndexesImpl(indexes, *data.stringsPtr);
       } else {
         switch (ti.get_size()) {
           case 1:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
             break;
           case 2:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
             break;
           case 4:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
             break;
           default:
             CHECK(false);
         }
       }
       break;
     case kDATE:
     case kTIME:
     case kTIMESTAMP:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kARRAY:
       shuffleByIndexesImpl(indexes, *data.arraysPtr);
       break;
     case kPOINT:
     case kMULTIPOINT:
     case kLINESTRING:
     case kMULTILINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
       shuffleByIndexesImpl(indexes, *data.stringsPtr);
       break;
     default:
       CHECK(false);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		T *	buffer
	)

Definition at line 26 of file SortedOrderFragmenter.cpp.

References heavydb.dtypes::T.

Referenced by shuffleByIndexes().

                                                                        {
   std::vector<T> new_buffer;
   new_buffer.reserve(indexes.size());
   for (const auto i : indexes) {
     new_buffer.push_back(buffer[i]);
   }
   std::memcpy(buffer, new_buffer.data(), indexes.size() * sizeof(T));
 }

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		std::vector< T > &	buffer
	)

Definition at line 36 of file SortedOrderFragmenter.cpp.

                                                                                   {
   std::vector<T> new_buffer;
   new_buffer.reserve(indexes.size());
   for (const auto i : indexes) {
     new_buffer.push_back(buffer[i]);
   }
   buffer.swap(new_buffer);
 }

size_t Fragmenter_Namespace::size_of_raw_column	(	const Catalog_Namespace::Catalog &	cat,
		const ColumnDescriptor *	cd,
		const bool	get_logical_size = `true`
	)

Definition at line 82 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_logical_size(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                               {
   switch (cd->columnType.get_type()) {
     case kPOINT:
     case kMULTIPOINT:
     case kLINESTRING:
     case kMULTILINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
     case kARRAY:
       throw std::runtime_error("geo and array columns have variable length elements");
     case kBOOLEAN:
     case kTINYINT:
     case kSMALLINT:
     case kINT:
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
     case kFLOAT:
     case kDOUBLE:
     case kTIMESTAMP:
     case kTIME:
     case kINTERVAL_DAY_TIME:
     case kINTERVAL_YEAR_MONTH:
     case kDATE:
       return get_logical_size ? cd->columnType.get_logical_size()
                               : cd->columnType.get_size();
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (cd->columnType.get_compression() == kENCODING_NONE) {
         throw std::runtime_error(
             "non encoded string columns have variable length elements");
       }
       return cd->columnType.get_size();
     default:
       throw std::runtime_error("not supported column type: " + cd->columnName + " (" +
                                cd->columnType.get_type_name() + ")");
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexes	(	const ColumnDescriptor *	cd,
		std::vector< size_t > &	indexes,
		const DataBlockPtr &	data
	)

Definition at line 140 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, sortIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                            {
   const auto& ti = cd->columnType;
   switch (ti.get_type()) {
     case kBOOLEAN:
       sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kTINYINT:
       sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kSMALLINT:
       sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
       break;
     case kINT:
       sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
       break;
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
       sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kFLOAT:
       sortIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
       break;
     case kDOUBLE:
       sortIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
       break;
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (ti.is_varlen()) {
         sortIndexesImpl(indexes, *data.stringsPtr);
       } else {
         switch (ti.get_size()) {
           case 1:
             sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
             break;
           case 2:
             sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
             break;
           case 4:
             sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
             break;
           default:
             CHECK(false);
         }
       }
       break;
     case kDATE:
     case kTIME:
     case kTIMESTAMP:
       sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kARRAY:
       sortIndexesImpl(indexes, *data.arraysPtr);
       break;
     default:
       CHECK(false) << "invalid type '" << ti.get_type() << "' to sort";
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const T *	buffer
	)

Definition at line 116 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, and gpu_enabled::sort().

Referenced by sortIndexes().

                                                                   {
   CHECK(buffer);
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a] < buffer[b];
   });
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< std::string > &	buffer
	)

Definition at line 123 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

                                                          {
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a].size() < buffer[b].size() ||
            (buffer[a].size() == buffer[b].size() && buffer[a] < buffer[b]);
   });
 }

Here is the call graph for this function:

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< ArrayDatum > &	buffer
	)

Definition at line 131 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

                                                           {
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a].is_null || buffer[a].length < buffer[b].length ||
            (!buffer[b].is_null && buffer[a].length == buffer[b].length &&
             memcmp(buffer[a].pointer, buffer[b].pointer, buffer[a].length) < 0);
   });
 }

Here is the call graph for this function:

void Fragmenter_Namespace::wait_cleanup_threads ( std::vector< std::future< void >> & threads )

inline

Definition at line 42 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows(), Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

                                                                       {
   for (auto& t : threads) {
     t.get();
   }
   threads.clear();
 }

Here is the caller graph for this function:

Namespaces

Classes

Enumerations

Functions

Enumeration Type Documentation

Function Documentation