#include <SortedOrderFragmenter.h>

Inheritance diagram for Fragmenter_Namespace::SortedOrderFragmenter:

Collaboration diagram for Fragmenter_Namespace::SortedOrderFragmenter:

Public Member Functions
	SortedOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr dataMgr, Catalog_Namespace::Catalog catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL)

	~SortedOrderFragmenter () override

void	insertData (InsertData &insert_data_struct) override
	appends data onto the most recently occuring fragment, creating a new one if necessary More...

void	insertDataNoCheckpoint (InsertData &insert_data_struct) override
	Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally. More...

	SortedOrderFragmenter (SortedOrderFragmenter &&)=default

	SortedOrderFragmenter (const SortedOrderFragmenter &)=delete

SortedOrderFragmenter &	operator= (const SortedOrderFragmenter &)=delete

Public Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
	InsertOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr dataMgr, Catalog_Namespace::Catalog catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL, const bool uses_foreign_storage=false)

	~InsertOrderFragmenter () override

size_t	getNumFragments () override
	returns the number of fragments in a table More...

TableInfo	getFragmentsForQuery () override
	returns (inside QueryInfo) object all ids and row sizes of fragments More...

void	insertChunks (const InsertChunks &insert_chunk) override
	Insert chunks into minimal number of fragments. More...

void	insertChunksNoCheckpoint (const InsertChunks &insert_chunk) override
	Insert chunks into minimal number of fragments; no locks or checkpoints taken. More...

void	dropFragmentsToSize (const size_t maxRows) override
	Will truncate table to less than maxRows by dropping fragments. More...

void	updateColumnChunkMetadata (const ColumnDescriptor *cd, const int fragment_id, const std::shared_ptr< ChunkMetadata > metadata) override
	Updates the metadata for a column chunk. More...

void	updateChunkStats (const ColumnDescriptor *cd, std::unordered_map< int, ChunkStats > &stats_map, std::optional< Data_Namespace::MemoryLevel > memory_level) override
	Update chunk stats. More...

FragmentInfo *	getFragmentInfo (const int fragment_id) const override
	Retrieve the fragment info object for an individual fragment for editing. More...

int	getFragmenterId () override
	get fragmenter's id More...

std::vector< int >	getChunkKeyPrefix () const

std::string	getFragmenterType () override
	get fragmenter's type (as string More...

size_t	getNumRows () override

void	setNumRows (const size_t numTuples) override

std::optional< ChunkUpdateStats >	updateColumn (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override

void	updateColumns (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const int fragmentId, const std::vector< TargetMetaInfo > sourceMetaInfo, const std::vector< const ColumnDescriptor * > columnDescriptors, const RowDataProvider &sourceDataProvider, const size_t indexOffFragmentOffsetColumn, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll, Executor *executor) override

void	updateColumn (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const ScalarTargetValue &rhs_value, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override

void	updateColumnMetadata (const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override

void	updateMetadata (const Catalog_Namespace::Catalog *catalog, const MetaDataKey &key, UpdelRoll &updel_roll) override

void	compactRows (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override

const std::vector< uint64_t >	getVacuumOffsets (const std::shared_ptr< Chunk_NS::Chunk > &chunk) override

auto	getChunksForAllColumns (const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level)

void	dropColumns (const std::vector< int > &columnIds) override

bool	hasDeletedRows (const int delete_column_id) override
	Iterates through chunk metadata to return whether any rows have been deleted. More...

void	resetSizesFromFragments () override

void	alterNonGeoColumnType (const std::list< const ColumnDescriptor * > &columns)

void	alterColumnGeoType (const std::list< std::pair< const ColumnDescriptor , std::list< const ColumnDescriptor >>> &src_dst_column_pairs)

Public Member Functions inherited from Fragmenter_Namespace::AbstractFragmenter
virtual	~AbstractFragmenter ()

Protected Member Functions
virtual void	sortData (InsertData &insertDataStruct)

Protected Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
FragmentInfo *	createNewFragment (const Data_Namespace::MemoryLevel memory_level=Data_Namespace::DISK_LEVEL)
	creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table. More...

void	deleteFragments (const std::vector< int > &dropFragIds)

void	conditionallyInstantiateFileMgrWithParams ()

void	getChunkMetadata ()

void	lockInsertCheckpointData (const InsertData &insertDataStruct)

void	insertDataImpl (InsertData &insert_data)

void	insertChunksImpl (const InsertChunks &insert_chunk)

void	addColumns (const InsertData &insertDataStruct)

	InsertOrderFragmenter (const InsertOrderFragmenter &)

InsertOrderFragmenter &	operator= (const InsertOrderFragmenter &)

FragmentInfo &	getFragmentInfoFromId (const int fragment_id)

auto	vacuum_fixlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)

auto	vacuum_varlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)

Additional Inherited Members
Public Types inherited from Fragmenter_Namespace::InsertOrderFragmenter
using	ModifyTransactionTracker = UpdelRoll

Protected Attributes inherited from Fragmenter_Namespace::InsertOrderFragmenter
std::vector< int >	chunkKeyPrefix_

std::map< int, Chunk_NS::Chunk >	columnMap_

std::vector< std::unique_ptr < Chunk_NS::Chunk > >	tracked_in_memory_chunks_

std::deque< std::unique_ptr < FragmentInfo > >	fragmentInfoVec_

Data_Namespace::DataMgr *	dataMgr_

Catalog_Namespace::Catalog *	catalog_

const int	physicalTableId_

const int	shard_

size_t	maxFragmentRows_

size_t	pageSize_

size_t	numTuples_

int	maxFragmentId_

size_t	maxChunkSize_

size_t	maxRows_

std::string	fragmenterType_

heavyai::shared_mutex	fragmentInfoMutex_

heavyai::shared_mutex	insertMutex_

Data_Namespace::MemoryLevel	defaultInsertLevel_

const bool	uses_foreign_storage_

bool	hasMaterializedRowId_

int	rowIdColId_

std::unordered_map< int, size_t >	varLenColInfo_

std::shared_ptr< std::mutex >	mutex_access_inmem_states

std::mutex	temp_mutex_

Detailed Description

Definition at line 23 of file SortedOrderFragmenter.h.

Constructor & Destructor Documentation

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter	(	const std::vector< int >	chunkKeyPrefix,
		std::vector< Chunk_NS::Chunk > &	chunkVec,
		Data_Namespace::DataMgr *	dataMgr,
		Catalog_Namespace::Catalog *	catalog,
		const int	physicalTableId,
		const int	shard,
		const size_t	maxFragmentRows = `DEFAULT_FRAGMENT_ROWS`,
		const size_t	maxChunkSize = `DEFAULT_MAX_CHUNK_SIZE`,
		const size_t	pageSize = `DEFAULT_PAGE_SIZE`,
		const size_t	maxRows = `DEFAULT_MAX_ROWS`,
		const Data_Namespace::MemoryLevel	defaultInsertLevel = `Data_Namespace::DISK_LEVEL`
	)

inline

Definition at line 25 of file SortedOrderFragmenter.h.

       : InsertOrderFragmenter(chunkKeyPrefix,
                               chunkVec,
                               dataMgr,
                               catalog,
                               physicalTableId,
                               shard,
                               maxFragmentRows,
                               maxChunkSize,
                               pageSize,
                               maxRows,
                               defaultInsertLevel) {}

Fragmenter_Namespace::SortedOrderFragmenter::~SortedOrderFragmenter ( )

inlineoverride

Definition at line 49 of file SortedOrderFragmenter.h.

49 {}

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( SortedOrderFragmenter && )

default

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( const SortedOrderFragmenter & )

delete

Member Function Documentation

void Fragmenter_Namespace::SortedOrderFragmenter::insertData ( InsertData & insert_data_struct )

inlineoverridevirtual

appends data onto the most recently occuring fragment, creating a new one if necessary

Todo:: be able to fill up current fragment in multi-row insert before creating new fragment

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 50 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertData(), and sortData().

                                                            {
     sortData(insert_data_struct);
     InsertOrderFragmenter::insertData(insert_data_struct);
   }

Here is the call graph for this function:

void Fragmenter_Namespace::SortedOrderFragmenter::insertDataNoCheckpoint ( InsertData & insert_data_struct )

inlineoverridevirtual

Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally.

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 55 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertDataNoCheckpoint(), and sortData().

                                                                        {
     sortData(insert_data_struct);
     InsertOrderFragmenter::insertDataNoCheckpoint(insert_data_struct);
   }

Here is the call graph for this function:

SortedOrderFragmenter& Fragmenter_Namespace::SortedOrderFragmenter::operator= ( const SortedOrderFragmenter & )

delete

void Fragmenter_Namespace::SortedOrderFragmenter::sortData ( InsertData & insertDataStruct )

protectedvirtual

Definition at line 202 of file SortedOrderFragmenter.cpp.

Referenced by insertData(), and insertDataNoCheckpoint().

                                                                  {
   // coming here table must have defined a sort_column for mini sort
   const auto table_desc = catalog_->getMetadataForTable(physicalTableId_);
   CHECK(table_desc);
   CHECK_GT(table_desc->sortedColumnId, 0);
   const auto logical_cd =
       catalog_->getMetadataForColumn(table_desc->tableId, table_desc->sortedColumnId);
   CHECK(logical_cd);
   const auto physical_cd = catalog_->getMetadataForColumn(
       table_desc->tableId,
       table_desc->sortedColumnId + (logical_cd->columnType.is_geometry() ? 1 : 0));
   const auto it = std::find(insertDataStruct.columnIds.begin(),
                             insertDataStruct.columnIds.end(),
                             physical_cd->columnId);
   CHECK(it != insertDataStruct.columnIds.end());
   // sort row indexes of the sort column
   const auto dist = std::distance(insertDataStruct.columnIds.begin(), it);
   if (!insertDataStruct.is_default[dist]) {
     std::vector<size_t> indexes(insertDataStruct.numRows);
     std::iota(indexes.begin(), indexes.end(), 0);
     CHECK_LT(static_cast<size_t>(dist), insertDataStruct.data.size());
     sortIndexes(physical_cd, indexes, insertDataStruct.data[dist]);
     // shuffle rows of all columns
     for (size_t i = 0; i < insertDataStruct.columnIds.size(); ++i) {
       if (insertDataStruct.is_default[i]) {
         continue;
       }
       const auto cd = catalog_->getMetadataForColumn(table_desc->tableId,
                                                      insertDataStruct.columnIds[i]);
       shuffleByIndexes(cd, indexes, insertDataStruct.data[i]);
     }
   } else {
     // nothing to shuffle, the column has the same value across all rows
     return;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/Fragmenter/SortedOrderFragmenter.h
/home/jenkins-slave/workspace/core-os-doxygen/Fragmenter/SortedOrderFragmenter.cpp

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation