UpdelRoll Struct Reference

#include <UpdelRoll.h>

Collaboration diagram for UpdelRoll:

Public Member Functions
	~UpdelRoll ()
void	cancelUpdate ()
bool	commitUpdate ()
void	stageUpdate ()
void	addDirtyChunk (std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
std::shared_ptr< ChunkMetadata >	getChunkMetadata (const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
ChunkMetadataMap	getChunkMetadataMap (const MetaDataKey &key) const
size_t	getNumTuple (const MetaDataKey &key) const
void	setNumTuple (const MetaDataKey &key, size_t num_tuple)

Public Attributes
const Catalog_Namespace::Catalog *	catalog = nullptr
int	logicalTableId
Data_Namespace::MemoryLevel	memoryLevel {Data_Namespace::MemoryLevel::CPU_LEVEL}
bool	is_varlen_update = false
const TableDescriptor *	table_descriptor {nullptr}

Private Member Functions
void	updateFragmenterAndCleanupChunks ()
void	initializeUnsetMetadata (const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)

Private Attributes
heavyai::shared_mutex	chunk_update_tracker_mutex
std::map< ChunkKey, std::shared_ptr < Chunk_NS::Chunk > >	dirty_chunks
std::map< MetaDataKey, size_t >	num_tuples
std::map< MetaDataKey, ChunkMetadataMap >	chunk_metadata_map_per_fragment

Detailed Description

Definition at line 44 of file UpdelRoll.h.

Constructor & Destructor Documentation

UpdelRoll::~UpdelRoll ( )

inline

Definition at line 45 of file UpdelRoll.h.

References cancelUpdate(), and dirty_chunks.

               {
    if (dirty_chunks.size()) {
      cancelUpdate();
    }
  }

Here is the call graph for this function:

Member Function Documentation

void UpdelRoll::addDirtyChunk	(	std::shared_ptr< Chunk_NS::Chunk >	chunk,
		int	fragment_id
	)

Definition at line 1486 of file UpdelStorage.cpp.

References CHECK, and Catalog_Namespace::Catalog::getDatabaseId().

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

                                                   {
  heavyai::unique_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  CHECK(catalog);
  ChunkKey chunk_key{catalog->getDatabaseId(),
                     chunk->getColumnDesc()->tableId,
                     chunk->getColumnDesc()->columnId,
                     fragment_id};
  dirty_chunks[chunk_key] = chunk;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void UpdelRoll::cancelUpdate

(

)

Definition at line 1460 of file UpdelStorage.cpp.

References CHECK, Data_Namespace::DataMgr::free(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, and Catalog_Namespace::Catalog::setTableEpochs().

Referenced by ~UpdelRoll().

                             {
  if (nullptr == catalog) {
    return;
  }

  // TODO: needed?
  ChunkKey chunk_key{catalog->getDatabaseId(), logicalTableId};
  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
  if (is_varlen_update) {
    int databaseId = catalog->getDatabaseId();
    auto table_epochs = catalog->getTableEpochs(databaseId, logicalTableId);

    dirty_chunks.clear();
    catalog->setTableEpochs(databaseId, table_epochs);
  } else {
    const auto td = catalog->getMetadataForTable(logicalTableId);
    CHECK(td);
    if (td->persistenceLevel != memoryLevel) {
      for (const auto& [chunk_key, chunk] : dirty_chunks) {
        catalog->getDataMgr().free(chunk->getBuffer());
        chunk->setBuffer(nullptr);
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool UpdelRoll::commitUpdate

(

)

Definition at line 1402 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::Catalog::checkpoint(), Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, Catalog_Namespace::Catalog::setTableEpochsLogExceptions(), and TableDescriptor::tableId.

Referenced by StorageIOFacility::TransactionParameters::finalizeTransaction().

                             {
  if (nullptr == catalog) {
    return false;
  }
  const auto td = catalog->getMetadataForTable(logicalTableId);
  CHECK(td);
  ChunkKey chunk_key{catalog->getDatabaseId(), td->tableId};
  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);

  // Checkpoint all shards. Otherwise, epochs can go out of sync.
  if (td->persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
    auto table_epochs = catalog->getTableEpochs(catalog->getDatabaseId(), logicalTableId);
    try {
      // `checkpointWithAutoRollback` is not called here because, if a failure occurs,
      // `dirtyChunks` has to be cleared before resetting epochs
      catalog->checkpoint(logicalTableId);
    } catch (...) {
      dirty_chunks.clear();
      catalog->setTableEpochsLogExceptions(catalog->getDatabaseId(), table_epochs);
      throw;
    }
  }
  updateFragmenterAndCleanupChunks();
  return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > UpdelRoll::getChunkMetadata	(	const MetaDataKey &	key,
		int32_t	column_id,
		Fragmenter_Namespace::FragmentInfo &	fragment_info
	)

Definition at line 1511 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

                                                     {
  initializeUnsetMetadata(key.first, fragment_info);
  heavyai::shared_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
  auto chunk_metadata_it = metadata_map_it->second.find(column_id);
  CHECK(chunk_metadata_it != metadata_map_it->second.end());
  return chunk_metadata_it->second;
}

Here is the caller graph for this function:

ChunkMetadataMap UpdelRoll::getChunkMetadataMap

(

const MetaDataKey &

key

)

const

Definition at line 1524 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

                                                                            {
  heavyai::shared_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
  return metadata_map_it->second;
}

Here is the caller graph for this function:

size_t UpdelRoll::getNumTuple

(

const MetaDataKey &

key

)

const

Definition at line 1531 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

                                                          {
  heavyai::shared_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  auto it = num_tuples.find(key);
  CHECK(it != num_tuples.end());
  return it->second;
}

Here is the caller graph for this function:

void UpdelRoll::initializeUnsetMetadata ( const TableDescriptor *  td, Fragmenter_Namespace::FragmentInfo &  fragment_info  )

private

Definition at line 1497 of file UpdelStorage.cpp.

References Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysicalCopy(), and Fragmenter_Namespace::FragmentInfo::shadowNumTuples.

                                                     {
  heavyai::unique_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  MetaDataKey key{td, &fragment_info};
  if (chunk_metadata_map_per_fragment.count(key) == 0) {
    chunk_metadata_map_per_fragment[key] =
        fragment_info.getChunkMetadataMapPhysicalCopy();
  }
  if (num_tuples.count(key) == 0) {
    num_tuples[key] = fragment_info.shadowNumTuples;
  }
}

Here is the call graph for this function:

void UpdelRoll::setNumTuple	(	const MetaDataKey &	key,
		size_t	num_tuple
	)

Definition at line 1538 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                                                    {
  heavyai::unique_lock<heavyai::shared_mutex> lock(chunk_update_tracker_mutex);
  num_tuples[key] = num_tuple;
}

Here is the caller graph for this function:

void UpdelRoll::stageUpdate

(

)

Definition at line 1428 of file UpdelStorage.cpp.

References CHECK, CHECK_EQ, Data_Namespace::DataMgr::checkpoint(), Data_Namespace::CPU_LEVEL, Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), and Catalog_Namespace::Catalog::getDataMgr().

Referenced by TableOptimizer::vacuumFragments().

                            {
  CHECK(catalog);
  auto db_id = catalog->getDatabaseId();
  CHECK(table_descriptor);
  auto table_id = table_descriptor->tableId;
  CHECK_EQ(memoryLevel, Data_Namespace::MemoryLevel::CPU_LEVEL);
  CHECK_EQ(table_descriptor->persistenceLevel, Data_Namespace::MemoryLevel::DISK_LEVEL);
  try {
    catalog->getDataMgr().checkpoint(db_id, table_id, memoryLevel);
  } catch (...) {
    dirty_chunks.clear();
    throw;
  }
  updateFragmenterAndCleanupChunks();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void UpdelRoll::updateFragmenterAndCleanupChunks ( )

private

Definition at line 1444 of file UpdelStorage.cpp.

References Data_Namespace::DataMgr::deleteChunksWithPrefix(), Catalog_Namespace::Catalog::getDataMgr(), and Data_Namespace::GPU_LEVEL.

                                                 {
  // for each dirty fragment
  for (auto& cm : chunk_metadata_map_per_fragment) {
    cm.first.first->fragmenter->updateMetadata(catalog, cm.first, *this);
  }

  // flush gpu dirty chunks if update was not on gpu
  if (memoryLevel != Data_Namespace::MemoryLevel::GPU_LEVEL) {
    for (const auto& [chunk_key, chunk] : dirty_chunks) {
      catalog->getDataMgr().deleteChunksWithPrefix(
          chunk_key, Data_Namespace::MemoryLevel::GPU_LEVEL);
    }
  }
  dirty_chunks.clear();
}

Here is the call graph for this function:

Member Data Documentation

const Catalog_Namespace::Catalog* UpdelRoll::catalog = nullptr

Definition at line 53 of file UpdelRoll.h.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and TableOptimizer::vacuumFragments().

std::map<MetaDataKey, ChunkMetadataMap> UpdelRoll::chunk_metadata_map_per_fragment

private

Definition at line 99 of file UpdelRoll.h.

heavyai::shared_mutex UpdelRoll::chunk_update_tracker_mutex

mutableprivate

Definition at line 90 of file UpdelRoll.h.

std::map<ChunkKey, std::shared_ptr<Chunk_NS::Chunk> > UpdelRoll::dirty_chunks

private

Definition at line 93 of file UpdelRoll.h.

Referenced by ~UpdelRoll().

bool UpdelRoll::is_varlen_update = false

Definition at line 57 of file UpdelRoll.h.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

int UpdelRoll::logicalTableId

Definition at line 54 of file UpdelRoll.h.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and TableOptimizer::vacuumFragments().

Data_Namespace::MemoryLevel UpdelRoll::memoryLevel {Data_Namespace::MemoryLevel::CPU_LEVEL}

Definition at line 55 of file UpdelRoll.h.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and TableOptimizer::vacuumFragments().

std::map<MetaDataKey, size_t> UpdelRoll::num_tuples

private

Definition at line 96 of file UpdelRoll.h.

const TableDescriptor* UpdelRoll::table_descriptor {nullptr}

Definition at line 58 of file UpdelRoll.h.

Referenced by TableOptimizer::vacuumFragments().

---

The documentation for this struct was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Shared/UpdelRoll.h
• /home/jenkins-slave/workspace/core-os-doxygen/Fragmenter/UpdelStorage.cpp