#include <CachingForeignStorageMgr.h>

Inheritance diagram for foreign_storage::CachingForeignStorageMgr:

Collaboration diagram for foreign_storage::CachingForeignStorageMgr:

Public Member Functions
	CachingForeignStorageMgr (ForeignStorageCache *cache)

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

void	getChunkMetadataVecFromDataWrapper (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix)

void	removeTableRelatedDS (const int db_id, const int table_id) override

void	refreshTable (const ChunkKey &table_key, const bool evict_cached_entries) override

bool	createDataWrapperIfNotExists (const ChunkKey &chunk_key) override

bool	hasStoredDataWrapper (int32_t db, int32_t tb) const

Public Member Functions inherited from foreign_storage::ForeignStorageMgr
	ForeignStorageMgr ()

	~ForeignStorageMgr () override

AbstractBuffer *	createBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override

void	deleteBuffer (const ChunkKey &chunk_key, const bool purge) override

void	deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override

AbstractBuffer *	getBuffer (const ChunkKey &chunk_key, const size_t num_bytes) override

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

AbstractBuffer *	putBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

bool	isBufferOnDevice (const ChunkKey &chunk_key) override

std::string	printSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	checkpoint () override

void	checkpoint (const int db_id, const int tb_id) override

AbstractBuffer *	alloc (const size_t num_bytes) override

void	free (AbstractBuffer *buffer) override

MgrType	getMgrType () override

std::string	getStringMgrType () override

size_t	getNumChunks () override

void	removeTableRelatedDS (const int db_id, const int table_id) override

bool	hasDataWrapperForChunk (const ChunkKey &chunk_key) const

bool	isDatawrapperRestored (const ChunkKey &chunk_key)

void	setDataWrapper (const ChunkKey &table_key, std::shared_ptr< MockForeignDataWrapper > data_wrapper)

std::shared_ptr < ForeignDataWrapper >	getDataWrapper (const ChunkKey &chunk_key) const

void	setParallelismHints (const std::map< ChunkKey, std::set< ParallelismHint >> &hints_per_table)

Private Member Functions
void	refreshTableInCache (const ChunkKey &table_key)

int	getHighestCachedFragId (const ChunkKey &table_key)

void	refreshAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)

void	refreshNonAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)

void	refreshChunksInCacheByFragment (const std::vector< ChunkKey > &old_chunk_keys, int last_frag_id)

void	populateChunkBuffersSafely (ForeignDataWrapper &data_wrapper, ChunkToBufferMap &required_buffers, ChunkToBufferMap &optional_buffers)

void	eraseDataWrapper (const ChunkKey &key) override

void	clearTable (const ChunkKey &table_key)

size_t	maxFetchSize (int32_t db_id) const override

bool	hasMaxFetchSize () const override

std::set< ChunkKey >	getOptionalKeysWithinSizeLimit (const ChunkKey &chunk_key, const std::set< ChunkKey, decltype(set_comp)* > &same_fragment_keys, const std::set< ChunkKey, decltype(set_comp)* > &diff_fragment_keys) const override

bool	isChunkCached (const ChunkKey &chunk_key) const override

void	evictChunkFromCache (const ChunkKey &chunk_key) override

size_t	getBufferSize (const ChunkKey &key) const

size_t	getRequiredBuffersSize (const ChunkKey &chunk_key) const

Private Attributes
ForeignStorageCache *	disk_cache_

Additional Inherited Members
Public Types inherited from foreign_storage::ForeignStorageMgr
using	ParallelismHint = std::pair< int, int >

Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
void	updateFragmenterMetadata (const ChunkToBufferMap &) const

void	createDataWrapperUnlocked (int32_t db, int32_t tb)

bool	fetchBufferIfTempBufferMapEntryExists (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes)

ChunkToBufferMap	allocateTempBuffersForChunks (const std::set< ChunkKey > &chunk_keys)

void	clearTempChunkBufferMapEntriesForTable (const ChunkKey &table_key)

void	clearTempChunkBufferMapEntriesForTableUnlocked (const ChunkKey &table_key)

std::set< ChunkKey >	getOptionalChunkKeySetAndNormalizeCache (const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys, const ForeignDataWrapper::ParallelismLevel parallelism_level)

std::pair< std::set< ChunkKey, decltype(set_comp)* > , std::set< ChunkKey, decltype(set_comp)* > >	getPrefetchSets (const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys, const ForeignDataWrapper::ParallelismLevel parallelism_level) const

Static Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
static void	checkIfS3NeedsToBeEnabled (const ChunkKey &chunk_key)

Protected Attributes inherited from foreign_storage::ForeignStorageMgr
std::shared_mutex	data_wrapper_mutex_

std::map< ChunkKey, std::shared_ptr < ForeignDataWrapper > >	data_wrapper_map_

std::map< ChunkKey, std::shared_ptr < MockForeignDataWrapper > >	mocked_wrapper_map_

std::map< ChunkKey, std::unique_ptr < AbstractBuffer > >	temp_chunk_buffer_map_

std::shared_mutex	temp_chunk_buffer_map_mutex_

std::shared_mutex	parallelism_hints_mutex_

std::map< ChunkKey, std::set < ParallelismHint > >	parallelism_hints_per_table_

Detailed Description

Definition at line 27 of file CachingForeignStorageMgr.h.

Constructor & Destructor Documentation

foreign_storage::CachingForeignStorageMgr::CachingForeignStorageMgr ( ForeignStorageCache * cache )

Definition at line 38 of file CachingForeignStorageMgr.cpp.

References CHECK, and disk_cache_.

     : ForeignStorageMgr(), disk_cache_(cache) {
   CHECK(disk_cache_);
 }

Member Function Documentation

void foreign_storage::CachingForeignStorageMgr::clearTable ( const ChunkKey & table_key )

private

Definition at line 265 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, foreign_storage::ForeignStorageMgr::eraseDataWrapper(), and foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix().

Referenced by getChunkMetadataVecForKeyPrefix(), getChunkMetadataVecFromDataWrapper(), refreshNonAppendTableInCache(), and refreshTable().

                                                                    {
   disk_cache_->clearForTablePrefix(table_key);
   CHECK(!disk_cache_->hasCachedMetadataForKeyPrefix(table_key));
   ForeignStorageMgr::eraseDataWrapper(table_key);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists ( const ChunkKey & chunk_key )

overridevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 379 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageMgr::createDataWrapperUnlocked(), foreign_storage::ForeignStorageMgr::data_wrapper_map_, foreign_storage::ForeignStorageMgr::data_wrapper_mutex_, disk_cache_, shared::get_from_map(), get_table_key(), get_table_prefix(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), and foreign_storage::ForeignStorageCache::getSerializedWrapperPath().

Referenced by getChunkMetadataVecForKeyPrefix().

                                                                                      {
   std::lock_guard data_wrapper_lock(data_wrapper_mutex_);
   ChunkKey table_key = get_table_key(chunk_key);
   auto data_wrapper_it = data_wrapper_map_.find(table_key);
   if (data_wrapper_it != data_wrapper_map_.end()) {
     return false;
   }
   auto [db, tb] = get_table_prefix(chunk_key);
   createDataWrapperUnlocked(db, tb);
   auto wrapper_file = disk_cache_->getSerializedWrapperPath(db, tb);
   if (boost::filesystem::exists(wrapper_file)) {
     ChunkMetadataVector chunk_metadata;
     disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, table_key);
     try {
       auto data_wrapper = shared::get_from_map(data_wrapper_map_, table_key);
       data_wrapper->restoreDataWrapperInternals(
           disk_cache_->getSerializedWrapperPath(db, tb), chunk_metadata);
     } catch (std::exception& e) {
       throw RestoreDataWrapperException(e.what());
     }
   }
   return true;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::eraseDataWrapper ( const ChunkKey & key )

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 253 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageMgr::data_wrapper_map_, foreign_storage::ForeignStorageMgr::data_wrapper_mutex_, disk_cache_, get_table_prefix(), foreign_storage::ForeignStorageCache::getSerializedWrapperPath(), and is_table_key().

Referenced by getChunkMetadataVecForKeyPrefix(), and removeTableRelatedDS().

                                                                    {
   CHECK(is_table_key(key));
   std::lock_guard data_wrapper_lock(data_wrapper_mutex_);
   auto [db, tb] = get_table_prefix(key);
   // Need to erase serialized version on disk if it exists so we don't accidentally
   // recover it after deleting.  It is possible for a cached wrapper file to exist without
   // a wrapper (in multi-instance-mode for instance, so we make sure to remove the file
   // regardless).
   boost::filesystem::remove_all(disk_cache_->getSerializedWrapperPath(db, tb));
   data_wrapper_map_.erase(key);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::evictChunkFromCache ( const ChunkKey & chunk_key )

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 478 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::eraseChunk().

                                                                             {
   disk_cache_->eraseChunk(chunk_key);
 }

Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::fetchBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	destination_buffer,
		const size_t	num_bytes
	)

override

Definition at line 83 of file CachingForeignStorageMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, foreign_storage::ForeignStorageCache::eraseChunk(), foreign_storage::ForeignStorageMgr::fetchBuffer(), foreign_storage::get_column_key_set(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), foreign_storage::ForeignStorageMgr::getOptionalChunkKeySetAndNormalizeCache(), getRequiredBuffersSize(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), Data_Namespace::AbstractBuffer::isDirty(), maxFetchSize(), populateChunkBuffersSafely(), and foreign_storage::ChunkSizeValidator::validateChunkSize().

                                                                    {
   ChunkSizeValidator chunk_size_validator(chunk_key);
   if (is_in_memory_system_table_chunk_key(chunk_key)) {
     ForeignStorageMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
     chunk_size_validator.validateChunkSize(destination_buffer);
     return;
   }
   CHECK(destination_buffer);
   CHECK(!destination_buffer->isDirty());
 
   AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
   if (buffer) {
     chunk_size_validator.validateChunkSize(buffer);
     buffer->copyTo(destination_buffer, num_bytes);
     return;
   } else {
     auto required_size = getRequiredBuffersSize(chunk_key);
     if (required_size > maxFetchSize(chunk_key[CHUNK_KEY_DB_IDX])) {
       // If we don't have space in the cache then skip the caching.
       ForeignStorageMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
       return;
     }
 
     auto column_keys = get_column_key_set(chunk_key);
     // Avoid caching only a subset of the column key set.
     for (const auto& key : column_keys) {
       disk_cache_->eraseChunk(key);
     }
 
     // Use hints to prefetch other chunks in fragment into cache
     auto& data_wrapper = *getDataWrapper(chunk_key);
     auto optional_set = getOptionalChunkKeySetAndNormalizeCache(
         chunk_key, column_keys, data_wrapper.getCachedParallelismLevel());
 
     auto optional_buffers = disk_cache_->getChunkBuffersForCaching(optional_set);
     auto required_buffers = disk_cache_->getChunkBuffersForCaching(column_keys);
     CHECK(required_buffers.find(chunk_key) != required_buffers.end());
     populateChunkBuffersSafely(data_wrapper, required_buffers, optional_buffers);
 
     AbstractBuffer* buffer = required_buffers.at(chunk_key);
     CHECK(buffer);
     buffer->copyTo(destination_buffer, num_bytes);
   }
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::getBufferSize ( const ChunkKey & key ) const

private

Definition at line 453 of file CachingForeignStorageMgr.cpp.

References CHECK_EQ, disk_cache_, get_fragment_key(), foreign_storage::get_max_chunk_size(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), is_varlen_data_key(), is_varlen_key(), and show_chunk().

Referenced by getOptionalKeysWithinSizeLimit(), and getRequiredBuffersSize().

                                                                         {
   size_t num_bytes = 0;
   ChunkMetadataVector meta;
   disk_cache_->getCachedMetadataVecForKeyPrefix(meta, get_fragment_key(key));
   CHECK_EQ(meta.size(), 1U) << show_chunk(key);
   auto metadata = meta.begin()->second;
 
   if (is_varlen_key(key)) {
     if (is_varlen_data_key(key)) {
       num_bytes = get_max_chunk_size(key);
     } else {
       num_bytes = (metadata->sqlType.is_string())
                       ? sizeof(StringOffsetT) * (metadata->numElements + 1)
                       : sizeof(ArrayOffsetT) * (metadata->numElements + 1);
     }
   } else {
     num_bytes = metadata->numBytes;
   }
   return num_bytes;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	chunk_key_prefix
	)

override

Definition at line 138 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, clearTable(), createDataWrapperIfNotExists(), disk_cache_, eraseDataWrapper(), logger::ERROR, foreign_storage::fragment_maps_to_leaf(), get_table_key(), get_table_prefix(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), getChunkMetadataVecFromDataWrapper(), has_table_prefix(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata(), dist::is_distributed(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), foreign_storage::is_shardable_key(), LOG, and show_chunk().

                                 {
   if (is_in_memory_system_table_chunk_key(key_prefix)) {
     ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
     return;
   }
   CHECK(has_table_prefix(key_prefix));
   // If the disk has any cached metadata for a prefix then it is guaranteed to have all
   // metadata for that table, so we can return a complete set.  If it has no metadata,
   // then it may be that the table has no data, or that it's just not cached, so we need
   // to go to storage to check.
   if (disk_cache_->hasCachedMetadataForKeyPrefix(key_prefix)) {
     disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
 
     // Assert all metadata in cache is mapped to this leaf node in distributed.
     if (is_shardable_key(key_prefix)) {
       for (auto& [key, meta] : chunk_metadata) {
         CHECK(fragment_maps_to_leaf(key)) << show_chunk(key);
       }
     }
 
     try {
       // If the data in cache was restored from disk then it is possible that the wrapper
       // does not exist yet.  In this case the wrapper will be restored from disk if
       // possible.
       createDataWrapperIfNotExists(key_prefix);
       return;
     } catch (RestoreDataWrapperException& e) {
       auto [db_id, table_id] = get_table_prefix(key_prefix);
       LOG(ERROR) << "An error occurred while attempting to restore data wrapper using "
                     "disk cached metadata. Clearing cached data for table and proceeding "
                     "with a new data wrapper instance. Database ID: "
                  << db_id << ", table ID: " << table_id << ", error: " << e.what();
       chunk_metadata.clear();
       clearTable({db_id, table_id});
     }
   } else if (dist::is_distributed() &&
              disk_cache_->hasStoredDataWrapperMetadata(key_prefix[CHUNK_KEY_DB_IDX],
                                                        key_prefix[CHUNK_KEY_TABLE_IDX])) {
     // In distributed mode, it is possible to have all the chunk metadata filtered out for
     // this node, after previously getting the chunk metadata from the wrapper and caching
     // the wrapper metadata. In this case, return immediately and avoid doing a redundant
     // metadata scan.
     return;
   }
 
   // If we have no cached data then either the data was evicted, was never populated, or
   // the data for the table is an empty set (no chunks).  In case we are hitting the first
   // two, we should repopulate the data wrapper so just do it in all cases.
   auto table_key = get_table_key(key_prefix);
   eraseDataWrapper(table_key);
   createDataWrapperIfNotExists(table_key);
 
   getChunkMetadataVecFromDataWrapper(chunk_metadata, key_prefix);
   disk_cache_->cacheMetadataVec(chunk_metadata);
 }

Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	chunk_key_prefix
	)

Definition at line 196 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageCache::checkpoint(), clearTable(), disk_cache_, get_table_prefix(), foreign_storage::ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getDataWrapper(), has_table_prefix(), foreign_storage::is_table_enabled_on_node(), and foreign_storage::ForeignStorageCache::storeDataWrapper().

Referenced by getChunkMetadataVecForKeyPrefix(), refreshAppendTableInCache(), and refreshNonAppendTableInCache().

                                       {
   CHECK(has_table_prefix(chunk_key_prefix));
   auto [db_id, tb_id] = get_table_prefix(chunk_key_prefix);
   try {
     ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, chunk_key_prefix);
   } catch (...) {
     clearTable({db_id, tb_id});
     throw;
   }
   // If the table was disabled then we will have no wrapper to serialize.
   if (is_table_enabled_on_node(chunk_key_prefix)) {
     auto doc = getDataWrapper(chunk_key_prefix)->getSerializedDataWrapper();
     disk_cache_->storeDataWrapper(doc, db_id, tb_id);
 
     // If the wrapper populated buffers we want that action to be checkpointed.
     disk_cache_->checkpoint(db_id, tb_id);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

int foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId ( const ChunkKey & table_key )

private

Definition at line 271 of file CachingForeignStorageMgr.cpp.

References CHUNK_KEY_FRAGMENT_IDX, disk_cache_, foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), and foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix().

Referenced by refreshAppendTableInCache().

                                                                               {
   // Determine last fragment ID
   int last_frag_id = 0;
   if (disk_cache_->hasCachedMetadataForKeyPrefix(table_key)) {
     ChunkMetadataVector cached_metadata;
     disk_cache_->getCachedMetadataVecForKeyPrefix(cached_metadata, table_key);
     for (const auto& [key, metadata] : cached_metadata) {
       last_frag_id = std::max(last_frag_id, key[CHUNK_KEY_FRAGMENT_IDX]);
     }
   }
   return last_frag_id;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::set< ChunkKey > foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit	(	const ChunkKey &	chunk_key,
		const std::set< ChunkKey, decltype(set_comp)* > &	same_fragment_keys,
		const std::set< ChunkKey, decltype(set_comp)* > &	diff_fragment_keys
	)		const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 426 of file CachingForeignStorageMgr.cpp.

References CHUNK_KEY_DB_IDX, foreign_storage::get_column_key_set(), getBufferSize(), getRequiredBuffersSize(), and maxFetchSize().

                                                                            {
   std::set<ChunkKey> optional_keys;
   auto total_chunk_size = getRequiredBuffersSize(chunk_key);
   auto max_size = maxFetchSize(chunk_key[CHUNK_KEY_DB_IDX]);
   // Add keys to the list of optional keys starting with the same fragment.  If we run out
   // of space, then exit early with what we have added so far.
   for (const auto& keys : {same_fragment_keys, diff_fragment_keys}) {
     for (const auto& key : keys) {
       auto column_keys = get_column_key_set(key);
       for (const auto& column_key : column_keys) {
         total_chunk_size += getBufferSize(column_key);
       }
       // Early exist if we exceed the size limit.
       if (total_chunk_size > max_size) {
         return optional_keys;
       }
       for (const auto& column_key : column_keys) {
         optional_keys.emplace(column_key);
       }
     }
   }
   return optional_keys;
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize ( const ChunkKey & chunk_key ) const

private

Definition at line 417 of file CachingForeignStorageMgr.cpp.

References foreign_storage::get_column_key_set(), and getBufferSize().

Referenced by fetchBuffer(), and getOptionalKeysWithinSizeLimit().

                                                                                        {
   auto key_set = get_column_key_set(chunk_key);
   size_t total_size = 0U;
   for (const auto& key : key_set) {
     total_size += getBufferSize(key);
   }
   return total_size;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::CachingForeignStorageMgr::hasMaxFetchSize ( ) const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 413 of file CachingForeignStorageMgr.cpp.

                                                      {
   return true;
 }

bool foreign_storage::CachingForeignStorageMgr::hasStoredDataWrapper	(	int32_t	db,
		int32_t	tb
	)		const

Definition at line 229 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata().

                                                                                 {
   return disk_cache_->hasStoredDataWrapperMetadata(db, tb);
 }

Here is the call graph for this function:

bool foreign_storage::CachingForeignStorageMgr::isChunkCached ( const ChunkKey & chunk_key ) const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 474 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::getCachedChunkIfExists().

                                                                             {
   return disk_cache_->getCachedChunkIfExists(chunk_key) != nullptr;
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::maxFetchSize ( int32_t db_id ) const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 409 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::getMaxChunkDataSize().

Referenced by fetchBuffer(), and getOptionalKeysWithinSizeLimit().

                                                                  {
   return disk_cache_->getMaxChunkDataSize();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::populateChunkBuffersSafely	(	ForeignDataWrapper &	data_wrapper,
		ChunkToBufferMap &	required_buffers,
		ChunkToBufferMap &	optional_buffers
	)

private

Definition at line 43 of file CachingForeignStorageMgr.cpp.

References CHECK_GT, foreign_storage::ForeignStorageCache::checkpoint(), disk_cache_, get_table_prefix(), foreign_storage::ForeignDataWrapper::populateChunkBuffers(), and foreign_storage::ForeignStorageMgr::updateFragmenterMetadata().

Referenced by fetchBuffer(), and refreshChunksInCacheByFragment().

                                         {
   CHECK_GT(required_buffers.size(), 0U) << "Must populate at least one buffer";
   try {
     ChunkSizeValidator chunk_size_validator(required_buffers.begin()->first);
     data_wrapper.populateChunkBuffers(required_buffers, optional_buffers);
     chunk_size_validator.validateChunkSizes(required_buffers);
     chunk_size_validator.validateChunkSizes(optional_buffers);
     updateFragmenterMetadata(required_buffers);
     updateFragmenterMetadata(optional_buffers);
   } catch (const std::runtime_error& error) {
     // clear any partially loaded but failed chunks (there may be some
     // fully-loaded chunks as well but they will be cleared conservatively
     // anyways)
     for (const auto& [chunk_key, buffer] : required_buffers) {
       if (auto file_buffer = dynamic_cast<File_Namespace::FileBuffer*>(buffer)) {
         file_buffer->freeChunkPages();
       }
     }
     for (const auto& [chunk_key, buffer] : optional_buffers) {
       if (auto file_buffer = dynamic_cast<File_Namespace::FileBuffer*>(buffer)) {
         file_buffer->freeChunkPages();
       }
     }
 
     throw ForeignStorageException(error.what());
   }
   // All required buffers should be from the same table.
   auto [db, tb] = get_table_prefix(required_buffers.begin()->first);
   disk_cache_->checkpoint(db, tb);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache	(	const ChunkKey &	table_key,
		const std::vector< ChunkKey > &	old_chunk_keys
	)

private

Definition at line 284 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, disk_cache_, getChunkMetadataVecFromDataWrapper(), getHighestCachedFragId(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

                                                {
   CHECK(is_table_key(table_key));
   int last_frag_id = getHighestCachedFragId(table_key);
 
   ChunkMetadataVector storage_metadata;
   getChunkMetadataVecFromDataWrapper(storage_metadata, table_key);
   try {
     disk_cache_->cacheMetadataVec(storage_metadata);
     refreshChunksInCacheByFragment(old_chunk_keys, last_frag_id);
   } catch (std::runtime_error& e) {
     throw PostEvictionRefreshException(e);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment	(	const std::vector< ChunkKey > &	old_chunk_keys,
		int	last_frag_id
	)

private

Definition at line 316 of file CachingForeignStorageMgr.cpp.

References CHECK, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_, foreign_storage::get_column_key_set(), get_table_key(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), foreign_storage::ForeignStorageCache::isMetadataCached(), LOG, foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::MAX_REFRESH_TIME_IN_SECONDS, populateChunkBuffersSafely(), and logger::WARNING.

Referenced by refreshAppendTableInCache(), and refreshNonAppendTableInCache().

                        {
   int64_t total_time{0};
   auto fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
 
   if (old_chunk_keys.empty()) {
     return;
   }
 
   // Iterate through previously cached chunks and re-cache them. Caching is
   // done one fragment at a time, for all applicable chunks in the fragment.
   ChunkToBufferMap optional_buffers;
   std::set<ChunkKey> chunk_keys_to_be_cached;
   auto fragment_id = old_chunk_keys[0][CHUNK_KEY_FRAGMENT_IDX];
   const ChunkKey table_key{get_table_key(old_chunk_keys[0])};
   std::set<ChunkKey> chunk_keys_in_fragment;
   for (const auto& chunk_key : old_chunk_keys) {
     CHECK(chunk_key[CHUNK_KEY_TABLE_IDX] == table_key[CHUNK_KEY_TABLE_IDX]);
     if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] < start_frag_id) {
       continue;
     }
     if (disk_cache_->isMetadataCached(chunk_key)) {
       if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] != fragment_id) {
         if (chunk_keys_in_fragment.size() > 0) {
           auto required_buffers =
               disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
           populateChunkBuffersSafely(
               *getDataWrapper(table_key), required_buffers, optional_buffers);
           chunk_keys_in_fragment.clear();
         }
         // At this point, cache buffers for refreshable chunks in the last fragment
         // have been populated. Exit if the max refresh time has been exceeded.
         // Otherwise, move to the next fragment.
         auto current_time = std::chrono::high_resolution_clock::now();
         total_time += std::chrono::duration_cast<std::chrono::seconds>(
                           current_time - fragment_refresh_start_time)
                           .count();
         if (total_time >= MAX_REFRESH_TIME_IN_SECONDS) {
           LOG(WARNING) << "Refresh time exceeded for table key: { " << table_key[0]
                        << ", " << table_key[1] << " } after fragment id: " << fragment_id;
           break;
         } else {
           fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
         }
         fragment_id = chunk_key[CHUNK_KEY_FRAGMENT_IDX];
       }
       // Key may have been cached during scan
       if (disk_cache_->getCachedChunkIfExists(chunk_key) == nullptr) {
         auto column_keys = get_column_key_set(chunk_key);
         chunk_keys_in_fragment.insert(column_keys.begin(), column_keys.end());
         chunk_keys_to_be_cached.insert(column_keys.begin(), column_keys.end());
       }
     }
   }
   if (chunk_keys_in_fragment.size() > 0) {
     auto required_buffers =
         disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
     populateChunkBuffersSafely(
         *getDataWrapper(table_key), required_buffers, optional_buffers);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache	(	const ChunkKey &	table_key,
		const std::vector< ChunkKey > &	old_chunk_keys
	)

private

Definition at line 300 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, clearTable(), disk_cache_, getChunkMetadataVecFromDataWrapper(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

                                                {
   CHECK(is_table_key(table_key));
   ChunkMetadataVector storage_metadata;
   clearTable(table_key);
   getChunkMetadataVecFromDataWrapper(storage_metadata, table_key);
 
   try {
     disk_cache_->cacheMetadataVec(storage_metadata);
     refreshChunksInCacheByFragment(old_chunk_keys, 0);
   } catch (std::runtime_error& e) {
     throw PostEvictionRefreshException(e);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTable	(	const ChunkKey &	table_key,
		const bool	evict_cached_entries
	)

overridevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 217 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageMgr::checkIfS3NeedsToBeEnabled(), clearTable(), foreign_storage::ForeignStorageMgr::clearTempChunkBufferMapEntriesForTable(), is_table_key(), and refreshTableInCache().

                                                                              {
   CHECK(is_table_key(table_key));
   ForeignStorageMgr::checkIfS3NeedsToBeEnabled(table_key);
   clearTempChunkBufferMapEntriesForTable(table_key);
   if (evict_cached_entries) {
     clearTable(table_key);
   } else {
     refreshTableInCache(table_key);
   }
 }

Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTableInCache ( const ChunkKey & table_key )

private

Definition at line 233 of file CachingForeignStorageMgr.cpp.

References CHECK, disk_cache_, foreign_storage::fragment_maps_to_leaf(), foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix(), foreign_storage::is_append_table_chunk_key(), foreign_storage::is_shardable_key(), is_table_key(), refreshAppendTableInCache(), refreshNonAppendTableInCache(), and show_chunk().

Referenced by refreshTable().

                                                                             {
   CHECK(is_table_key(table_key));
 
   // Preserve the list of which chunks were cached per table to refresh after clear.
   std::vector<ChunkKey> old_chunk_keys =
       disk_cache_->getCachedChunksForKeyPrefix(table_key);
 
   // Assert all data in cache is mapped to this leaf node in distributed.
   if (is_shardable_key(table_key)) {
     for (auto& key : old_chunk_keys) {
       CHECK(fragment_maps_to_leaf(key)) << show_chunk(key);
     }
   }
 
   auto append_mode = is_append_table_chunk_key(table_key);
 
   append_mode ? refreshAppendTableInCache(table_key, old_chunk_keys)
               : refreshNonAppendTableInCache(table_key, old_chunk_keys);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::removeTableRelatedDS	(	const int	db_id,
		const int	table_id
	)

override

Definition at line 403 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, eraseDataWrapper(), and foreign_storage::ForeignStorageMgr::removeTableRelatedDS().

                                                                                        {
   disk_cache_->clearForTablePrefix({db_id, table_id});
   eraseDataWrapper({db_id, table_id});
   ForeignStorageMgr::removeTableRelatedDS(db_id, table_id);
 }

Here is the call graph for this function:

Member Data Documentation

ForeignStorageCache* foreign_storage::CachingForeignStorageMgr::disk_cache_

private

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CachingForeignStorageMgr.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CachingForeignStorageMgr.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation