#include <CachingGlobalFileMgr.h>

Inheritance diagram for File_Namespace::CachingGlobalFileMgr:

Collaboration diagram for File_Namespace::CachingGlobalFileMgr:

Public Member Functions
	CachingGlobalFileMgr (int32_t device_id, std::shared_ptr< ForeignStorageInterface > fsi, const std::string &base_path, size_t num_reader_threads, foreign_storage::ForeignStorageCache *disk_cache, size_t defaultPageSize=DEFAULT_PAGE_SIZE)

AbstractBuffer *	createBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override

void	deleteBuffer (const ChunkKey &chunk_key, const bool purge) override

void	deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &keyPrefix) override

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

AbstractBuffer *	putBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override

void	checkpoint () override

void	checkpoint (const int db_id, const int tb_id) override

void	removeTableRelatedDS (const int db_id, const int table_id) override

void	removeCachedData (const int db_id, const int table_id)

Public Member Functions inherited from File_Namespace::GlobalFileMgr
	GlobalFileMgr (const int32_t device_id, std::shared_ptr< ForeignStorageInterface > fsi, std::string base_path=".", const size_t num_reader_threads=0, const size_t page_size=DEFAULT_PAGE_SIZE, const size_t metadata_page_size=DEFAULT_METADATA_PAGE_SIZE)
	Constructor. More...

	~GlobalFileMgr () override

AbstractBuffer *	createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
	Creates a chunk with the specified key and page size. More...

bool	isBufferOnDevice (const ChunkKey &key) override

void	deleteBuffer (const ChunkKey &key, const bool purge=true) override
	Deletes the chunk with the specified key. More...

void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override

AbstractBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override
	Returns the a pointer to the chunk with the specified key. More...

void	fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override

AbstractBuffer *	putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
	Puts the contents of d into the Chunk with the given key. More...

AbstractBuffer *	alloc (const size_t numBytes) override

void	free (AbstractBuffer *buffer) override

MgrType	getMgrType () override

std::string	getStringMgrType () override

std::string	printSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	init ()

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override

void	checkpoint () override
	Fsyncs data files, writes out epoch and fsyncs that. More...

void	checkpoint (const int32_t db_id, const int32_t tb_id) override

size_t	getNumReaderThreads ()
	Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...

size_t	getNumChunks () override

void	compactDataFiles (const int32_t db_id, const int32_t tb_id)

AbstractBufferMgr *	findFileMgr (const int32_t db_id, const int32_t tb_id)

void	setFileMgrParams (const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)

AbstractBufferMgr *	getFileMgr (const int32_t db_id, const int32_t tb_id)

AbstractBufferMgr *	getFileMgr (const ChunkKey &key)

std::string	getBasePath () const

size_t	getPageSize () const

size_t	getMetadataPageSize () const

void	writeFileMgrData (FileMgr *fileMgr=0)

bool	getDBConvert () const

void	setDBConvert (bool val)

void	removeTableRelatedDS (const int32_t db_id, const int32_t tb_id) override

void	setTableEpoch (const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)

size_t	getTableEpoch (const int32_t db_id, const int32_t tb_id)

void	resetTableEpochFloor (const int32_t db_id, const int32_t tb_id)

StorageStats	getStorageStats (const int32_t db_id, const int32_t tb_id)

std::shared_ptr< FileMgr >	getSharedFileMgr (const int db_id, const int table_id)

void	setFileMgr (const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)

void	closeFileMgr (const int32_t db_id, const int32_t tb_id)

Private Member Functions
bool	isChunkPrefixCacheable (const ChunkKey &chunk_prefix) const

Private Attributes
foreign_storage::ForeignStorageCache *	disk_cache_

std::set< ChunkKey >	cached_chunk_keys_

Additional Inherited Members
Static Public Attributes inherited from File_Namespace::GlobalFileMgr
static constexpr int32_t	db_version_ {2}

Protected Attributes inherited from File_Namespace::GlobalFileMgr
std::shared_ptr < ForeignStorageInterface >	fsi_

Detailed Description

Definition at line 26 of file CachingGlobalFileMgr.h.

Constructor & Destructor Documentation

File_Namespace::CachingGlobalFileMgr::CachingGlobalFileMgr	(	int32_t	device_id,
		std::shared_ptr< ForeignStorageInterface >	fsi,
		const std::string &	base_path,
		size_t	num_reader_threads,
		foreign_storage::ForeignStorageCache *	disk_cache,
		size_t	defaultPageSize = `DEFAULT_PAGE_SIZE`
	)

Definition at line 22 of file CachingGlobalFileMgr.cpp.

References CHECK, and disk_cache_.

     : GlobalFileMgr(device_id, fsi, base_path, num_reader_threads, default_page_size)
     , disk_cache_(disk_cache) {
   CHECK(disk_cache_);
 }

Member Function Documentation

void File_Namespace::CachingGlobalFileMgr::checkpoint ( )

override

Definition at line 129 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::checkpoint(), File_Namespace::GlobalFileMgr::checkpoint(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), get_table_prefix(), File_Namespace::GlobalFileMgr::getBuffer(), isChunkPrefixCacheable(), and foreign_storage::ForeignStorageCache::putBuffer().

                                       {
   std::set<File_Namespace::TablePair> tables_to_checkpoint;
   for (auto& key : cached_chunk_keys_) {
     if (isChunkPrefixCacheable(key) && GlobalFileMgr::getBuffer(key)->isDirty()) {
       tables_to_checkpoint.emplace(get_table_prefix(key));
       foreign_storage::ForeignStorageBuffer temp_buf;
       GlobalFileMgr::fetchBuffer(key, &temp_buf, 0);
       disk_cache_->putBuffer(key, &temp_buf);
     }
   }
   for (auto [db, tb] : tables_to_checkpoint) {
     disk_cache_->checkpoint(db, tb);
   }
   GlobalFileMgr::checkpoint();
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::checkpoint	(	const int	db_id,
		const int	tb_id
	)

override

Definition at line 145 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::checkpoint(), File_Namespace::GlobalFileMgr::checkpoint(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), File_Namespace::GlobalFileMgr::getBuffer(), isChunkPrefixCacheable(), Data_Namespace::AbstractBuffer::isDirty(), and foreign_storage::ForeignStorageCache::putBuffer().

                                                                       {
   if (isChunkPrefixCacheable({db_id, tb_id})) {
     bool need_checkpoint{false};
     ChunkKey chunk_prefix{db_id, tb_id};
     ChunkKey upper_prefix(chunk_prefix);
     upper_prefix.push_back(std::numeric_limits<int>::max());
     auto end_it =
         cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
     for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(chunk_prefix);
          chunk_key_it != end_it;
          ++chunk_key_it) {
       if (GlobalFileMgr::getBuffer(*chunk_key_it)->isDirty()) {
         need_checkpoint = true;
         foreign_storage::ForeignStorageBuffer temp_buf;
         GlobalFileMgr::fetchBuffer(*chunk_key_it, &temp_buf, 0);
         disk_cache_->putBuffer(*chunk_key_it, &temp_buf);
       }
     }
     if (need_checkpoint) {
       disk_cache_->checkpoint(db_id, tb_id);
     }
   }
   GlobalFileMgr::checkpoint(db_id, tb_id);
 }

Here is the call graph for this function:

AbstractBuffer * File_Namespace::CachingGlobalFileMgr::createBuffer	(	const ChunkKey &	chunk_key,
		const size_t	page_size,
		const size_t	initial_size
	)

override

Definition at line 34 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, File_Namespace::GlobalFileMgr::createBuffer(), and isChunkPrefixCacheable().

                                                                               {
   auto buf = GlobalFileMgr::createBuffer(chunk_key, page_size, initial_size);
   if (isChunkPrefixCacheable(chunk_key)) {
     cached_chunk_keys_.emplace(chunk_key);
   }
   return buf;
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::deleteBuffer	(	const ChunkKey &	chunk_key,
		const bool	purge
	)

override

Definition at line 44 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, File_Namespace::GlobalFileMgr::deleteBuffer(), foreign_storage::ForeignStorageCache::deleteBufferIfExists(), disk_cache_, and isChunkPrefixCacheable().

                                                                                    {
   if (isChunkPrefixCacheable(chunk_key)) {
     disk_cache_->deleteBufferIfExists(chunk_key);
     cached_chunk_keys_.erase(chunk_key);
   }
   GlobalFileMgr::deleteBuffer(chunk_key, purge);
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix	(	const ChunkKey &	chunk_key_prefix,
		const bool	purge
	)

override

Definition at line 52 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, CHECK, foreign_storage::ForeignStorageCache::clearForTablePrefix(), File_Namespace::GlobalFileMgr::deleteBuffersWithPrefix(), disk_cache_, get_table_key(), has_table_prefix(), and isChunkPrefixCacheable().

                                                                      {
   if (isChunkPrefixCacheable(chunk_key_prefix)) {
     CHECK(has_table_prefix(chunk_key_prefix));
     disk_cache_->clearForTablePrefix(get_table_key(chunk_key_prefix));
 
     ChunkKey upper_prefix(chunk_key_prefix);
     upper_prefix.push_back(std::numeric_limits<int>::max());
     auto end_it =
         cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
     for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(chunk_key_prefix);
          chunk_key_it != end_it;) {
       chunk_key_it = cached_chunk_keys_.erase(chunk_key_it);
     }
   }
   GlobalFileMgr::deleteBuffersWithPrefix(chunk_key_prefix, purge);
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::fetchBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	destination_buffer,
		const size_t	num_bytes
	)

override

Definition at line 90 of file CachingGlobalFileMgr.cpp.

References CHECK, Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), File_Namespace::GlobalFileMgr::findFileMgr(), get_table_prefix(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), has_table_prefix(), isChunkPrefixCacheable(), and foreign_storage::ForeignStorageCache::putBuffer().

                                                                {
   if (isChunkPrefixCacheable(chunk_key)) {
     // If we are recovering after a shutdown, it is possible for there to be cached data
     // without the file_mgr being initialized, so we need to check if the file_mgr exists.
     CHECK(has_table_prefix(chunk_key));
     auto [db, table_id] = get_table_prefix(chunk_key);
     auto file_mgr = GlobalFileMgr::findFileMgr(db, table_id);
     if (file_mgr && file_mgr->getBuffer(chunk_key)->isDirty()) {
       // It is possible for the fragmenter to write data to a FileBuffer and then attempt
       // to fetch that bufer without checkpointing. In that case the cache will not have
       // been updated and the cached buffer will be out of date, so we need to fetch the
       // storage buffer.
       GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
     } else {
       AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
       if (buffer) {
         buffer->copyTo(destination_buffer, num_bytes);
       } else {
         GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
         disk_cache_->putBuffer(chunk_key, destination_buffer, num_bytes);
       }
     }
   } else {
     GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
   }
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	keyPrefix
	)

override

Definition at line 70 of file CachingGlobalFileMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, disk_cache_, foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), File_Namespace::GlobalFileMgr::getChunkMetadataVecForKeyPrefix(), has_table_prefix(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), and isChunkPrefixCacheable().

                                 {
   CHECK(has_table_prefix(key_prefix));
   if (isChunkPrefixCacheable(key_prefix)) {
     // If the disk has any cached metadata for a prefix then it is guaranteed to have all
     // metadata for that table, so we can return a complete set.  If it has no metadata,
     // then it may be that the table has no data, or that it's just not cached, so we need
     // to go to storage to check.
     if (disk_cache_->hasCachedMetadataForKeyPrefix(key_prefix)) {
       disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
       return;
     }
   }
   GlobalFileMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
   if (isChunkPrefixCacheable(key_prefix)) {
     disk_cache_->cacheMetadataVec(chunk_metadata);
   }
 }

Here is the call graph for this function:

bool File_Namespace::CachingGlobalFileMgr::isChunkPrefixCacheable ( const ChunkKey & chunk_prefix ) const

private

Definition at line 190 of file CachingGlobalFileMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, File_Namespace::GlobalFileMgr::fsi_, and has_table_prefix().

Referenced by checkpoint(), createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), putBuffer(), and removeCachedData().

                                                                                     {
   CHECK(has_table_prefix(chunk_prefix));
   // If this is an Arrow FSI table then we can't cache it.
   if (fsi_->lookupBufferManager(chunk_prefix[CHUNK_KEY_DB_IDX],
                                 chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
     return false;
   }
   return true;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer * File_Namespace::CachingGlobalFileMgr::putBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	source_buffer,
		const size_t	num_bytes
	)

override

Definition at line 119 of file CachingGlobalFileMgr.cpp.

References disk_cache_, isChunkPrefixCacheable(), foreign_storage::ForeignStorageCache::putBuffer(), and File_Namespace::GlobalFileMgr::putBuffer().

                                                                         {
   auto buf = GlobalFileMgr::putBuffer(chunk_key, source_buffer, num_bytes);
   if (isChunkPrefixCacheable(chunk_key)) {
     disk_cache_->putBuffer(chunk_key, source_buffer, num_bytes);
   }
   return buf;
 }

Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::removeCachedData	(	const int	db_id,
		const int	table_id
	)

Definition at line 170 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, and isChunkPrefixCacheable().

Referenced by removeTableRelatedDS().

                                                                                {
   if (isChunkPrefixCacheable({db_id, table_id})) {
     const ChunkKey table_key{db_id, table_id};
     disk_cache_->clearForTablePrefix(table_key);
     ChunkKey upper_prefix(table_key);
     upper_prefix.push_back(std::numeric_limits<int>::max());
     auto end_it =
         cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
     for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(table_key);
          chunk_key_it != end_it;) {
       chunk_key_it = cached_chunk_keys_.erase(chunk_key_it);
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS	(	const int	db_id,
		const int	table_id
	)

override

Definition at line 185 of file CachingGlobalFileMgr.cpp.

References removeCachedData(), and File_Namespace::GlobalFileMgr::removeTableRelatedDS().

                                                                                    {
   removeCachedData(db_id, table_id);
   GlobalFileMgr::removeTableRelatedDS(db_id, table_id);
 }

Here is the call graph for this function:

Member Data Documentation

std::set<ChunkKey> File_Namespace::CachingGlobalFileMgr::cached_chunk_keys_

private

Definition at line 67 of file CachingGlobalFileMgr.h.

Referenced by checkpoint(), createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), and removeCachedData().

foreign_storage::ForeignStorageCache* File_Namespace::CachingGlobalFileMgr::disk_cache_

private

Definition at line 66 of file CachingGlobalFileMgr.h.

Referenced by CachingGlobalFileMgr(), checkpoint(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), putBuffer(), and removeCachedData().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/CachingGlobalFileMgr.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/CachingGlobalFileMgr.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation