#include <HashingSchemeRecycler.h>

Inheritance diagram for HashingSchemeRecycler:

Collaboration diagram for HashingSchemeRecycler:

Public Member Functions
	HashingSchemeRecycler ()

	std::numeric_limits ()

	std::numeric_limits ()

void	putItemToCache (QueryPlanHash key, std::optional< HashType > item, CacheItemType item_type, DeviceIdentifier device_identifier, size_t item_size, size_t compute_time, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override

void	initCache () override

void	clearCache () override

void	markCachedItemAsDirty (size_t table_key, std::unordered_set< QueryPlanHash > &key_set, CacheItemType item_type, DeviceIdentifier device_identifier) override

std::string	toString () const override

Public Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
	DataRecycler (const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)

virtual	~DataRecycler ()=default

virtual std::optional< HashType >	getItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::optional< EMPTY_META_INFO > meta_info=std::nullopt)=0

void	markCachedItemAsDirtyImpl (QueryPlanHash key, CachedItemContainer &m) const

bool	isCachedItemDirty (QueryPlanHash key, CachedItemContainer &m) const

std::shared_ptr < CachedItemContainer >	getCachedItemContainer (CacheItemType item_type, DeviceIdentifier device_identifier) const

std::optional< CachedItem < std::optional< HashType > , EMPTY_META_INFO > >	getCachedItemWithoutConsideringMetaInfo (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, CachedItemContainer &m, std::lock_guard< std::mutex > &lock)

size_t	getCurrentNumCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const

size_t	getCurrentNumDirtyCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const

size_t	getCurrentNumCleanCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const

size_t	getCurrentCacheSizeForDevice (CacheItemType item_type, DeviceIdentifier device_identifier) const

std::shared_ptr< CacheItemMetric >	getCachedItemMetric (CacheItemType item_type, DeviceIdentifier device_identifier, QueryPlanHash key) const

void	setTotalCacheSize (CacheItemType item_type, size_t new_total_cache_size)

void	setMaxCacheItemSize (CacheItemType item_type, size_t new_max_cache_item_size)

Private Member Functions
bool	hasItemInCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) const override

void	removeItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override

void	cleanupCacheForInsertion (CacheItemType item_type, DeviceIdentifier device_identifier, size_t required_size, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override

Additional Inherited Members
Public Types inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
using	CachedItemContainer = std::vector< CachedItem< std::optional< HashType >, EMPTY_META_INFO >>

using	PerDeviceCacheItemContainer = std::unordered_map< DeviceIdentifier, std::shared_ptr< CachedItemContainer >>

using	PerTypeCacheItemContainer = std::unordered_map< CacheItemType, std::shared_ptr< PerDeviceCacheItemContainer >>

using	PerTypeCacheMetricTracker = std::unordered_map< CacheItemType, CacheMetricTracker >

Protected Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
void	removeCachedItemFromBeginning (CacheItemType item_type, DeviceIdentifier device_identifier, int offset)

void	sortCacheContainerByQueryMetric (CacheItemType item_type, DeviceIdentifier device_identifier)

std::mutex &	getCacheLock () const

CacheMetricTracker &	getMetricTracker (CacheItemType item_type)

CacheMetricTracker const &	getMetricTracker (CacheItemType item_type) const

std::unordered_set < CacheItemType > const &	getCacheItemType () const

PerTypeCacheItemContainer const &	getItemCache () const

Detailed Description

Definition at line 24 of file HashingSchemeRecycler.h.

Constructor & Destructor Documentation

HashingSchemeRecycler::HashingSchemeRecycler ( )

inline

Definition at line 30 of file HashingSchemeRecycler.h.

References HT_HASHING_SCHEME.

31 : DataRecycler({CacheItemType::HT_HASHING_SCHEME},

DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::DataRecycler

DataRecycler(const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)

Definition: DataRecycler.h:458

HT_HASHING_SCHEME

Definition: DataRecycler.h:42

Member Function Documentation

void HashingSchemeRecycler::cleanupCacheForInsertion	(	CacheItemType	item_type,
		DeviceIdentifier	device_identifier,
		size_t	required_size,
		std::lock_guard< std::mutex > &	lock,
		std::optional< EMPTY_META_INFO >	meta_info = `std::nullopt`
	)

inlineoverrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 79 of file HashingSchemeRecycler.h.

84 {}

void HashingSchemeRecycler::clearCache ( )

overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 90 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), DataRecyclerUtil::getDeviceIdentifierString(), HT_HASHING_SCHEME, LAYOUT_CACHE_DEVICE_IDENTIFIER, and VLOG.

                                        {
   std::lock_guard<std::mutex> lock(getCacheLock());
   auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
                                                        LAYOUT_CACHE_DEVICE_IDENTIFIER);
   if (!layout_cache_container->empty()) {
     VLOG(1) << "[" << CacheItemType::HT_HASHING_SCHEME << ", "
             << DataRecyclerUtil::getDeviceIdentifierString(LAYOUT_CACHE_DEVICE_IDENTIFIER)
             << "] clear cache (# items: " << layout_cache_container->size() << ")";
     layout_cache_container->clear();
   }
 }

Here is the call graph for this function:

bool HashingSchemeRecycler::hasItemInCache	(	QueryPlanHash	key,
		CacheItemType	item_type,
		DeviceIdentifier	device_identifier,
		std::lock_guard< std::mutex > &	lock,
		std::optional< EMPTY_META_INFO >	meta_info = `std::nullopt`
	)		const

overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 133 of file HashingSchemeRecycler.cpp.

References anonymous_namespace{QueryMemoryDescriptor.cpp}::any_of(), CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), and HT_HASHING_SCHEME.

                                                   {
   if (!g_enable_data_recycler || !g_use_hashtable_cache ||
       key == EMPTY_HASHED_PLAN_DAG_KEY) {
     return false;
   }
   CHECK_EQ(item_type, CacheItemType::HT_HASHING_SCHEME);
   auto layout_cache = getCachedItemContainer(item_type, device_identifier);
   return std::any_of(layout_cache->begin(),
                      layout_cache->end(),
                      [&key](const auto& cached_item) { return cached_item.key == key; });
 }

Here is the call graph for this function:

void HashingSchemeRecycler::initCache ( )

inlineoverridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 51 of file HashingSchemeRecycler.h.

51 {}

void HashingSchemeRecycler::markCachedItemAsDirty	(	size_t	table_key,
		std::unordered_set< QueryPlanHash > &	key_set,
		CacheItemType	item_type,
		DeviceIdentifier	device_identifier
	)

overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 102 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), HT_HASHING_SCHEME, and DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::markCachedItemAsDirtyImpl().

                                         {
   if (!g_enable_data_recycler || !g_use_hashtable_cache || key_set.empty()) {
     return;
   }
   CHECK_EQ(item_type, CacheItemType::HT_HASHING_SCHEME);
   std::lock_guard<std::mutex> lock(getCacheLock());
   auto layout_cache = getCachedItemContainer(item_type, device_identifier);
   for (auto key : key_set) {
     markCachedItemAsDirtyImpl(key, *layout_cache);
   }
 }

Here is the call graph for this function:

void HashingSchemeRecycler::putItemToCache	(	QueryPlanHash	key,
		std::optional< HashType >	item,
		CacheItemType	item_type,
		DeviceIdentifier	device_identifier,
		size_t	item_size,
		size_t	compute_time,
		std::optional< EMPTY_META_INFO >	meta_info = `std::nullopt`
	)

overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 45 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemWithoutConsideringMetaInfo(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), DataRecyclerUtil::getDeviceIdentifierString(), HT_HASHING_SCHEME, and VLOG.

                                                                                    {
   if (!g_enable_data_recycler || !g_use_hashtable_cache ||
       key == EMPTY_HASHED_PLAN_DAG_KEY) {
     return;
   }
   CHECK_EQ(item_type, CacheItemType::HT_HASHING_SCHEME);
   std::lock_guard<std::mutex> lock(getCacheLock());
   auto layout_cache = getCachedItemContainer(item_type, device_identifier);
   auto candidate_layout = getCachedItemWithoutConsideringMetaInfo(
       key, item_type, device_identifier, *layout_cache, lock);
   if (candidate_layout) {
     return;
   }
   layout_cache->emplace_back(key, item, nullptr, meta_info);
   VLOG(1) << "[" << item_type << ", "
           << DataRecyclerUtil::getDeviceIdentifierString(device_identifier)
           << "] Put hashtable layout to cache (key: " << key << ")";
   return;
 }

Here is the call graph for this function:

void HashingSchemeRecycler::removeItemFromCache	(	QueryPlanHash	key,
		CacheItemType	item_type,
		DeviceIdentifier	device_identifier,
		std::lock_guard< std::mutex > &	lock,
		std::optional< EMPTY_META_INFO >	meta_info = `std::nullopt`
	)

overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 71 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecyclerUtil::getDeviceIdentifierString(), and VLOG.

                                             {
   auto layout_cache = getCachedItemContainer(item_type, device_identifier);
   auto filter = [key](auto const& item) { return item.key == key; };
   auto itr = std::find_if(layout_cache->cbegin(), layout_cache->cend(), filter);
   if (itr == layout_cache->cend()) {
     return;
   } else {
     VLOG(1) << "[" << item_type << ", "
             << DataRecyclerUtil::getDeviceIdentifierString(device_identifier)
             << "] remove cached item from cache (key: " << key << ")";
     layout_cache->erase(itr);
   }
 }

Here is the call graph for this function:

HashingSchemeRecycler::std::numeric_limits ( )

std::string HashingSchemeRecycler::toString ( ) const

overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 118 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecyclerUtil::getDeviceIdentifierString(), HashJoin::getHashTypeString(), HT_HASHING_SCHEME, and LAYOUT_CACHE_DEVICE_IDENTIFIER.

                                                 {
   auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
                                                        LAYOUT_CACHE_DEVICE_IDENTIFIER);
   std::ostringstream oss;
   oss << "Hashing scheme cache:\n";
   oss << "Device: "
       << DataRecyclerUtil::getDeviceIdentifierString(LAYOUT_CACHE_DEVICE_IDENTIFIER)
       << "\n";
   for (auto& kv : *layout_cache_container) {
     oss << "\tkey: " << kv.key
         << ", layout: " << HashJoin::getHashTypeString(*kv.cached_item) << "\n";
   }
   return oss.str();
 }

Here is the call graph for this function:

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/DataRecycler/HashingSchemeRecycler.h
/home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/DataRecycler/HashingSchemeRecycler.cpp

Public Member Functions

Private Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation