StringDictionary.cpp File Reference

#include "Shared/DatumFetchers.h"
#include "StringDictionary/StringDictionaryProxy.h"
#include "StringOps/StringOps.h"
#include <tbb/parallel_for.h>
#include <tbb/task_arena.h>
#include <algorithm>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/iterator/transform_iterator.hpp>
#include <boost/sort/spreadsort/string_sort.hpp>
#include <functional>
#include <future>
#include <iostream>
#include <numeric>
#include <string_view>
#include <thread>
#include <type_traits>
#include <sys/fcntl.h>
#include "Logger/Logger.h"
#include "OSDependent/heavyai_fs.h"
#include "Shared/sqltypes.h"
#include "Shared/thread_count.h"
#include "StringDictionaryClient.h"
#include "Utils/Regexp.h"
#include "Utils/StringLike.h"
#include "LeafHostInfo.h"
#include "Shared/measure.h"

Include dependency graph for StringDictionary.cpp:

Go to the source code of this file.

Classes
struct	anonymous_namespace{StringDictionary.cpp}::ThreadInfo
class	anonymous_namespace{StringDictionary.cpp}::MapMaker

Namespaces
	anonymous_namespace{StringDictionary.cpp}

Functions
int	anonymous_namespace{StringDictionary.cpp}::checked_open (const char *path, const bool recover)
const uint64_t	anonymous_namespace{StringDictionary.cpp}::round_up_p2 (const uint64_t num)
string_dict_hash_t	anonymous_namespace{StringDictionary.cpp}::hash_string (const std::string_view &str)
	client_no_timeout_ (new StringDictionaryClient(host,{dict_key.db_id, dict_key.dict_id}, false))
template<class T >
void	anonymous_namespace{StringDictionary.cpp}::throw_encoding_error (std::string_view str, const shared::StringDictKey &dict_key)
void	anonymous_namespace{StringDictionary.cpp}::throw_string_too_long_error (std::string_view str, const shared::StringDictKey &dict_key)
bool	anonymous_namespace{StringDictionary.cpp}::is_regexp_like (const std::string &str, const std::string &pattern, const char escape)
void	order_translation_locks (const shared::StringDictKey &source_dict_key, const shared::StringDictKey &dest_dict_key, std::shared_lock< std::shared_mutex > &source_read_lock, std::shared_lock< std::shared_mutex > &dest_read_lock)
void	translate_string_ids (std::vector< int32_t > &dest_ids, const LeafHostInfo &dict_server_host, const shared::StringDictKey &dest_dict_key, const std::vector< int32_t > &source_ids, const shared::StringDictKey &source_dict_key, const int32_t dest_generation)

Variables
bool	g_cache_string_hash {true}
const int	anonymous_namespace{StringDictionary.cpp}::SYSTEM_PAGE_SIZE = heavyai::get_page_size()
bool	g_enable_stringdict_parallel {false}
	true

Function Documentation

client_no_timeout_

(

new

StringDictionaryClienthost,{dict_key.db_id, dict_key.dict_id}, false

)

Definition at line 361 of file StringDictionary.cpp.

                                           {dict_key.db_id, dict_key.dict_id}, false)) {}

void order_translation_locks	(	const shared::StringDictKey &	source_dict_key,
		const shared::StringDictKey &	dest_dict_key,
		std::shared_lock< std::shared_mutex > &	source_read_lock,
		std::shared_lock< std::shared_mutex > &	dest_read_lock
	)

Definition at line 1863 of file StringDictionary.cpp.

Referenced by StringDictionary::buildDictionaryTranslationMap(), StringDictionaryProxy::buildIntersectionTranslationMapToOtherProxy(), and StringDictionaryProxy::buildUnionTranslationMapToOtherProxy().

                                                                              {
  const bool dicts_are_same = (source_dict_key == dest_dict_key);
  const bool source_dict_is_locked_first = (source_dict_key < dest_dict_key);
  if (dicts_are_same) {
    // dictionaries are same, only take one write lock
    dest_read_lock.lock();
  } else if (source_dict_is_locked_first) {
    source_read_lock.lock();
    dest_read_lock.lock();
  } else {
    dest_read_lock.lock();
    source_read_lock.lock();
  }
}

Here is the caller graph for this function:

void translate_string_ids	(	std::vector< int32_t > &	dest_ids,
		const LeafHostInfo &	dict_server_host,
		const shared::StringDictKey &	dest_dict_key,
		const std::vector< int32_t > &	source_ids,
		const shared::StringDictKey &	source_dict_key,
		const int32_t	dest_generation
	)

Definition at line 2095 of file StringDictionary.cpp.

References shared::StringDictKey::db_id, shared::StringDictKey::dict_id, and StringDictionaryClient::translate_string_ids().

Referenced by anonymous_namespace{RelAlgTranslator.cpp}::fill_dictionary_encoded_in_vals(), and ResultSet::translateDictEncodedColumns().

                                                         {
  shared::StringDictKey temp_dict_key(-1, -1);
  StringDictionaryClient string_client(
      dict_server_host, {temp_dict_key.db_id, temp_dict_key.dict_id}, false);
  string_client.translate_string_ids(dest_ids,
                                     {dest_dict_key.db_id, dest_dict_key.dict_id},
                                     source_ids,
                                     {source_dict_key.db_id, source_dict_key.dict_id},
                                     dest_generation);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

bool g_cache_string_hash {true}

Definition at line 55 of file StringDictionary.cpp.

Referenced by CommandLineOptions::fillOptions(), Catalog_Namespace::Catalog::getMetadataForDict(), and RowSetMemoryOwner::getOrAddStringDictProxy().

bool g_enable_stringdict_parallel {false}

Definition at line 116 of file StringDictionary.cpp.

Referenced by CommandLineOptions::fillOptions(), and StringDictionary::getOrAddBulk().

true

Definition at line 360 of file StringDictionary.cpp.