StringDictionaryTranslationMgr Class Reference

#include <StringDictionaryTranslationMgr.h>

Collaboration diagram for StringDictionaryTranslationMgr:

Public Member Functions
	StringDictionaryTranslationMgr (const shared::StringDictKey &source_string_dict_key, const shared::StringDictKey &dest_string_dict_key, const bool translate_intersection_only, const SQLTypeInfo &output_ti, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, const Data_Namespace::MemoryLevel memory_level, const int device_count, Executor *executor, Data_Namespace::DataMgr *data_mgr, const bool delay_translation)
	StringDictionaryTranslationMgr (const shared::StringDictKey &source_string_dict_key, const SQLTypeInfo &output_ti, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, const Data_Namespace::MemoryLevel memory_level, const int device_count, Executor *executor, Data_Namespace::DataMgr *data_mgr, const bool delay_translation)
	~StringDictionaryTranslationMgr ()
void	buildTranslationMap ()
void	createKernelBuffers ()
llvm::Value *	codegen (llvm::Value *str_id_input, const SQLTypeInfo &input_ti, const bool add_nullcheck, const CompilationOptions &co) const
bool	isMapValid () const
const int8_t *	data () const
int32_t	minSourceStringId () const
size_t	mapSize () const

Private Attributes
const shared::StringDictKey	source_string_dict_key_
const shared::StringDictKey	dest_string_dict_key_
const bool	translate_intersection_only_
const SQLTypeInfo	output_ti_
const std::vector < StringOps_Namespace::StringOpInfo >	string_op_infos_
const bool	has_null_string_op_
const Data_Namespace::MemoryLevel	memory_level_
const int	device_count_
Executor *	executor_
Data_Namespace::DataMgr *	data_mgr_
const bool	dest_type_is_string_
const StringDictionaryProxy::IdMap *host_translation_map_	nullptr
const StringDictionaryProxy::TranslationMap < Datum > *	host_numeric_translation_map_
std::vector< const int8_t * >	kernel_translation_maps_
std::vector < Data_Namespace::AbstractBuffer * >	device_buffers_

Detailed Description

Definition at line 48 of file StringDictionaryTranslationMgr.h.

Constructor & Destructor Documentation

StringDictionaryTranslationMgr::StringDictionaryTranslationMgr	(	const shared::StringDictKey &	source_string_dict_key,
		const shared::StringDictKey &	dest_string_dict_key,
		const bool	translate_intersection_only,
		const SQLTypeInfo &	output_ti,
		const std::vector< StringOps_Namespace::StringOpInfo > &	string_op_infos,
		const Data_Namespace::MemoryLevel	memory_level,
		const int	device_count,
		Executor *	executor,
		Data_Namespace::DataMgr *	data_mgr,
		const bool	delay_translation
	)

Definition at line 51 of file StringDictionaryTranslationMgr.cpp.

References buildTranslationMap(), CHECK, CHECK_EQ, Data_Namespace::CPU_LEVEL, createKernelBuffers(), Data_Namespace::GPU_LEVEL, has_null_string_op_, and memory_level_.

    : source_string_dict_key_(source_string_dict_key)
    , dest_string_dict_key_(dest_string_dict_key)
    , translate_intersection_only_(translate_intersection_only)
    , output_ti_(output_ti)
    , string_op_infos_(string_op_infos)
    , has_null_string_op_(one_or_more_string_ops_is_null(string_op_infos))
    , memory_level_(memory_level)
    , device_count_(device_count)
    , executor_(executor)
    , data_mgr_(data_mgr)
    , dest_type_is_string_(true) {
#ifdef HAVE_CUDA
  CHECK(memory_level_ == Data_Namespace::CPU_LEVEL ||
        memory_level_ == Data_Namespace::GPU_LEVEL);
#else
  CHECK_EQ(Data_Namespace::CPU_LEVEL, memory_level_);
#endif  // HAVE_CUDA
  if (!delay_translation && !has_null_string_op_) {
    buildTranslationMap();
    createKernelBuffers();
  }
}

Here is the call graph for this function:

StringDictionaryTranslationMgr::StringDictionaryTranslationMgr	(	const shared::StringDictKey &	source_string_dict_key,
		const SQLTypeInfo &	output_ti,
		const std::vector< StringOps_Namespace::StringOpInfo > &	string_op_infos,
		const Data_Namespace::MemoryLevel	memory_level,
		const int	device_count,
		Executor *	executor,
		Data_Namespace::DataMgr *	data_mgr,
		const bool	delay_translation
	)

Definition at line 85 of file StringDictionaryTranslationMgr.cpp.

    : source_string_dict_key_(source_string_dict_key)
    , dest_string_dict_key_({-1, -1})

StringDictionaryTranslationMgr::~StringDictionaryTranslationMgr

(

)

Definition at line 119 of file StringDictionaryTranslationMgr.cpp.

References CHECK, data_mgr_, device_buffers_, and Data_Namespace::DataMgr::free().

                                                                {
  CHECK(data_mgr_);
  for (auto& device_buffer : device_buffers_) {
    data_mgr_->free(device_buffer);
  }
}

Here is the call graph for this function:

Member Function Documentation

void StringDictionaryTranslationMgr::buildTranslationMap

(

)

Definition at line 126 of file StringDictionaryTranslationMgr.cpp.

References dest_string_dict_key_, dest_type_is_string_, executor_, host_numeric_translation_map_, RowSetMemoryOwner::SOURCE_INTERSECTION, source_string_dict_key_, RowSetMemoryOwner::SOURCE_UNION, string_op_infos_, and translate_intersection_only_.

Referenced by StringDictionaryTranslationMgr().

                                                         {
  if (dest_type_is_string_) {
    host_translation_map_ = executor_->getStringProxyTranslationMap(
        source_string_dict_key_,
        dest_string_dict_key_,
        translate_intersection_only_
            ? RowSetMemoryOwner::StringTranslationType::SOURCE_INTERSECTION
            : RowSetMemoryOwner::StringTranslationType::SOURCE_UNION,
        string_op_infos_,
        executor_->getRowSetMemoryOwner(),
        true);
  } else {
    host_numeric_translation_map_ =
        executor_->getStringProxyNumericTranslationMap(source_string_dict_key_,
                                                       string_op_infos_,
                                                       executor_->getRowSetMemoryOwner(),
                                                       true);
  }
}

Here is the caller graph for this function:

llvm::Value * StringDictionaryTranslationMgr::codegen	(	llvm::Value *	str_id_input,
		const SQLTypeInfo &	input_ti,
		const bool	add_nullcheck,
		const CompilationOptions &	co
	)		const

Definition at line 171 of file StringDictionaryTranslationMgr.cpp.

References Parser::IntLiteral::analyzeValue(), AUTOMATIC_IR_METADATA, CHECK, CHECK_EQ, CHECK_GE, dest_type_is_string_, device_count_, CompilationOptions::device_type, executor_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_type(), GPU, Data_Namespace::GPU_LEVEL, has_null_string_op_, CompilationOptions::hoist_literals, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kENCODING_NONE, kernel_translation_maps_, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, memory_level_, minSourceStringId(), output_ti_, to_lower(), and toString().

Referenced by CodeGenerator::codegen(), CodeGenerator::codegenCastFromString(), and CodeGenerator::codegenPseudoStringOper().

                                                                                         {
  CHECK(kernel_translation_maps_.size() == static_cast<size_t>(device_count_) ||
        has_null_string_op_);
  if (!co.hoist_literals && kernel_translation_maps_.size() > 1UL) {
    // Currently the only way to have multiple kernel translation maps is
    // to be running on GPU, where we would need to have a different pointer
    // per GPU to the translation map, as the address space is not shared
    // between GPUs

    CHECK(memory_level_ == Data_Namespace::GPU_LEVEL);
    CHECK(co.device_type == ExecutorDeviceType::GPU);

    // Since we currently cannot support different code per device, the only
    // way to allow for a different kernel translation map/kernel per
    // device(i.e. GPU) is via hoisting the map handle literal so that
    // it can be paramertized as a kernel argument. Hence if literal
    // hoisting is disabled (generally b/c we have an update query),
    // the surest fire way of ensuring one and only one translation map
    // that can have a hard-coded handle in the generated code is by running
    // on CPU (which per the comment above currently always has a device
    // count of 1).

    // This is not currently a major limitation as we currently run
    // all update queries on CPU, but it would be if we want to run
    // on multiple GPUs.

    // Todo(todd): Examine ways around the above limitation, likely either
    // a dedicated kernel parameter for translation maps (like we have for
    // join hash tables), or perhaps better for a number of reasons, reworking
    // the translation map plumbing to use the join infra (which would also
    // mean we could use pieces like the baseline hash join for multiple
    // input string dictionaries, i.e. CONCAT on two string columns).

    throw QueryMustRunOnCpu();
  }
  CHECK(co.hoist_literals || kernel_translation_maps_.size() == 1UL);

  auto cgen_state_ptr = executor_->getCgenStatePtr();
  AUTOMATIC_IR_METADATA(cgen_state_ptr);

  if (has_null_string_op_) {
    // If any of the string ops can statically be determined to output all nulls
    // (currently determined by whether any of the constant literal inputs to the
    // string operation are null), then simply generate codegen a null
    // dictionary-encoded value
    const auto null_ti = SQLTypeInfo(kTEXT, true /* is_nullable */, kENCODING_DICT);
    return static_cast<llvm::Value*>(executor_->cgen_state_->inlineIntNull(null_ti));
  }

  std::vector<std::shared_ptr<const Analyzer::Constant>> constants_owned;
  std::vector<const Analyzer::Constant*> constants;
  for (const auto kernel_translation_map : kernel_translation_maps_) {
    const int64_t translation_map_handle =
        reinterpret_cast<int64_t>(kernel_translation_map);
    const auto translation_map_handle_literal =
        std::dynamic_pointer_cast<Analyzer::Constant>(
            Parser::IntLiteral::analyzeValue(translation_map_handle));
    CHECK(translation_map_handle_literal);
    CHECK_EQ(kENCODING_NONE,
             translation_map_handle_literal->get_type_info().get_compression());
    constants_owned.push_back(translation_map_handle_literal);
    constants.push_back(translation_map_handle_literal.get());
  }
  CHECK_GE(constants.size(), 1UL);
  CHECK(co.hoist_literals || constants.size() == 1UL);

  CodeGenerator code_generator(executor_);

  const auto translation_map_handle_lvs =
      co.hoist_literals
          ? code_generator.codegenHoistedConstants(constants, kENCODING_NONE, {})
          : code_generator.codegen(constants[0], false, co);
  CHECK_EQ(size_t(1), translation_map_handle_lvs.size());

  std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
  const bool is_nullable = !input_ti.get_notnull();
  const auto decoded_input_ti = SQLTypeInfo(kTEXT, is_nullable, kENCODING_DICT);
  if (add_nullcheck && is_nullable) {
    nullcheck_codegen = std::make_unique<CodeGenerator::NullCheckCodegen>(
        cgen_state_ptr,
        executor_,
        input_str_id_lv,
        decoded_input_ti,
        "dict_encoded_str_cast_nullcheck");
  }
  llvm::Value* ret;
  if (dest_type_is_string_) {
    ret = cgen_state_ptr->emitCall(
        "map_string_dict_id",
        {input_str_id_lv,
         cgen_state_ptr->castToTypeIn(translation_map_handle_lvs.front(), 64),
         cgen_state_ptr->llInt(minSourceStringId())});
  } else {
    std::string fn_call = "map_string_to_datum_";
    const auto sql_type = output_ti_.get_type();
    switch (sql_type) {
      case kBOOLEAN: {
        fn_call += "bool";
        break;
      }
      case kTINYINT:
      case kSMALLINT:
      case kINT:
      case kBIGINT:
      case kFLOAT:
      case kDOUBLE: {
        fn_call += to_lower(toString(sql_type));
        break;
      }
      case kNUMERIC:
      case kDECIMAL:
      case kTIME:
      case kTIMESTAMP:
      case kDATE: {
        fn_call += "bigint";
        break;
      }
      default: {
        throw std::runtime_error("Unimplemented type for string-to-numeric translation");
      }
    }
    ret = cgen_state_ptr->emitCall(
        fn_call,
        {input_str_id_lv,
         cgen_state_ptr->castToTypeIn(translation_map_handle_lvs.front(), 64),
         cgen_state_ptr->llInt(minSourceStringId())});
  }

  if (nullcheck_codegen) {
    ret = nullcheck_codegen->finalize(cgen_state_ptr->inlineNull(output_ti_), ret);
  }
  return ret;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void StringDictionaryTranslationMgr::createKernelBuffers

(

)

Definition at line 146 of file StringDictionaryTranslationMgr.cpp.

References CudaAllocator::allocGpuAbstractBuffer(), CHECK_EQ, copy_to_nvidia_gpu(), Data_Namespace::CPU_LEVEL, data(), data_mgr_, device_buffers_, device_count_, Data_Namespace::GPU_LEVEL, kernel_translation_maps_, mapSize(), and memory_level_.

Referenced by StringDictionaryTranslationMgr().

                                                         {
#ifdef HAVE_CUDA
  if (memory_level_ == Data_Namespace::GPU_LEVEL) {
    const size_t translation_map_size_bytes = mapSize();
    for (int device_id = 0; device_id < device_count_; ++device_id) {
      device_buffers_.emplace_back(CudaAllocator::allocGpuAbstractBuffer(
          data_mgr_, translation_map_size_bytes, device_id));
      auto device_buffer =
          reinterpret_cast<int8_t*>(device_buffers_.back()->getMemoryPtr());
      copy_to_nvidia_gpu(data_mgr_,
                         reinterpret_cast<CUdeviceptr>(device_buffer),
                         data(),
                         translation_map_size_bytes,
                         device_id);
      kernel_translation_maps_.push_back(device_buffer);
    }
  }
#else
  CHECK_EQ(1, device_count_);
#endif  // HAVE_CUDA
  if (memory_level_ == Data_Namespace::CPU_LEVEL) {
    kernel_translation_maps_.push_back(data());
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

const int8_t * StringDictionaryTranslationMgr::data

(

)

const

Definition at line 316 of file StringDictionaryTranslationMgr.cpp.

References StringDictionaryProxy::TranslationMap< T >::data(), dest_type_is_string_, host_numeric_translation_map_, and isMapValid().

Referenced by createKernelBuffers().

                                                         {
  if (isMapValid()) {
    if (dest_type_is_string_) {
      return reinterpret_cast<const int8_t*>(host_translation_map_->data());
    } else {
      return reinterpret_cast<const int8_t*>(host_numeric_translation_map_->data());
    }
  }
  return nullptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool StringDictionaryTranslationMgr::isMapValid

(

)

const

Definition at line 308 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, StringDictionaryProxy::TranslationMap< T >::empty(), and host_numeric_translation_map_.

Referenced by data(), mapSize(), and minSourceStringId().

                                                      {
  if (dest_type_is_string_) {
    return host_translation_map_ && !host_translation_map_->empty();
  } else {
    return host_numeric_translation_map_ && !host_numeric_translation_map_->empty();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t StringDictionaryTranslationMgr::mapSize

(

)

const

Definition at line 335 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, SQLTypeInfo::get_logical_size(), StringDictionaryProxy::TranslationMap< T >::getVectorMap(), host_numeric_translation_map_, isMapValid(), and output_ti_.

Referenced by createKernelBuffers().

                                                     {
  if (isMapValid()) {
    const size_t num_elems = dest_type_is_string_
                                 ? host_translation_map_->getVectorMap().size()
                                 : host_numeric_translation_map_->getVectorMap().size();
    const size_t elem_size =
        dest_type_is_string_ ? output_ti_.get_logical_size() : sizeof(Datum);
    return num_elems * elem_size;
  }
  return 0UL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t StringDictionaryTranslationMgr::minSourceStringId

(

)

const

Definition at line 327 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, StringDictionaryProxy::TranslationMap< T >::domainStart(), host_numeric_translation_map_, and isMapValid().

Referenced by codegen().

                                                                {
  if (isMapValid()) {
    return dest_type_is_string_ ? host_translation_map_->domainStart()
                                : host_numeric_translation_map_->domainStart();
  }
  return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Data_Namespace::DataMgr* StringDictionaryTranslationMgr::data_mgr_

private

Definition at line 95 of file StringDictionaryTranslationMgr.h.

Referenced by createKernelBuffers(), and ~StringDictionaryTranslationMgr().

const shared::StringDictKey StringDictionaryTranslationMgr::dest_string_dict_key_

private

Definition at line 87 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const bool StringDictionaryTranslationMgr::dest_type_is_string_

private

Definition at line 96 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap(), codegen(), data(), isMapValid(), mapSize(), and minSourceStringId().

std::vector<Data_Namespace::AbstractBuffer*> StringDictionaryTranslationMgr::device_buffers_

private

Definition at line 101 of file StringDictionaryTranslationMgr.h.

Referenced by createKernelBuffers(), and ~StringDictionaryTranslationMgr().

const int StringDictionaryTranslationMgr::device_count_

private

Definition at line 93 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and createKernelBuffers().

Executor* StringDictionaryTranslationMgr::executor_

private

Definition at line 94 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap(), and codegen().

const bool StringDictionaryTranslationMgr::has_null_string_op_

private

Definition at line 91 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and StringDictionaryTranslationMgr().

const StringDictionaryProxy::TranslationMap<Datum>* StringDictionaryTranslationMgr::host_numeric_translation_map_

private

Initial value:

{
      nullptr}

Definition at line 98 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap(), data(), isMapValid(), mapSize(), and minSourceStringId().

std::vector<const int8_t*> StringDictionaryTranslationMgr::kernel_translation_maps_

private

Definition at line 100 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and createKernelBuffers().

const Data_Namespace::MemoryLevel StringDictionaryTranslationMgr::memory_level_

private

Definition at line 92 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), createKernelBuffers(), and StringDictionaryTranslationMgr().

const StringDictionaryProxy::IdMap* host_translation_map_ StringDictionaryTranslationMgr::nullptr

private

Definition at line 97 of file StringDictionaryTranslationMgr.h.

const SQLTypeInfo StringDictionaryTranslationMgr::output_ti_

private

Definition at line 89 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and mapSize().

const shared::StringDictKey StringDictionaryTranslationMgr::source_string_dict_key_

private

Definition at line 86 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const std::vector<StringOps_Namespace::StringOpInfo> StringDictionaryTranslationMgr::string_op_infos_

private

Definition at line 90 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const bool StringDictionaryTranslationMgr::translate_intersection_only_

private

Definition at line 88 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/StringDictionaryTranslationMgr.h
• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/StringDictionaryTranslationMgr.cpp