anonymous_namespace{ArrowResultSetConverter.cpp} Namespace Reference

Classes
class	ResultSetBuffer
class	null_type
struct	null_type< TYPE, std::enable_if_t< std::is_integral< TYPE >::value > >
struct	null_type< TYPE, std::enable_if_t< std::is_floating_point< TYPE >::value > >

Typedefs
template<typename TYPE >
using	null_type_t = typename null_type< TYPE >::type

Functions
SQLTypes	get_dict_index_type (const SQLTypeInfo &ti)
SQLTypes	get_physical_type (const SQLTypeInfo &ti)
template<typename TYPE , typename VALUE_ARRAY_TYPE >
void	create_or_append_value (const ScalarTargetValue &val_cty, std::shared_ptr< ValueArray > &values, const size_t max_size)
template<typename TYPE , typename VALUE_ARRAY_TYPE >
void	create_or_append_value (const ArrayTargetValue &val_ctys, std::shared_ptr< ValueArray > &values, const size_t max_size)
void	create_or_append_validity (const ArrayTargetValue &value, const SQLTypeInfo &col_type, std::shared_ptr< std::vector< bool >> &null_bitmap, const size_t max_size)
template<typename TYPE >
void	create_or_append_validity (const ScalarTargetValue &value, const SQLTypeInfo &col_type, std::shared_ptr< std::vector< bool >> &null_bitmap, const size_t max_size)
template<typename C_TYPE , typename ARROW_TYPE = typename arrow::CTypeTraits<C_TYPE>::ArrowType>
void	convert_column (ResultSetPtr result, size_t col, size_t entry_count, std::shared_ptr< arrow::Array > &out)
std::pair< key_t, void * >	get_shm (size_t shmsz)
std::pair< key_t, std::shared_ptr< arrow::Buffer > >	get_shm_buffer (size_t size)
void	remap_string_values (const ArrowResultSetConverter::ColumnBuilder &column_builder, const std::vector< uint8_t > &bitmap, std::vector< int64_t > &vec1d)
std::shared_ptr< arrow::DataType >	get_arrow_type (const SQLTypeInfo &sql_type, const ExecutorDeviceType device_type)
template<typename BUILDER_TYPE , typename VALUE_ARRAY_TYPE >
void	appendToColumnBuilder (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)
template<>
void	appendToColumnBuilder< arrow::Decimal128Builder, int64_t > (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)
template<>
void	appendToColumnBuilder< arrow::StringBuilder, std::string > (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)
template<>
void	appendToColumnBuilder< arrow::StringDictionary32Builder, int32_t > (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)
template<typename BUILDER_TYPE , typename VALUE_TYPE >
void	appendToListColumnBuilder (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)
template<>
void	appendToListColumnBuilder< arrow::StringDictionaryBuilder, int64_t > (ArrowResultSetConverter::ColumnBuilder &column_builder, const ValueArray &values, const std::shared_ptr< std::vector< bool >> &is_valid)

Typedef Documentation

template<typename TYPE >

using anonymous_namespace{ArrowResultSetConverter.cpp}::null_type_t = typedef typename null_type<TYPE>::type

Definition at line 228 of file ArrowResultSetConverter.cpp.

Function Documentation

template<typename BUILDER_TYPE , typename VALUE_ARRAY_TYPE >

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToColumnBuilder	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1396 of file ArrowResultSetConverter.cpp.

References ARROW_THROW_NOT_OK, ArrowResultSetConverter::ColumnBuilder::builder, CHECK, ArrowResultSetConverter::ColumnBuilder::field, DateConverters::get_epoch_days_from_seconds(), kMilliSecsPerSec, and shared::transform().

                                                                           {
  static_assert(!std::is_same<BUILDER_TYPE, arrow::StringDictionary32Builder>::value,
                "Dictionary encoded string builder requires function specialization.");

  std::vector<VALUE_ARRAY_TYPE> vals = boost::get<std::vector<VALUE_ARRAY_TYPE>>(values);

  if (scale_epoch_values<BUILDER_TYPE>()) {
    auto scale_sec_to_millisec = [](auto seconds) { return seconds * kMilliSecsPerSec; };
    auto scale_values = [&](auto epoch) {
      return std::is_same<BUILDER_TYPE, arrow::Date32Builder>::value
                 ? DateConverters::get_epoch_days_from_seconds(epoch)
                 : scale_sec_to_millisec(epoch);
    };
    std::transform(vals.begin(), vals.end(), vals.begin(), scale_values);
  }

  auto typed_builder = dynamic_cast<BUILDER_TYPE*>(column_builder.builder.get());
  CHECK(typed_builder);
  if (column_builder.field->nullable()) {
    CHECK(is_valid.get());
    ARROW_THROW_NOT_OK(typed_builder->AppendValues(vals, *is_valid));
  } else {
    ARROW_THROW_NOT_OK(typed_builder->AppendValues(vals));
  }
}

Here is the call graph for this function:

template<>

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToColumnBuilder< arrow::Decimal128Builder, int64_t >	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1425 of file ArrowResultSetConverter.cpp.

References ARROW_THROW_NOT_OK, CHECK, and CHECK_EQ.

                                                    {
  std::vector<int64_t> vals = boost::get<std::vector<int64_t>>(values);
  auto typed_builder =
      dynamic_cast<arrow::Decimal128Builder*>(column_builder.builder.get());
  CHECK(typed_builder);
  CHECK_EQ(is_valid->size(), vals.size());
  if (column_builder.field->nullable()) {
    CHECK(is_valid.get());
    for (size_t i = 0; i < vals.size(); i++) {
      const auto v = vals[i];
      const auto valid = (*is_valid)[i];
      if (valid) {
        ARROW_THROW_NOT_OK(typed_builder->Append(v));
      } else {
        ARROW_THROW_NOT_OK(typed_builder->AppendNull());
      }
    }
  } else {
    for (const auto& v : vals) {
      ARROW_THROW_NOT_OK(typed_builder->Append(v));
    }
  }
}

template<>

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToColumnBuilder< arrow::StringBuilder, std::string >	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1453 of file ArrowResultSetConverter.cpp.

References ARROW_THROW_NOT_OK, CHECK, and CHECK_EQ.

                                                    {
  std::vector<std::string> vals = boost::get<std::vector<std::string>>(values);
  auto typed_builder = dynamic_cast<arrow::StringBuilder*>(column_builder.builder.get());
  CHECK(typed_builder);
  CHECK_EQ(is_valid->size(), vals.size());

  if (column_builder.field->nullable()) {
    CHECK(is_valid.get());

    // TODO: Generate this instead of the boolean bitmap
    std::vector<uint8_t> transformed_bitmap;
    transformed_bitmap.reserve(is_valid->size());
    std::for_each(
        is_valid->begin(), is_valid->end(), [&transformed_bitmap](const bool is_valid) {
          transformed_bitmap.push_back(is_valid ? 1 : 0);
        });
    ARROW_THROW_NOT_OK(typed_builder->AppendValues(vals, transformed_bitmap.data()));
  } else {
    ARROW_THROW_NOT_OK(typed_builder->AppendValues(vals));
  }
}

template<>

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToColumnBuilder< arrow::StringDictionary32Builder, int32_t >	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1479 of file ArrowResultSetConverter.cpp.

References ALL_STRINGS_REMAPPED, ARROW_THROW_NOT_OK, CHECK, and INVALID.

                                                    {
  auto typed_builder =
      dynamic_cast<arrow::StringDictionary32Builder*>(column_builder.builder.get());
  CHECK(typed_builder);

  std::vector<int32_t> vals = boost::get<std::vector<int32_t>>(values);
  // remap negative values if ArrowStringRemapMode == ONLY_TRANSIENT_STRINGS_REMAPPED or
  // everything if ALL_STRINGS_REMAPPED
  CHECK(column_builder.string_remap_mode != ArrowStringRemapMode::INVALID);
  for (size_t i = 0; i < vals.size(); i++) {
    auto& val = vals[i];
    if ((column_builder.string_remap_mode == ArrowStringRemapMode::ALL_STRINGS_REMAPPED ||
         val < 0) &&
        (*is_valid)[i]) {
      vals[i] = column_builder.string_remapping.at(val);
    }
  }

  if (column_builder.field->nullable()) {
    CHECK(is_valid.get());
    // TODO(adb): Generate this instead of the boolean bitmap
    std::vector<uint8_t> transformed_bitmap;
    transformed_bitmap.reserve(is_valid->size());
    std::for_each(
        is_valid->begin(), is_valid->end(), [&transformed_bitmap](const bool is_valid) {
          transformed_bitmap.push_back(is_valid ? 1 : 0);
        });

    ARROW_THROW_NOT_OK(typed_builder->AppendIndices(
        vals.data(), static_cast<int64_t>(vals.size()), transformed_bitmap.data()));
  } else {
    ARROW_THROW_NOT_OK(
        typed_builder->AppendIndices(vals.data(), static_cast<int64_t>(vals.size())));
  }
}

template<typename BUILDER_TYPE , typename VALUE_TYPE >

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToListColumnBuilder	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1519 of file ArrowResultSetConverter.cpp.

References ARROW_THROW_NOT_OK, ArrowResultSetConverter::ColumnBuilder::builder, CHECK, gpu_enabled::copy(), ArrowResultSetConverter::ColumnBuilder::field, and shared::transform().

                                                                               {
  Vec2<VALUE_TYPE> vals = boost::get<Vec2<VALUE_TYPE>>(values);
  auto list_builder = dynamic_cast<arrow::ListBuilder*>(column_builder.builder.get());
  CHECK(list_builder);

  auto value_builder = static_cast<BUILDER_TYPE*>(list_builder->value_builder());

  if (column_builder.field->nullable()) {
    for (size_t i = 0; i < vals.size(); i++) {
      if ((*is_valid)[i]) {
        const auto& val = vals[i];
        std::vector<uint8_t> bitmap(val.size());
        std::transform(val.begin(), val.end(), bitmap.begin(), [](VALUE_TYPE pvalue) {
          return static_cast<VALUE_TYPE>(pvalue) != null_type<VALUE_TYPE>::value;
        });
        ARROW_THROW_NOT_OK(list_builder->Append());
        if constexpr (std::is_same_v<BUILDER_TYPE, arrow::BooleanBuilder>) {
          std::vector<uint8_t> bval(val.size());
          std::copy(val.begin(), val.end(), bval.begin());
          ARROW_THROW_NOT_OK(
              value_builder->AppendValues(bval.data(), bval.size(), bitmap.data()));
        } else {
          ARROW_THROW_NOT_OK(
              value_builder->AppendValues(val.data(), val.size(), bitmap.data()));
        }
      } else {
        ARROW_THROW_NOT_OK(list_builder->AppendNull());
      }
    }
  } else {
    for (size_t i = 0; i < vals.size(); i++) {
      if ((*is_valid)[i]) {
        const auto& val = vals[i];
        ARROW_THROW_NOT_OK(list_builder->Append());
        if constexpr (std::is_same_v<BUILDER_TYPE, arrow::BooleanBuilder>) {
          std::vector<uint8_t> bval(val.size());
          std::copy(val.begin(), val.end(), bval.begin());
          ARROW_THROW_NOT_OK(value_builder->AppendValues(bval.data(), bval.size()));
        } else {
          ARROW_THROW_NOT_OK(value_builder->AppendValues(val.data(), val.size()));
        }
      } else {
        ARROW_THROW_NOT_OK(list_builder->AppendNull());
      }
    }
  }
}

Here is the call graph for this function:

template<>

void anonymous_namespace{ArrowResultSetConverter.cpp}::appendToListColumnBuilder< arrow::StringDictionaryBuilder, int64_t >	(	ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const ValueArray &	values,
		const std::shared_ptr< std::vector< bool >> &	is_valid
	)

Definition at line 1570 of file ArrowResultSetConverter.cpp.

References ARROW_THROW_NOT_OK, CHECK, remap_string_values(), and shared::transform().

                                                    {
  Vec2<int64_t> vec2d = boost::get<Vec2<int64_t>>(values);

  auto* list_builder = dynamic_cast<arrow::ListBuilder*>(column_builder.builder.get());
  CHECK(list_builder);

  // todo: fix value_builder being a StringDictionaryBuilder and not
  // StringDictionary32Builder
  auto* value_builder =
      dynamic_cast<arrow::StringDictionaryBuilder*>(list_builder->value_builder());
  CHECK(value_builder);

  if (column_builder.field->nullable()) {
    for (size_t i = 0; i < vec2d.size(); i++) {
      if ((*is_valid)[i]) {
        auto& vec1d = vec2d[i];
        std::vector<uint8_t> bitmap(vec1d.size());
        std::transform(vec1d.begin(), vec1d.end(), bitmap.begin(), [](int64_t pvalue) {
          return pvalue != null_type<int32_t>::value;
        });
        ARROW_THROW_NOT_OK(list_builder->Append());
        ARROW_THROW_NOT_OK(value_builder->InsertMemoValues(*column_builder.string_array));
        remap_string_values(column_builder, bitmap, vec1d);
        ARROW_THROW_NOT_OK(value_builder->AppendIndices(
            vec1d.data(), static_cast<int64_t>(vec1d.size()), bitmap.data()));
      } else {
        ARROW_THROW_NOT_OK(list_builder->AppendNull());
      }
    }
  } else {
    for (size_t i = 0; i < vec2d.size(); i++) {
      if ((*is_valid)[i]) {
        auto& vec1d = vec2d[i];
        ARROW_THROW_NOT_OK(list_builder->Append());
        remap_string_values(column_builder, {}, vec1d);
        ARROW_THROW_NOT_OK(value_builder->AppendIndices(vec1d.data(), vec1d.size()));
      } else {
        ARROW_THROW_NOT_OK(list_builder->AppendNull());
      }
    }
  }
}

Here is the call graph for this function:

template<typename C_TYPE , typename ARROW_TYPE = typename arrow::CTypeTraits<C_TYPE>::ArrowType>

void anonymous_namespace{ArrowResultSetConverter.cpp}::convert_column	(	ResultSetPtr	result,
		size_t	col,
		size_t	entry_count,
		std::shared_ptr< arrow::Array > &	out
	)

Definition at line 232 of file ArrowResultSetConverter.cpp.

References CHECK, and run_benchmark_import::res.

                                                    {
  CHECK(sizeof(C_TYPE) == result->getColType(col).get_size());

  std::shared_ptr<arrow::Buffer> values;
  std::shared_ptr<arrow::Buffer> is_valid;
  const int64_t buf_size = entry_count * sizeof(C_TYPE);
  if (result->isZeroCopyColumnarConversionPossible(col)) {
    values.reset(new ResultSetBuffer(
        reinterpret_cast<const uint8_t*>(result->getColumnarBuffer(col)),
        buf_size,
        result));
  } else {
    auto res = arrow::AllocateBuffer(buf_size);
    CHECK(res.ok());
    values = std::move(res).ValueOrDie();
    result->copyColumnIntoBuffer(
        col, reinterpret_cast<int8_t*>(values->mutable_data()), buf_size);
  }

  int64_t null_count = 0;
  auto res = arrow::AllocateBuffer((entry_count + 7) / 8);
  CHECK(res.ok());
  is_valid = std::move(res).ValueOrDie();

  auto is_valid_data = is_valid->mutable_data();

  const null_type_t<C_TYPE>* vals =
      reinterpret_cast<const null_type_t<C_TYPE>*>(values->data());
  null_type_t<C_TYPE> null_val = null_type<C_TYPE>::value;

  size_t unroll_count = entry_count & 0xFFFFFFFFFFFFFFF8ULL;
  for (size_t i = 0; i < unroll_count; i += 8) {
    uint8_t valid_byte = 0;
    uint8_t valid;
    valid = vals[i + 0] != null_val;
    valid_byte |= valid << 0;
    null_count += !valid;
    valid = vals[i + 1] != null_val;
    valid_byte |= valid << 1;
    null_count += !valid;
    valid = vals[i + 2] != null_val;
    valid_byte |= valid << 2;
    null_count += !valid;
    valid = vals[i + 3] != null_val;
    valid_byte |= valid << 3;
    null_count += !valid;
    valid = vals[i + 4] != null_val;
    valid_byte |= valid << 4;
    null_count += !valid;
    valid = vals[i + 5] != null_val;
    valid_byte |= valid << 5;
    null_count += !valid;
    valid = vals[i + 6] != null_val;
    valid_byte |= valid << 6;
    null_count += !valid;
    valid = vals[i + 7] != null_val;
    valid_byte |= valid << 7;
    null_count += !valid;
    is_valid_data[i >> 3] = valid_byte;
  }
  if (unroll_count != entry_count) {
    uint8_t valid_byte = 0;
    for (size_t i = unroll_count; i < entry_count; ++i) {
      bool valid = vals[i] != null_val;
      valid_byte |= valid << (i & 7);
      null_count += !valid;
    }
    is_valid_data[unroll_count >> 3] = valid_byte;
  }

  if (!null_count) {
    is_valid.reset();
  }

  // TODO: support date/time + scaling
  // TODO: support booleans
  if (null_count) {
    out.reset(
        new arrow::NumericArray<ARROW_TYPE>(entry_count, values, is_valid, null_count));
  } else {
    out.reset(new arrow::NumericArray<ARROW_TYPE>(entry_count, values));
  }
}

void anonymous_namespace{ArrowResultSetConverter.cpp}::create_or_append_validity	(	const ArrayTargetValue &	value,
		const SQLTypeInfo &	col_type,
		std::shared_ptr< std::vector< bool >> &	null_bitmap,
		const size_t	max_size
	)

Definition at line 159 of file ArrowResultSetConverter.cpp.

References CHECK, and SQLTypeInfo::get_notnull().

Referenced by ArrowResultSetConverter::getArrowBatch().

                                                      {
  if (col_type.get_notnull()) {
    CHECK(!null_bitmap);
    return;
  }

  if (!null_bitmap) {
    null_bitmap = std::make_shared<std::vector<bool>>();
    null_bitmap->reserve(max_size);
  }
  CHECK(null_bitmap);
  null_bitmap->push_back(value ? true : false);
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename TYPE >

void anonymous_namespace{ArrowResultSetConverter.cpp}::create_or_append_validity	(	const ScalarTargetValue &	value,
		const SQLTypeInfo &	col_type,
		std::shared_ptr< std::vector< bool >> &	null_bitmap,
		const size_t	max_size
	)

Definition at line 177 of file ArrowResultSetConverter.cpp.

References CHECK, SQLTypeInfo::get_notnull(), inline_fp_null_val(), inline_int_null_val(), SQLTypeInfo::is_boolean(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_dict_encoded_string(), SQLTypeInfo::is_fp(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), and UNREACHABLE.

                                                      {
  if (col_type.get_notnull()) {
    CHECK(!null_bitmap);
    return;
  }
  auto pvalue = boost::get<TYPE>(&value);
  CHECK(pvalue);
  bool is_valid = false;
  if constexpr (std::is_same_v<TYPE, NullableString>) {
    is_valid = boost::get<std::string>(pvalue) != nullptr;
  } else {
    if (col_type.is_boolean()) {
      is_valid = inline_int_null_val(col_type) != static_cast<int8_t>(*pvalue);
    } else if (col_type.is_dict_encoded_string()) {
      is_valid = inline_int_null_val(col_type) != static_cast<int32_t>(*pvalue);
    } else if (col_type.is_integer() || col_type.is_time() || col_type.is_decimal()) {
      is_valid = inline_int_null_val(col_type) != static_cast<int64_t>(*pvalue);
    } else if (col_type.is_fp()) {
      is_valid = inline_fp_null_val(col_type) != static_cast<double>(*pvalue);
    } else {
      UNREACHABLE();
    }
  }

  if (!null_bitmap) {
    null_bitmap = std::make_shared<std::vector<bool>>();
    null_bitmap->reserve(max_size);
  }
  CHECK(null_bitmap);
  null_bitmap->push_back(is_valid);
}

Here is the call graph for this function:

template<typename TYPE , typename VALUE_ARRAY_TYPE >

void anonymous_namespace{ArrowResultSetConverter.cpp}::create_or_append_value	(	const ScalarTargetValue &	val_cty,
		std::shared_ptr< ValueArray > &	values,
		const size_t	max_size
	)

Definition at line 110 of file ArrowResultSetConverter.cpp.

References CHECK.

                                                   {
  if (!values) {
    values = std::make_shared<ValueArray>(std::vector<TYPE>());
    boost::get<std::vector<TYPE>>(*values).reserve(max_size);
  }
  CHECK(values);
  auto values_ty = boost::get<std::vector<TYPE>>(values.get());
  CHECK(values_ty);

  auto pval_cty = boost::get<VALUE_ARRAY_TYPE>(&val_cty);
  CHECK(pval_cty);
  if constexpr (std::is_same_v<VALUE_ARRAY_TYPE, NullableString>) {
    if (auto str = boost::get<std::string>(pval_cty)) {
      values_ty->push_back(*str);
    } else {
      values_ty->push_back("");
    }
  } else {
    auto val_ty = static_cast<TYPE>(*pval_cty);
    values_ty->push_back(val_ty);
  }
}

template<typename TYPE , typename VALUE_ARRAY_TYPE >

void anonymous_namespace{ArrowResultSetConverter.cpp}::create_or_append_value	(	const ArrayTargetValue &	val_ctys,
		std::shared_ptr< ValueArray > &	values,
		const size_t	max_size
	)

Definition at line 136 of file ArrowResultSetConverter.cpp.

References CHECK.

                                                   {
  if (!values) {
    values = std::make_shared<ValueArray>(Vec2<TYPE>());
    boost::get<Vec2<TYPE>>(*values).reserve(max_size);
  }
  CHECK(values);

  Vec2<TYPE>* values_ty = boost::get<Vec2<TYPE>>(values.get());
  CHECK(values_ty);

  values_ty->emplace_back(std::vector<TYPE>{});

  if (val_ctys) {
    for (auto val_cty : val_ctys.value()) {
      auto pval_cty = boost::get<VALUE_ARRAY_TYPE>(&val_cty);
      CHECK(pval_cty);
      values_ty->back().emplace_back(static_cast<TYPE>(*pval_cty));
    }
  }
}

std::shared_ptr<arrow::DataType> anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type	(	const SQLTypeInfo &	sql_type,
		const ExecutorDeviceType	device_type
	)

Definition at line 1114 of file ArrowResultSetConverter.cpp.

References foreign_storage::get_physical_type(), SQLTypeInfo::get_precision(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type_name(), GPU, SQLTypeInfo::is_dict_encoded_string(), SQLTypeInfo::is_dict_encoded_type(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and to_string().

Referenced by ArrowResultSetConverter::makeField().

                                                                                      {
  switch (get_physical_type(sql_type)) {
    case kBOOLEAN:
      return arrow::boolean();
    case kTINYINT:
      return arrow::int8();
    case kSMALLINT:
      return arrow::int16();
    case kINT:
      return arrow::int32();
    case kBIGINT:
      return arrow::int64();
    case kFLOAT:
      return arrow::float32();
    case kDOUBLE:
      return arrow::float64();
    case kCHAR:
    case kVARCHAR:
    case kTEXT:
      if (sql_type.is_dict_encoded_string()) {
        auto value_type = std::make_shared<arrow::StringType>();
        return arrow::dictionary(arrow::int32(), value_type, false);
      }
      return arrow::utf8();
    case kDECIMAL:
    case kNUMERIC:
      return arrow::decimal(sql_type.get_precision(), sql_type.get_scale());
    case kTIME:
      return time32(arrow::TimeUnit::SECOND);
    case kDATE: {
      // TODO(wamsi) : Remove date64() once date32() support is added in cuDF. date32()
      // Currently support for date32() is missing in cuDF.Hence, if client requests for
      // date on GPU, return date64() for the time being, till support is added.
      if (device_type == ExecutorDeviceType::GPU) {
        return arrow::date64();
      } else {
        return arrow::date32();
      }
    }
    case kTIMESTAMP:
      switch (sql_type.get_precision()) {
        case 0:
          return timestamp(arrow::TimeUnit::SECOND);
        case 3:
          return timestamp(arrow::TimeUnit::MILLI);
        case 6:
          return timestamp(arrow::TimeUnit::MICRO);
        case 9:
          return timestamp(arrow::TimeUnit::NANO);
        default:
          throw std::runtime_error(
              "Unsupported timestamp precision for Arrow result sets: " +
              std::to_string(sql_type.get_precision()));
      }
    case kARRAY:
      switch (sql_type.get_subtype()) {
        case kBOOLEAN:
          return arrow::list(arrow::boolean());
        case kTINYINT:
          return arrow::list(arrow::int8());
        case kSMALLINT:
          return arrow::list(arrow::int16());
        case kINT:
          return arrow::list(arrow::int32());
        case kBIGINT:
          return arrow::list(arrow::int64());
        case kFLOAT:
          return arrow::list(arrow::float32());
        case kDOUBLE:
          return arrow::list(arrow::float64());
        case kTEXT:
          if (sql_type.is_dict_encoded_type()) {
            auto value_type = std::make_shared<arrow::StringType>();
            return arrow::list(arrow::dictionary(arrow::int32(), value_type, false));
          }
        default:
          throw std::runtime_error("Unsupported array type for Arrow result sets: " +
                                   sql_type.get_type_name());
      }
    case kINTERVAL_DAY_TIME:
    case kINTERVAL_YEAR_MONTH:
    default:
      throw std::runtime_error(sql_type.get_type_name() +
                               " is not supported in Arrow result sets.");
  }
  return nullptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypes anonymous_namespace{ArrowResultSetConverter.cpp}::get_dict_index_type ( const SQLTypeInfo &  ti )

inline

Definition at line 73 of file ArrowResultSetConverter.cpp.

References CHECK, SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::is_dict_encoded_string(), kBIGINT, kINT, kSMALLINT, and kTINYINT.

Referenced by ArrowResultSetConverter::initializeColumnBuilder().

                                                           {
  CHECK(ti.is_dict_encoded_string());
  switch (ti.get_size()) {
    case 1:
      return kTINYINT;
    case 2:
      return kSMALLINT;
    case 4:
      return kINT;
    case 8:
      return kBIGINT;
    default:
      CHECK(false);
  }
  return ti.get_type();
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypes anonymous_namespace{ArrowResultSetConverter.cpp}::get_physical_type ( const SQLTypeInfo &  ti )

inline

Definition at line 90 of file ArrowResultSetConverter.cpp.

References CHECK, SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), IS_INTEGER, kBIGINT, kINT, kSMALLINT, and kTINYINT.

                                                         {
  auto logical_type = ti.get_type();
  if (IS_INTEGER(logical_type)) {
    switch (ti.get_size()) {
      case 1:
        return kTINYINT;
      case 2:
        return kSMALLINT;
      case 4:
        return kINT;
      case 8:
        return kBIGINT;
      default:
        CHECK(false);
    }
  }
  return logical_type;
}

Here is the call graph for this function:

std::pair<key_t, void*> anonymous_namespace{ArrowResultSetConverter.cpp}::get_shm

(

size_t

shmsz

)

Definition at line 320 of file ArrowResultSetConverter.cpp.

Referenced by arrow::get_and_copy_to_shm(), and get_shm_buffer().

                                            {
  if (!shmsz) {
    return std::make_pair(IPC_PRIVATE, nullptr);
  }
  // Generate a new key for a shared memory segment. Keys to shared memory segments
  // are OS global, so we need to try a new key if we encounter a collision. It seems
  // incremental keygen would be deterministically worst-case. If we use a hash
  // (like djb2) + nonce, we could still get collisions if multiple clients specify
  // the same nonce, so using rand() in lieu of a better approach
  // TODO(ptaylor): Is this common? Are these assumptions true?
  auto key = static_cast<key_t>(rand());
  int shmid = -1;
  // IPC_CREAT - indicates we want to create a new segment for this key if it doesn't
  // exist IPC_EXCL - ensures failure if a segment already exists for this key
  while ((shmid = shmget(key, shmsz, IPC_CREAT | IPC_EXCL | 0666)) < 0) {
    // If shmget fails and errno is one of these four values, try a new key.
    // TODO(ptaylor): is checking for the last three values really necessary? Checking
    // them by default to be safe. EEXIST - a shared memory segment is already associated
    // with this key EACCES - a shared memory segment is already associated with this key,
    // but we don't have permission to access it EINVAL - a shared memory segment is
    // already associated with this key, but the size is less than shmsz ENOENT -
    // IPC_CREAT was not set in shmflg and no shared memory segment associated with key
    // was found
    if (!(errno & (EEXIST | EACCES | EINVAL | ENOENT))) {
      throw std::runtime_error("failed to create a shared memory");
    }
    key = static_cast<key_t>(rand());
  }
  // get a pointer to the shared memory segment
  auto ipc_ptr = shmat(shmid, NULL, 0);
  if (reinterpret_cast<int64_t>(ipc_ptr) == -1) {
    throw std::runtime_error("failed to attach a shared memory");
  }

  return std::make_pair(key, ipc_ptr);
}

Here is the caller graph for this function:

std::pair<key_t, std::shared_ptr<arrow::Buffer> > anonymous_namespace{ArrowResultSetConverter.cpp}::get_shm_buffer

(

size_t

size

)

Definition at line 358 of file ArrowResultSetConverter.cpp.

References get_shm().

Referenced by ArrowResultSetConverter::getArrowResult().

                                                                       {
#ifdef _MSC_VER
  throw std::runtime_error("Arrow IPC not yet supported on Windows.");
  return std::make_pair(0, nullptr);
#else
  auto [key, ipc_ptr] = get_shm(size);
  std::shared_ptr<arrow::Buffer> buffer(
      new arrow::MutableBuffer(static_cast<uint8_t*>(ipc_ptr), size));
  return std::make_pair<key_t, std::shared_ptr<arrow::Buffer>>(std::move(key),
                                                               std::move(buffer));
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{ArrowResultSetConverter.cpp}::remap_string_values	(	const ArrowResultSetConverter::ColumnBuilder &	column_builder,
		const std::vector< uint8_t > &	bitmap,
		std::vector< int64_t > &	vec1d
	)

Definition at line 371 of file ArrowResultSetConverter.cpp.

References ALL_STRINGS_REMAPPED, ONLY_TRANSIENT_STRINGS_REMAPPED, ArrowResultSetConverter::ColumnBuilder::string_remap_mode, ArrowResultSetConverter::ColumnBuilder::string_remapping, and UNREACHABLE.

Referenced by appendToListColumnBuilder< arrow::StringDictionaryBuilder, int64_t >().

                                                    {
  /*
   remap negative values if ArrowStringRemapMode == ONLY_TRANSIENT_STRINGS_REMAPPED or
   everything if ALL_STRINGS_REMAPPED
  */

  auto all_strings_remapped_bitmap = [&column_builder, &vec1d, &bitmap]() {
    for (size_t i = 0; i < vec1d.size(); i++) {
      if (bitmap[i]) {
        vec1d[i] = column_builder.string_remapping.at(vec1d[i]);
      }
    }
  };

  auto all_strings_remapped = [&column_builder, &vec1d]() {
    for (size_t i = 0; i < vec1d.size(); i++) {
      vec1d[i] = column_builder.string_remapping.at(vec1d[i]);
    }
  };

  auto only_transient_strings_remapped = [&column_builder, &vec1d]() {
    for (size_t i = 0; i < vec1d.size(); i++) {
      if (vec1d[i] < 0) {
        vec1d[i] = column_builder.string_remapping.at(vec1d[i]);
      }
    }
  };

  auto only_transient_strings_remapped_bitmap = [&column_builder, &vec1d, &bitmap]() {
    for (size_t i = 0; i < vec1d.size(); i++) {
      if (bitmap[i] && vec1d[i] < 0) {
        vec1d[i] = column_builder.string_remapping.at(vec1d[i]);
      }
    }
  };

  switch (column_builder.string_remap_mode) {
    case ArrowStringRemapMode::ALL_STRINGS_REMAPPED:
      bitmap.empty() ? all_strings_remapped() : all_strings_remapped_bitmap();
      break;
    case ArrowStringRemapMode::ONLY_TRANSIENT_STRINGS_REMAPPED:
      bitmap.empty() ? only_transient_strings_remapped()
                     : only_transient_strings_remapped_bitmap();
      break;
    default:
      UNREACHABLE();
  }
}

Here is the caller graph for this function: