foreign_storage::Csv Namespace Reference

Namespaces
	anonymous_namespace{CsvShared.cpp}

Functions
bool	validate_and_get_is_s3_select (const ForeignTable *foreign_table)
void	validate_options (const ForeignTable *foreign_table)
import_export::CopyParams	validate_and_get_copy_params (const ForeignTable *foreign_table)
void	init_chunk_for_column (const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)
std::shared_ptr< ChunkMetadata >	get_placeholder_metadata (const ColumnDescriptor *column, size_t num_elements)

Function Documentation

std::shared_ptr<ChunkMetadata> foreign_storage::Csv::get_placeholder_metadata	(	const ColumnDescriptor *	column,
		size_t	num_elements
	)

Definition at line 254 of file CsvShared.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_array(), and SQLTypeInfo::is_varlen_indeed().

                                                                             {
  ForeignStorageBuffer empty_buffer;
  // Use default encoder metadata as in parquet wrapper
  empty_buffer.initEncoder(column->columnType);
  auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(column->columnType);
  chunk_metadata->numElements = num_elements;

  if (!column->columnType.is_varlen_indeed()) {
    chunk_metadata->numBytes = column->columnType.get_size() * num_elements;
  }
  // min/max not set by default for arrays, so get from elem type encoder
  if (column->columnType.is_array()) {
    ForeignStorageBuffer scalar_buffer;
    scalar_buffer.initEncoder(column->columnType.get_elem_type());
    auto scalar_metadata =
        scalar_buffer.getEncoder()->getMetadata(column->columnType.get_elem_type());
    chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
    chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
  }
  chunk_metadata->chunkStats.has_nulls = true;
  return chunk_metadata;
}

Here is the call graph for this function:

void foreign_storage::Csv::init_chunk_for_column	(	const ChunkKey &	chunk_key,
		const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &	chunk_metadata_map,
		const std::map< ChunkKey, AbstractBuffer * > &	buffers,
		Chunk_NS::Chunk &	chunk
	)

Definition at line 201 of file CsvShared.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::initEncoder(), Catalog_Namespace::SysCatalog::instance(), Data_Namespace::AbstractBuffer::reserve(), Chunk_NS::Chunk::setBuffer(), Chunk_NS::Chunk::setColumnDesc(), Chunk_NS::Chunk::setIndexBuffer(), Data_Namespace::AbstractBuffer::size(), and UNREACHABLE.

                          {
  auto catalog =
      Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
  CHECK(catalog);

  ChunkKey data_chunk_key = chunk_key;
  AbstractBuffer* data_buffer = nullptr;
  AbstractBuffer* index_buffer = nullptr;
  const auto column = catalog->getMetadataForColumnUnlocked(
      chunk_key[CHUNK_KEY_TABLE_IDX], chunk_key[CHUNK_KEY_COLUMN_IDX]);

  if (column->columnType.is_varlen_indeed()) {
    data_chunk_key.push_back(1);
    ChunkKey index_chunk_key = chunk_key;
    index_chunk_key.push_back(2);

    CHECK(buffers.find(data_chunk_key) != buffers.end());
    CHECK(buffers.find(index_chunk_key) != buffers.end());

    data_buffer = buffers.find(data_chunk_key)->second;
    index_buffer = buffers.find(index_chunk_key)->second;
    CHECK_EQ(data_buffer->size(), static_cast<size_t>(0));
    CHECK_EQ(index_buffer->size(), static_cast<size_t>(0));

    size_t index_offset_size{0};
    if (column->columnType.is_string() || column->columnType.is_geometry()) {
      index_offset_size = sizeof(StringOffsetT);
    } else if (column->columnType.is_array()) {
      index_offset_size = sizeof(ArrayOffsetT);
    } else {
      UNREACHABLE();
    }
    CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
    index_buffer->reserve(index_offset_size *
                          (chunk_metadata_map.at(data_chunk_key)->numElements + 1));
  } else {
    data_chunk_key = chunk_key;
    CHECK(buffers.find(data_chunk_key) != buffers.end());
    data_buffer = buffers.find(data_chunk_key)->second;
  }
  CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
  data_buffer->reserve(chunk_metadata_map.at(data_chunk_key)->numBytes);

  chunk.setColumnDesc(column);
  chunk.setBuffer(data_buffer);
  chunk.setIndexBuffer(index_buffer);
  chunk.initEncoder();
}

Here is the call graph for this function:

import_export::CopyParams foreign_storage::Csv::validate_and_get_copy_params

(

const ForeignTable *

foreign_table

)

Definition at line 146 of file CsvShared.cpp.

References foreign_storage::OptionsContainer::options, import_export::CopyParams::plain_text, foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_and_get_bool_value(), foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_and_get_delimiter(), and foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_and_get_string_with_length().

Referenced by validate_options().

                                       {
  import_export::CopyParams copy_params{};
  copy_params.plain_text = true;
  if (const auto& value =
          validate_and_get_string_with_length(foreign_table, "ARRAY_DELIMITER", 1);
      !value.empty()) {
    copy_params.array_delim = value[0];
  }
  if (const auto& value =
          validate_and_get_string_with_length(foreign_table, "ARRAY_MARKER", 2);
      !value.empty()) {
    copy_params.array_begin = value[0];
    copy_params.array_end = value[1];
  }
  if (auto it = foreign_table->options.find("BUFFER_SIZE");
      it != foreign_table->options.end()) {
    copy_params.buffer_size = std::stoi(it->second);
  }
  if (const auto& value = validate_and_get_delimiter(foreign_table, "DELIMITER");
      !value.empty()) {
    copy_params.delimiter = value[0];
  }
  if (const auto& value = validate_and_get_string_with_length(foreign_table, "ESCAPE", 1);
      !value.empty()) {
    copy_params.escape = value[0];
  }
  auto has_header = validate_and_get_bool_value(foreign_table, "HEADER");
  if (has_header.has_value()) {
    if (has_header.value()) {
      copy_params.has_header = import_export::ImportHeaderRow::HAS_HEADER;
    } else {
      copy_params.has_header = import_export::ImportHeaderRow::NO_HEADER;
    }
  }
  if (const auto& value = validate_and_get_delimiter(foreign_table, "LINE_DELIMITER");
      !value.empty()) {
    copy_params.line_delim = value[0];
  }
  copy_params.lonlat =
      validate_and_get_bool_value(foreign_table, "LONLAT").value_or(copy_params.lonlat);

  if (auto it = foreign_table->options.find("NULLS");
      it != foreign_table->options.end()) {
    copy_params.null_str = it->second;
  }
  if (const auto& value = validate_and_get_string_with_length(foreign_table, "QUOTE", 1);
      !value.empty()) {
    copy_params.quote = value[0];
  }
  copy_params.quoted =
      validate_and_get_bool_value(foreign_table, "QUOTED").value_or(copy_params.quoted);
  return copy_params;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::Csv::validate_and_get_is_s3_select

(

const ForeignTable *

foreign_table

)

Definition at line 113 of file CsvShared.cpp.

References foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::options, foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by validate_options().

                                                                      {
  static constexpr const char* S3_DIRECT = "S3_DIRECT";
  static constexpr const char* S3_SELECT = "S3_SELECT";
  static constexpr const char* S3_ACCESS_TYPE = "S3_ACCESS_TYPE";
  auto access_type = foreign_table->options.find(S3_ACCESS_TYPE);

  if (access_type != foreign_table->options.end()) {
    auto& server_options = foreign_table->foreign_server->options;
    if (server_options.find(AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY)->second !=
        AbstractFileStorageDataWrapper::S3_STORAGE_TYPE) {
      throw std::runtime_error{
          "The \"" + std::string{S3_ACCESS_TYPE} +
          "\" option is only valid for foreign tables using servers with \"" +
          AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY + "\" option value of \"" +
          AbstractFileStorageDataWrapper::S3_STORAGE_TYPE + "\"."};
    }
    if (access_type->second != S3_DIRECT && access_type->second != S3_SELECT) {
      throw std::runtime_error{
          "Invalid value provided for the \"" + std::string{S3_ACCESS_TYPE} +
          "\" option. Value must be one of the following: " + S3_DIRECT + ", " +
          S3_SELECT + "."};
    }
    return (access_type->second == S3_SELECT);
  } else {
    return false;
  }
}

Here is the caller graph for this function:

void foreign_storage::Csv::validate_options

(

const ForeignTable *

foreign_table

)

Definition at line 141 of file CsvShared.cpp.

References validate_and_get_copy_params(), and validate_and_get_is_s3_select().

                                                         {
  validate_and_get_copy_params(foreign_table);
  validate_and_get_is_s3_select(foreign_table);
}

Here is the call graph for this function: