foreign_storage::ParseBufferRequest Struct Reference

#include <TextFileBufferParser.h>

Collaboration diagram for foreign_storage::ParseBufferRequest:

Public Member Functions
	ParseBufferRequest (const ParseBufferRequest &request)=delete
	ParseBufferRequest (ParseBufferRequest &&request)=default
	ParseBufferRequest (size_t buffer_size, const import_export::CopyParams &copy_params, int db_id, const ForeignTable *foreign_table, const std::set< int > column_filter_set, const std::string &full_path, const bool track_rejected_rows=false)
std::shared_ptr < Catalog_Namespace::Catalog >	getCatalog () const
std::list< const ColumnDescriptor * >	getColumns () const
int32_t	getTableId () const
std::string	getTableName () const
size_t	getMaxFragRows () const
std::string	getFilePath () const

Public Attributes
std::unique_ptr< char[]>	buffer
size_t	buffer_size
size_t	buffer_alloc_size
const import_export::CopyParams	copy_params
const int	db_id
std::unique_ptr < ForeignTableSchema >	foreign_table_schema
std::vector< std::unique_ptr < import_export::TypedImportBuffer > >	import_buffers
size_t	buffer_row_count
size_t	begin_pos
size_t	end_pos
size_t	first_row_index
size_t	file_offset
size_t	process_row_count
std::string	full_path
const bool	track_rejected_rows
size_t	processed_row_count

Detailed Description

Definition at line 26 of file TextFileBufferParser.h.

Constructor & Destructor Documentation

foreign_storage::ParseBufferRequest::ParseBufferRequest ( const ParseBufferRequest &  request )

delete

foreign_storage::ParseBufferRequest::ParseBufferRequest ( ParseBufferRequest &&  request )

default

foreign_storage::ParseBufferRequest::ParseBufferRequest	(	size_t	buffer_size,
		const import_export::CopyParams &	copy_params,
		int	db_id,
		const ForeignTable *	foreign_table,
		const std::set< int >	column_filter_set,
		const std::string &	full_path,
		const bool	track_rejected_rows = false
	)

Definition at line 23 of file TextFileBufferParser.cpp.

References buffer, buffer_size, getCatalog(), getColumns(), import_buffers, IS_STRING, and kENCODING_DICT.

    : buffer_size(buffer_size)
    , buffer_alloc_size(buffer_size)
    , copy_params(copy_params)
    , db_id(db_id)
    , foreign_table_schema(std::make_unique<ForeignTableSchema>(db_id, foreign_table))
    , full_path(full_path)
    , track_rejected_rows(track_rejected_rows) {
  if (buffer_size > 0) {
    buffer = std::make_unique<char[]>(buffer_size);
  }
  // initialize import buffers from columns.
  for (const auto column : getColumns()) {
    if (column_filter_set.find(column->columnId) == column_filter_set.end()) {
      import_buffers.emplace_back(nullptr);
    } else {
      StringDictionary* string_dictionary = nullptr;
      if (column->columnType.is_dict_encoded_string() ||
          (column->columnType.is_array() && IS_STRING(column->columnType.get_subtype()) &&
           column->columnType.get_compression() == kENCODING_DICT)) {
        auto dict_descriptor =
            getCatalog()->getMetadataForDict(column->columnType.get_comp_param(), true);
        string_dictionary = dict_descriptor->stringDict.get();
      }
      import_buffers.emplace_back(
          std::make_unique<import_export::TypedImportBuffer>(column, string_dictionary));
    }
  }
}

Here is the call graph for this function:

Member Function Documentation

std::shared_ptr<Catalog_Namespace::Catalog> foreign_storage::ParseBufferRequest::getCatalog ( ) const

inline

Definition at line 37 of file TextFileBufferParser.h.

References CHECK, Catalog_Namespace::SysCatalog::getCatalog(), and Catalog_Namespace::SysCatalog::instance().

Referenced by foreign_storage::TextFileBufferParser::fillRejectedRowWithInvalidData(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), and ParseBufferRequest().

                                                                    {
    auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
    CHECK(catalog);
    return catalog;
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::list<const ColumnDescriptor*> foreign_storage::ParseBufferRequest::getColumns ( ) const

inline

Definition at line 43 of file TextFileBufferParser.h.

References foreign_table_schema.

Referenced by foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), ParseBufferRequest(), and foreign_storage::populate_chunks().

                                                             {
    return foreign_table_schema->getLogicalAndPhysicalColumns();
  }

Here is the caller graph for this function:

std::string foreign_storage::ParseBufferRequest::getFilePath ( ) const

inline

Definition at line 59 of file TextFileBufferParser.h.

References full_path.

Referenced by foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), foreign_storage::populate_chunks_using_data_blocks(), and foreign_storage::process_data_blocks().

{ return full_path; }

Here is the caller graph for this function:

size_t foreign_storage::ParseBufferRequest::getMaxFragRows ( ) const

inline

Definition at line 55 of file TextFileBufferParser.h.

References foreign_table_schema.

Referenced by foreign_storage::populate_chunks().

                                       {
    return foreign_table_schema->getForeignTable()->maxFragRows;
  }

Here is the caller graph for this function:

int32_t foreign_storage::ParseBufferRequest::getTableId ( ) const

inline

Definition at line 47 of file TextFileBufferParser.h.

References foreign_table_schema.

Referenced by foreign_storage::process_data_blocks().

                                    {
    return foreign_table_schema->getForeignTable()->tableId;
  }

Here is the caller graph for this function:

std::string foreign_storage::ParseBufferRequest::getTableName ( ) const

inline

Definition at line 51 of file TextFileBufferParser.h.

References foreign_table_schema.

Referenced by foreign_storage::parse_file_regions().

                                        {
    return foreign_table_schema->getForeignTable()->tableName;
  }

Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::ParseBufferRequest::begin_pos

Definition at line 72 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), and foreign_storage::populate_chunks().

std::unique_ptr<char[]> foreign_storage::ParseBufferRequest::buffer

Definition at line 62 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), and ParseBufferRequest().

size_t foreign_storage::ParseBufferRequest::buffer_alloc_size

Definition at line 64 of file TextFileBufferParser.h.

size_t foreign_storage::ParseBufferRequest::buffer_row_count

Definition at line 71 of file TextFileBufferParser.h.

Referenced by foreign_storage::populate_chunks().

size_t foreign_storage::ParseBufferRequest::buffer_size

Definition at line 63 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), and ParseBufferRequest().

const import_export::CopyParams foreign_storage::ParseBufferRequest::copy_params

Definition at line 65 of file TextFileBufferParser.h.

Referenced by foreign_storage::TextFileBufferParser::fillRejectedRowWithInvalidData(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), heavydb.thrift.ttypes.TDetectResult::read(), heavydb.thrift.Heavy.detect_column_types_args::read(), heavydb.thrift.Heavy.import_table_args::read(), heavydb.thrift.Heavy.import_geo_table_args::read(), heavydb.thrift.Heavy.get_first_geo_file_in_archive_args::read(), heavydb.thrift.Heavy.get_all_files_in_archive_args::read(), heavydb.thrift.Heavy.get_layers_in_geo_file_args::read(), heavydb.thrift.ttypes.TDetectResult::write(), heavydb.thrift.Heavy.detect_column_types_args::write(), heavydb.thrift.Heavy.import_table_args::write(), heavydb.thrift.Heavy.import_geo_table_args::write(), heavydb.thrift.Heavy.get_first_geo_file_in_archive_args::write(), heavydb.thrift.Heavy.get_all_files_in_archive_args::write(), and heavydb.thrift.Heavy.get_layers_in_geo_file_args::write().

const int foreign_storage::ParseBufferRequest::db_id

Definition at line 66 of file TextFileBufferParser.h.

Referenced by foreign_storage::process_data_blocks(), heavydb.common.ttypes.TStringDictKey::read(), heavydb.thrift.ttypes.TColumnRange::read(), heavydb.thrift.ttypes.TDictionaryGeneration::read(), heavydb.thrift.ttypes.TTableGeneration::read(), heavydb.thrift.ttypes.TTableCacheStatus::read(), heavydb.thrift.ttypes.TInsertData::read(), heavydb.thrift.ttypes.TInsertChunks::read(), heavydb.thrift.Heavy.set_table_epoch_args::read(), heavydb.thrift.Heavy.get_table_epoch_args::read(), heavydb.thrift.Heavy.get_table_epochs_args::read(), heavydb.thrift.Heavy.set_table_epochs_args::read(), heavydb.common.ttypes.TStringDictKey::write(), heavydb.thrift.ttypes.TColumnRange::write(), heavydb.thrift.ttypes.TDictionaryGeneration::write(), heavydb.thrift.ttypes.TTableGeneration::write(), heavydb.thrift.ttypes.TTableCacheStatus::write(), heavydb.thrift.ttypes.TInsertData::write(), heavydb.thrift.ttypes.TInsertChunks::write(), heavydb.thrift.Heavy.set_table_epoch_args::write(), heavydb.thrift.Heavy.get_table_epoch_args::write(), heavydb.thrift.Heavy.get_table_epochs_args::write(), and heavydb.thrift.Heavy.set_table_epochs_args::write().

size_t foreign_storage::ParseBufferRequest::end_pos

Definition at line 73 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), and foreign_storage::RegexFileBufferParser::parseBuffer().

size_t foreign_storage::ParseBufferRequest::file_offset

Definition at line 75 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), and foreign_storage::populate_chunks().

size_t foreign_storage::ParseBufferRequest::first_row_index

Definition at line 74 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), foreign_storage::populate_chunks(), foreign_storage::populate_chunks_using_data_blocks(), and foreign_storage::process_data_blocks().

std::unique_ptr<ForeignTableSchema> foreign_storage::ParseBufferRequest::foreign_table_schema

Definition at line 67 of file TextFileBufferParser.h.

Referenced by getColumns(), getMaxFragRows(), getTableId(), getTableName(), foreign_storage::CsvFileBufferParser::parseBuffer(), and foreign_storage::RegexFileBufferParser::parseBuffer().

std::string foreign_storage::ParseBufferRequest::full_path

Definition at line 77 of file TextFileBufferParser.h.

Referenced by getFilePath().

std::vector<std::unique_ptr<import_export::TypedImportBuffer> > foreign_storage::ParseBufferRequest::import_buffers

Definition at line 68 of file TextFileBufferParser.h.

Referenced by foreign_storage::TextFileBufferParser::fillRejectedRowWithInvalidData(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), ParseBufferRequest(), foreign_storage::populate_chunks(), foreign_storage::populate_chunks_using_data_blocks(), and foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::skip_column_import().

size_t foreign_storage::ParseBufferRequest::process_row_count

Definition at line 76 of file TextFileBufferParser.h.

Referenced by foreign_storage::parse_file_regions(), foreign_storage::CsvFileBufferParser::parseBuffer(), foreign_storage::RegexFileBufferParser::parseBuffer(), and foreign_storage::populate_chunks().

size_t foreign_storage::ParseBufferRequest::processed_row_count

Definition at line 84 of file TextFileBufferParser.h.

Referenced by foreign_storage::populate_chunks(), and foreign_storage::update_delete_buffer().

const bool foreign_storage::ParseBufferRequest::track_rejected_rows

Definition at line 80 of file TextFileBufferParser.h.

Referenced by foreign_storage::CsvFileBufferParser::parseBuffer(), and foreign_storage::RegexFileBufferParser::parseBuffer().

---

The documentation for this struct was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/TextFileBufferParser.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/TextFileBufferParser.cpp