ArrowCsvForeignStorage Class Reference

Inheritance diagram for ArrowCsvForeignStorage:

Collaboration diagram for ArrowCsvForeignStorage:

Public Member Functions
	ArrowCsvForeignStorage ()
void	prepareTable (const int db_id, const std::string &type, TableDescriptor &td, std::list< ColumnDescriptor > &cols) override
void	registerTable (Catalog_Namespace::Catalog *catalog, std::pair< int, int > table_key, const std::string &type, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols, Data_Namespace::AbstractBufferMgr *mgr) override
std::string	getType () const override
Public Member Functions inherited from ArrowForeignStorageBase
void	append (const std::vector< ForeignStorageColumnBuffer > &column_buffers) override
void	read (const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, int8_t *dest, const size_t numBytes) override
int8_t *	tryZeroCopy (const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, const size_t numBytes) override
void	dropTable (const int db_id, const int table_id) override
void	parseArrowTable (Catalog_Namespace::Catalog *catalog, std::pair< int, int > table_key, const std::string &type, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols, Data_Namespace::AbstractBufferMgr *mgr, const arrow::Table &table)
std::shared_ptr < arrow::ChunkedArray >	createDictionaryEncodedColumn (StringDictionary *dict, const ColumnDescriptor &c, std::shared_ptr< arrow::ChunkedArray > arr_col_chunked_array)
std::shared_ptr < arrow::ChunkedArray >	convertArrowDictionary (StringDictionary *dict, const ColumnDescriptor &c, std::shared_ptr< arrow::ChunkedArray > arr_col_chunked_array)
template<typename T , typename ChunkType >
std::shared_ptr < arrow::ChunkedArray >	createDecimalColumn (const ColumnDescriptor &c, std::shared_ptr< arrow::ChunkedArray > arr_col_chunked_array)
std::shared_ptr < arrow::ChunkedArray >	replaceNullValues (const SQLTypeInfo &columnType, std::shared_ptr< arrow::ChunkedArray > arr_col_chunked_array)
template<typename T >
std::shared_ptr < arrow::ChunkedArray >	replaceNullValuesImpl (std::shared_ptr< arrow::ChunkedArray > arr_col_chunked_array)
void	getSizeAndOffset (const Frag &frag, const std::shared_ptr< arrow::Array > &chunk, size_t i, int &size, int &offset)
int64_t	makeFragment (const Frag &frag, ArrowFragment &arrowFrag, const std::vector< std::shared_ptr< arrow::Array >> &chunks, bool is_varlen)
Public Member Functions inherited from PersistentForeignStorageInterface
virtual	~PersistentForeignStorageInterface ()

Additional Inherited Members
Public Attributes inherited from ArrowForeignStorageBase
std::map< std::array< int, 3 > , std::vector< ArrowFragment > >	m_columns

Detailed Description

Definition at line 897 of file ArrowForeignStorage.cpp.

Constructor & Destructor Documentation

ArrowCsvForeignStorage::ArrowCsvForeignStorage ( )

inline

Definition at line 899 of file ArrowForeignStorage.cpp.

{}

Member Function Documentation

std::string ArrowCsvForeignStorage::getType ( ) const

overridevirtual

Implements PersistentForeignStorageInterface.

Definition at line 1057 of file ArrowForeignStorage.cpp.

References logger::INFO, and LOG.

                                                {
  LOG(INFO) << "CSV backed temporary tables has been activated. Create table `with "
               "(storage_type='CSV:path/to/file.csv');`\n";
  return "CSV";
}

void ArrowCsvForeignStorage::prepareTable ( const int  db_id, const std::string &  type, TableDescriptor &  td, std::list< ColumnDescriptor > &  cols  )

overridevirtual

Reimplemented from PersistentForeignStorageInterface.

Definition at line 915 of file ArrowForeignStorage.cpp.

References TableDescriptor::hasDeletedCol.

                                                                           {
  td.hasDeletedCol = false;
}

void ArrowCsvForeignStorage::registerTable ( Catalog_Namespace::Catalog *  catalog, std::pair< int, int >  table_key, const std::string &  type, const TableDescriptor &  td, const std::list< ColumnDescriptor > &  cols, Data_Namespace::AbstractBufferMgr *  mgr  )

overridevirtual

Implements PersistentForeignStorageInterface.

Definition at line 986 of file ArrowForeignStorage.cpp.

References ARROW_THROW_NOT_OK, CHECK, DataframeTableDescriptor::delimiter, measure< TimeT >::execution(), getArrowImportType(), DataframeTableDescriptor::hasHeader, ArrowForeignStorageBase::parseArrowTable(), DataframeTableDescriptor::skipRows, and VLOG.

                                                                                 {
  const DataframeTableDescriptor* df_td =
      dynamic_cast<const DataframeTableDescriptor*>(&td);
  bool isDataframe = df_td ? true : false;
  std::unique_ptr<DataframeTableDescriptor> df_td_owned;
  if (!isDataframe) {
    df_td_owned = std::make_unique<DataframeTableDescriptor>(td);
    CHECK(df_td_owned);
    df_td = df_td_owned.get();
  }

#if defined(ENABLE_ARROW_4) || defined(_WIN32)
  auto io_context = arrow::io::default_io_context();
#else
  auto io_context = arrow::default_memory_pool();
#endif
  auto arrow_parse_options = arrow::csv::ParseOptions::Defaults();
  arrow_parse_options.quoting = false;
  arrow_parse_options.escaping = false;
  arrow_parse_options.newlines_in_values = false;
  arrow_parse_options.delimiter = *df_td->delimiter.c_str();
  auto arrow_read_options = arrow::csv::ReadOptions::Defaults();
  arrow_read_options.use_threads = true;

  arrow_read_options.block_size = 20 * 1024 * 1024;
  arrow_read_options.autogenerate_column_names = false;
  arrow_read_options.skip_rows =
      df_td->hasHeader ? (df_td->skipRows + 1) : df_td->skipRows;

  auto arrow_convert_options = arrow::csv::ConvertOptions::Defaults();
  arrow_convert_options.check_utf8 = false;
  arrow_convert_options.include_columns = arrow_read_options.column_names;
  arrow_convert_options.strings_can_be_null = true;

  for (auto& c : cols) {
    if (c.isSystemCol) {
      continue;  // must be processed by base interface implementation
    }
    arrow_convert_options.column_types.emplace(c.columnName,
                                               getArrowImportType(c.columnType));
    arrow_read_options.column_names.push_back(c.columnName);
  }

  std::shared_ptr<arrow::io::ReadableFile> inp;
  auto file_result = arrow::io::ReadableFile::Open(info.c_str());
  ARROW_THROW_NOT_OK(file_result.status());
  inp = file_result.ValueOrDie();

  auto table_reader_result = arrow::csv::TableReader::Make(
      io_context, inp, arrow_read_options, arrow_parse_options, arrow_convert_options);
  ARROW_THROW_NOT_OK(table_reader_result.status());
  auto table_reader = table_reader_result.ValueOrDie();

  std::shared_ptr<arrow::Table> arrowTable;
  auto time = measure<>::execution([&]() {
    auto arrow_table_result = table_reader->Read();
    ARROW_THROW_NOT_OK(arrow_table_result.status());
    arrowTable = arrow_table_result.ValueOrDie();
  });

  VLOG(1) << "Read Arrow CSV file " << info << " in " << time << "ms";

  arrow::Table& table = *arrowTable.get();
  parseArrowTable(catalog, table_key, info, td, cols, mgr, table);
}

Here is the call graph for this function:

---

The documentation for this class was generated from the following file:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ArrowForeignStorage.cpp