QueryRunner::ImportDriver Class Reference

#include <QueryRunner.h>

Inheritance diagram for QueryRunner::ImportDriver:

Collaboration diagram for QueryRunner::ImportDriver:

Public Member Functions
	ImportDriver (std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU, const std::string session_id="")
void	importGeoTable (const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
Public Member Functions inherited from QueryRunner::QueryRunner
std::shared_ptr < Catalog_Namespace::SessionInfo >	getSession () const
void	addSessionId (const std::string &session_id, ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
void	clearSessionId ()
std::shared_ptr < Catalog_Namespace::Catalog >	getCatalog () const
std::shared_ptr< Calcite >	getCalcite () const
std::shared_ptr< Executor >	getExecutor () const
Catalog_Namespace::UserMetadata &	getUserMetadata () const
bool	gpusPresent () const
virtual void	clearGpuMemory () const
virtual void	clearCpuMemory () const
std::vector< MemoryInfo >	getMemoryInfo (const Data_Namespace::MemoryLevel memory_level) const
BufferPoolStats	getBufferPoolStats (const Data_Namespace::MemoryLevel memory_level, const bool current_db_only) const
virtual std::unique_ptr < Parser::Stmt >	createStatement (const std::string &)
virtual void	runDDLStatement (const std::string &)
virtual void	validateDDLStatement (const std::string &)
virtual std::shared_ptr < ResultSet >	runSQL (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
virtual std::shared_ptr < ExecutionResult >	runSelectQuery (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
virtual std::shared_ptr < ResultSet >	runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
virtual std::shared_ptr < ExecutionResult >	runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
virtual std::shared_ptr < ResultSet >	runSQLWithAllowingInterrupt (const std::string &query_str, const std::string &session_id, const ExecutorDeviceType device_type, const double running_query_check_freq=0.9, const unsigned pending_query_check_freq=1000)
virtual std::vector < std::shared_ptr< ResultSet > >	runMultipleStatements (const std::string &, const ExecutorDeviceType)
virtual void	runImport (Parser::CopyTableStmt *import_stmt)
virtual std::unique_ptr < import_export::Loader >	getLoader (const TableDescriptor *td) const
RegisteredQueryHint	getParsedQueryHint (const std::string &)
std::optional < std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > >	getParsedQueryHints (const std::string &query_str)
std::shared_ptr< const RelAlgNode >	getRootNodeFromParsedQuery (const std::string &query_str)
std::optional < RegisteredQueryHint >	getParsedGlobalQueryHints (const std::string &query_str)
RaExecutionSequence	getRaExecutionSequence (const std::string &query_str)
virtual std::shared_ptr < ResultSet >	getCalcitePlan (const std::string &query_str, bool enable_watchdog, bool is_explain_as_json_str, bool is_explain_detailed) const
std::tuple< QueryPlanHash, std::shared_ptr< HashTable > , std::optional < HashtableCacheMetaInfo > >	getCachedHashtableWithoutCacheKey (std::set< size_t > &visited, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
std::shared_ptr< CacheItemMetric >	getCacheItemMetric (QueryPlanHash cache_key, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
size_t	getNumberOfCachedItem (CacheItemStatus item_status, CacheItemType hash_table_type, bool with_bbox_intersect_tuning_param=false) const
void	resizeDispatchQueue (const size_t num_executors)
QueryPlanDagInfo	getQueryInfoForDataRecyclerTest (const std::string &)
std::shared_ptr< RelAlgTranslator >	getRelAlgTranslator (const std::string &, Executor *)
ExtractedQueryPlanDag	extractQueryPlanDag (const std::string &)
std::unique_ptr< RelAlgDag >	getRelAlgDag (const std::string &)
	QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
virtual	~QueryRunner ()=default
void	setExplainType (const ExecutorExplainType explain_type)

Additional Inherited Members
Static Public Member Functions inherited from QueryRunner::QueryRunner
static QueryRunner *	init (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
static QueryRunner *	init (const File_Namespace::DiskCacheConfig *disk_cache_config, const char *db_path, const std::vector< LeafHostInfo > &string_servers={}, const std::vector< LeafHostInfo > &leaf_servers={})
static QueryRunner *	init (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
static QueryRunner *	init (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false, const File_Namespace::DiskCacheConfig *config=nullptr)
static QueryRunner *	init (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
static QueryRunner *	get ()
static void	reset ()
static ExecutionOptions	defaultExecutionOptionsForRunSQL (bool allow_loop_joins=true, bool just_explain=false)
template<typename... Ts>
static std::shared_ptr < query_state::QueryState >	create_query_state (Ts &&...args)
Static Public Attributes inherited from QueryRunner::QueryRunner
static query_state::QueryStates	query_states_
Protected Member Functions inherited from QueryRunner::QueryRunner
	QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db, const File_Namespace::DiskCacheConfig *disk_cache_config=nullptr)
Protected Attributes inherited from QueryRunner::QueryRunner
ExecutorExplainType	explain_type_ = ExecutorExplainType::Default
Catalog_Namespace::DBMetadata	db_metadata_
std::shared_ptr < Catalog_Namespace::SessionInfo >	session_info_
std::unique_ptr < QueryDispatchQueue >	dispatch_queue_
std::shared_ptr< QueryEngine >	query_engine_
Static Protected Attributes inherited from QueryRunner::QueryRunner
static std::unique_ptr < QueryRunner >	qr_instance_ = nullptr

Detailed Description

Definition at line 336 of file QueryRunner.h.

Constructor & Destructor Documentation

QueryRunner::ImportDriver::ImportDriver	(	std::shared_ptr< Catalog_Namespace::Catalog >	cat,
		const Catalog_Namespace::UserMetadata &	user,
		const ExecutorDeviceType	dt = ExecutorDeviceType::GPU,
		const std::string	session_id = ""
	)

Definition at line 1224 of file QueryRunner.cpp.

    : QueryRunner(
          std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, session_id)) {}

Member Function Documentation

void QueryRunner::ImportDriver::importGeoTable	(	const std::string &	file_path,
		const std::string &	table_name,
		const bool	compression,
		const bool	create_table,
		const bool	explode_collections
	)

Definition at line 1231 of file QueryRunner.cpp.

References cat(), CHECK, measure< TimeT >::execution(), import_export::Importer::importGDAL(), logger::INFO, join(), shared::kDataDirectoryName, kDECIMAL, kENCODING_GEOINT, kENCODING_NONE, Geospatial::kGeoColumnName, import_export::kGeoFile, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, LOG, QueryRunner::QueryRunner::runDDLStatement(), ImportHelpers::sanitize_name(), QueryRunner::QueryRunner::session_info_, and import_export::CopyParams::source_type.

                                                                  {
  using namespace import_export;

  static constexpr bool kIsGeoRaster{false};

  CHECK(session_info_);

  CopyParams copy_params;
  copy_params.source_type = import_export::SourceType::kGeoFile;
  if (compression) {
    copy_params.geo_coords_encoding = EncodingType::kENCODING_GEOINT;
    copy_params.geo_coords_comp_param = 32;
  } else {
    copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
    copy_params.geo_coords_comp_param = 0;
  }
  copy_params.geo_explode_collections = explode_collections;

  std::map<std::string, std::string> colname_to_src;
  auto& cat = session_info_->getCatalog();
  auto cds = Importer::gdalToColumnDescriptors(
      file_path, kIsGeoRaster, Geospatial::kGeoColumnName, copy_params);

  for (auto& cd : cds) {
    const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
    const auto ret =
        colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
    CHECK(ret.second);
    cd.columnName = col_name_sanitized;
  }

  if (create_table) {
    const auto td = cat.getMetadataForTable(table_name);
    if (td != nullptr) {
      throw std::runtime_error(
          "Error: Table " + table_name +
          " already exists. Possible failure to correctly re-create " +
          shared::kDataDirectoryName + " directory.");
    }
    if (table_name != ImportHelpers::sanitize_name(table_name)) {
      throw std::runtime_error("Invalid characters in table name: " + table_name);
    }

    std::string stmt{"CREATE TABLE " + table_name};
    std::vector<std::string> col_stmts;

    for (auto& cd : cds) {
      if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
          cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
        throw std::runtime_error(
            "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
            cd.columnName + " (table: " + table_name + ")");
      }

      if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
        if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
          cd.columnType.set_precision(14);
          cd.columnType.set_scale(7);
        }
      }

      std::string col_stmt;
      col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");

      if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
        col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
      } else {
        if (cd.columnType.is_string()) {
          col_stmt.append("ENCODING NONE");
        } else if (cd.columnType.is_geometry()) {
          if (cd.columnType.get_output_srid() == 4326) {
            col_stmt.append("ENCODING NONE");
          }
        }
      }
      col_stmts.push_back(col_stmt);
    }

    stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
    runDDLStatement(stmt);

    LOG(INFO) << "Created table: " << table_name;
  } else {
    LOG(INFO) << "Not creating table: " << table_name;
  }

  const auto td = cat.getMetadataForTable(table_name);
  if (td == nullptr) {
    throw std::runtime_error("Error: Failed to create table " + table_name);
  }

  import_export::Importer importer(cat, td, file_path, copy_params);
  auto ms = measure<>::execution(
      [&]() { importer.importGDAL(colname_to_src, session_info_.get(), kIsGeoRaster); });
  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
}

Here is the call graph for this function:

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/QueryRunner/QueryRunner.h
• /home/jenkins-slave/workspace/core-os-doxygen/QueryRunner/QueryRunner.cpp