anonymous_namespace{DBHandler.cpp} Namespace Reference

Classes
struct	ForceDisconnect
struct	ProjectionTokensForCompletion

Functions
bool	dashboard_exists (const Catalog_Namespace::Catalog &cat, const int32_t user_id, const std::string &dashboard_name)
ProjectionTokensForCompletion	extract_projection_tokens_for_completion (const std::string &sql)
std::string	dump_table_col_names (const std::map< std::string, std::vector< std::string >> &table_col_names)
TTableRefreshInfo	get_refresh_info (const TableDescriptor *td)
ScopeGuard	pause_and_resume_executor_queue ()
void	check_table_not_sharded (const TableDescriptor *td)
void	check_valid_column_names (const std::list< const ColumnDescriptor * > &descs, const std::vector< std::string > &column_names)
std::vector< int >	column_ids_by_names (const std::list< const ColumnDescriptor * > &descs, const std::vector< std::string > &column_names)
void	log_cache_size (const Catalog_Namespace::Catalog &cat)
void	log_system_cpu_memory_status (std::string const &query, const Catalog_Namespace::Catalog &cat)
std::string	get_load_tag (const std::string &load_tag, const std::string &table_name)
std::string	get_import_tag (const std::string &import_tag, const std::string &table_name, const std::string &file_path)
size_t	get_column_size (const TColumn &column)
RecordBatchVector	loadArrowStream (const std::string &stream)
void	add_vsi_network_prefix (std::string &path)
void	add_vsi_geo_prefix (std::string &path)
void	add_vsi_archive_prefix (std::string &path)
std::string	remove_vsi_prefixes (const std::string &path_in)
bool	path_is_relative (const std::string &path)
bool	path_has_valid_filename (const std::string &path)
bool	is_a_supported_geo_file (const std::string &path)
bool	is_a_supported_archive_file (const std::string &path)
std::string	find_first_geo_file_in_archive (const std::string &archive_path, const import_export::CopyParams &copy_params)
bool	is_local_file (const std::string &file_path)
void	validate_import_file_path_if_local (const std::string &file_path)
std::unique_ptr < Catalog_Namespace::CustomExpression >	create_custom_expr_from_thrift_obj (const TCustomExpression &t_custom_expr, const Catalog &catalog)
TCustomExpression	create_thrift_obj_from_custom_expr (const CustomExpression &custom_expr, const Catalog &catalog)
bool	TTypeInfo_IsGeo (const TDatumType::type &t)
std::string	TTypeInfo_TypeToString (const TDatumType::type &t)
std::string	get_mismatch_attr_warning_text (const std::string &table_name, const std::string &file_path, const std::string &column_name, const std::string &attr, const std::string &got, const std::string &expected)
bool	check_and_reset_in_memory_system_table (const Catalog &catalog, const TableDescriptor &td)
void	check_in_memory_system_table_query (const std::vector< std::vector< std::string >> &selected_tables)

Function Documentation

void anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix

(

std::string &

path

)

Definition at line 4212 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                             {
  // check for compressed file or file bundle
  if (boost::iends_with(path, ".zip")) {
    // zip archive
    path = "/vsizip/" + path;
  } else if (boost::iends_with(path, ".tar") || boost::iends_with(path, ".tgz") ||
             boost::iends_with(path, ".tar.gz")) {
    // tar archive (compressed or uncompressed)
    path = "/vsitar/" + path;
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix

(

std::string &

path

)

Definition at line 4205 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                         {
  // single gzip'd file (not an archive)?
  if (boost::iends_with(path, ".gz") && !boost::iends_with(path, ".tar.gz")) {
    path = "/vsigzip/" + path;
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix

(

std::string &

path

)

Definition at line 4183 of file DBHandler.cpp.

References Geospatial::GDAL::supportsNetworkFileAccess(), and THROW_DB_EXCEPTION.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                             {
  // do we support network file access?
  bool gdal_network = Geospatial::GDAL::supportsNetworkFileAccess();

  // modify head of filename based on source location
  if (boost::istarts_with(path, "http://") || boost::istarts_with(path, "https://")) {
    if (!gdal_network) {
      THROW_DB_EXCEPTION(
          "HTTP geo file import not supported! Update to GDAL 2.2 or later!");
    }
    // invoke GDAL CURL virtual file reader
    path = "/vsicurl/" + path;
  } else if (boost::istarts_with(path, "s3://")) {
    if (!gdal_network) {
      THROW_DB_EXCEPTION(
          "S3 geo file import not supported! Update to GDAL 2.2 or later!");
    }
    // invoke GDAL S3 virtual file reader
    boost::replace_first(path, "s3://", "/vsis3/");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::check_and_reset_in_memory_system_table	(	const Catalog &	catalog,
		const TableDescriptor &	td
	)

Definition at line 6843 of file DBHandler.cpp.

References g_enable_system_tables, Catalog_Namespace::Catalog::getMetadataForTable(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), lockmgr::TableLockMgrImpl< TableSchemaLockMgr >::getWriteLockForTable(), TableDescriptor::is_in_memory_system_table, Catalog_Namespace::Catalog::removeFragmenterForTable(), TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by check_in_memory_system_table_query(), and DBHandler::checkInMemorySystemTableQuery().

                                                                       {
  if (td.is_in_memory_system_table) {
    if (g_enable_system_tables) {
      // Reset system table fragmenter in order to force chunk metadata refetch.
      auto table_schema_lock =
          lockmgr::TableSchemaLockMgr::getWriteLockForTable(catalog, td.tableName);
      auto table_data_lock =
          lockmgr::TableDataLockMgr::getWriteLockForTable(catalog, td.tableName);
      catalog.removeFragmenterForTable(td.tableId);
      catalog.getMetadataForTable(td.tableId, true);
      return true;
    } else {
      throw std::runtime_error(
          "Query cannot be executed because use of system tables is currently "
          "disabled.");
    }
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::check_in_memory_system_table_query

(

const std::vector< std::vector< std::string >> &

selected_tables

)

Definition at line 6864 of file DBHandler.cpp.

References CHECK, check_and_reset_in_memory_system_table(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and shared::kInfoSchemaDbName.

Referenced by DBHandler::parse_to_ra().

                                                            {
  const auto info_schema_catalog =
      Catalog_Namespace::SysCatalog::instance().getCatalog(shared::kInfoSchemaDbName);
  if (info_schema_catalog) {
    for (const auto& table : selected_tables) {
      if (table[1] == shared::kInfoSchemaDbName) {
        auto td = info_schema_catalog->getMetadataForTable(table[0], false);
        CHECK(td);
        check_and_reset_in_memory_system_table(*info_schema_catalog, *td);
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::check_table_not_sharded

(

const TableDescriptor *

td

)

Definition at line 3099 of file DBHandler.cpp.

References TableDescriptor::nShards.

Referenced by DBHandler::prepare_loader_generic().

                                                        {
  if (td && td->nShards) {
    throw std::runtime_error("Cannot import a sharded table directly to a leaf");
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::check_valid_column_names	(	const std::list< const ColumnDescriptor * > &	descs,
		const std::vector< std::string > &	column_names
	)

Definition at line 3105 of file DBHandler.cpp.

References setup::name, THROW_DB_EXCEPTION, and to_lower().

Referenced by DBHandler::prepare_loader_generic().

                                                                        {
  std::unordered_set<std::string> unique_names;
  for (const auto& name : column_names) {
    auto lower_name = to_lower(name);
    if (unique_names.find(lower_name) != unique_names.end()) {
      THROW_DB_EXCEPTION("Column " + name + " is mentioned multiple times");
    } else {
      unique_names.insert(lower_name);
    }
  }
  for (const auto& cd : descs) {
    auto iter = unique_names.find(to_lower(cd->columnName));
    if (iter != unique_names.end()) {
      unique_names.erase(iter);
    }
  }
  if (!unique_names.empty()) {
    THROW_DB_EXCEPTION("Column " + *unique_names.begin() + " does not exist");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<int> anonymous_namespace{DBHandler.cpp}::column_ids_by_names	(	const std::list< const ColumnDescriptor * > &	descs,
		const std::vector< std::string > &	column_names
	)

Definition at line 3131 of file DBHandler.cpp.

References THROW_DB_EXCEPTION, and to_lower().

Referenced by DBHandler::load_table(), DBHandler::load_table_binary(), DBHandler::load_table_binary_arrow(), and DBHandler::load_table_binary_columnar().

                                                                               {
  std::vector<int> desc_to_column_ids;
  if (column_names.empty()) {
    int col_idx = 0;
    for (const auto& cd : descs) {
      if (!cd->isGeoPhyCol) {
        desc_to_column_ids.push_back(col_idx);
        ++col_idx;
      }
    }
  } else {
    for (const auto& cd : descs) {
      if (!cd->isGeoPhyCol) {
        bool found = false;
        for (size_t j = 0; j < column_names.size(); ++j) {
          if (to_lower(cd->columnName) == to_lower(column_names[j])) {
            found = true;
            desc_to_column_ids.push_back(j);
            break;
          }
        }
        if (!found) {
          if (!cd->columnType.get_notnull()) {
            desc_to_column_ids.push_back(-1);
          } else {
            THROW_DB_EXCEPTION("Column '" + cd->columnName +
                               "' cannot be omitted due to NOT NULL constraint");
          }
        }
      }
    }
  }
  return desc_to_column_ids;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::unique_ptr<Catalog_Namespace::CustomExpression> anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj	(	const TCustomExpression &	t_custom_expr,
		const Catalog &	catalog
	)

Definition at line 4582 of file DBHandler.cpp.

References CHECK, Catalog_Namespace::Catalog::getMetadataForTable(), and THROW_DB_EXCEPTION.

Referenced by DBHandler::create_custom_expression().

                            {
  if (t_custom_expr.data_source_name.empty()) {
    THROW_DB_EXCEPTION("Custom expression data source name cannot be empty.")
  }
  CHECK(t_custom_expr.data_source_type == TDataSourceType::type::TABLE)
      << "Unexpected data source type: "
      << static_cast<int>(t_custom_expr.data_source_type);
  auto td = catalog.getMetadataForTable(t_custom_expr.data_source_name, false);
  if (!td) {
    THROW_DB_EXCEPTION("Custom expression references a table \"" +
                       t_custom_expr.data_source_name + "\" that does not exist.")
  }
  DataSourceType data_source_type = DataSourceType::TABLE;
  return std::make_unique<CustomExpression>(
      t_custom_expr.name, t_custom_expr.expression_json, data_source_type, td->tableId);
}

Here is the call graph for this function:

Here is the caller graph for this function:

TCustomExpression anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr	(	const CustomExpression &	custom_expr,
		const Catalog &	catalog
	)

Definition at line 4601 of file DBHandler.cpp.

References CHECK, Catalog_Namespace::CustomExpression::data_source_id, Catalog_Namespace::CustomExpression::data_source_type, Catalog_Namespace::CustomExpression::expression_json, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::CustomExpression::id, Catalog_Namespace::CustomExpression::is_deleted, LOG, Catalog_Namespace::CustomExpression::name, TableDescriptor::tableName, and logger::WARNING.

Referenced by DBHandler::get_custom_expressions().

                                                                             {
  TCustomExpression t_custom_expr;
  t_custom_expr.id = custom_expr.id;
  t_custom_expr.name = custom_expr.name;
  t_custom_expr.expression_json = custom_expr.expression_json;
  t_custom_expr.data_source_id = custom_expr.data_source_id;
  t_custom_expr.is_deleted = custom_expr.is_deleted;
  CHECK(custom_expr.data_source_type == DataSourceType::TABLE)
      << "Unexpected data source type: "
      << static_cast<int>(custom_expr.data_source_type);
  t_custom_expr.data_source_type = TDataSourceType::type::TABLE;
  auto td = catalog.getMetadataForTable(custom_expr.data_source_id, false);
  if (td) {
    t_custom_expr.data_source_name = td->tableName;
  } else {
    LOG(WARNING)
        << "Custom expression references a deleted data source. Custom expression id: "
        << custom_expr.id << ", name: " << custom_expr.name;
  }
  return t_custom_expr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::dashboard_exists	(	const Catalog_Namespace::Catalog &	cat,
		const int32_t	user_id,
		const std::string &	dashboard_name
	)

Definition at line 152 of file DBHandler.cpp.

References Catalog_Namespace::Catalog::getMetadataForDashboard(), and to_string().

Referenced by DBHandler::create_dashboard().

                                                       {
  return (cat.getMetadataForDashboard(std::to_string(user_id), dashboard_name));
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::dump_table_col_names

(

const std::map< std::string, std::vector< std::string >> &

table_col_names

)

Definition at line 2304 of file DBHandler.cpp.

Referenced by DBHandler::get_result_row_for_pixel().

                                                                    {
  std::ostringstream oss;
  for (const auto& [table_name, col_names] : table_col_names) {
    oss << ":" << table_name;
    for (const auto& col_name : col_names) {
      oss << "," << col_name;
    }
  }
  return oss.str();
}

Here is the caller graph for this function:

ProjectionTokensForCompletion anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion

(

const std::string &

sql

)

Definition at line 1637 of file DBHandler.cpp.

References split(), and to_upper().

Referenced by DBHandler::get_completion_hints().

                          {
  boost::regex id_regex{R"(([[:alnum:]]|_|\.)+)",
                        boost::regex::extended | boost::regex::icase};
  boost::sregex_token_iterator tok_it(sql.begin(), sql.end(), id_regex, 0);
  boost::sregex_token_iterator end;
  std::unordered_set<std::string> uc_column_names;
  std::unordered_set<std::string> uc_column_table_qualifiers;
  for (; tok_it != end; ++tok_it) {
    std::string column_name = *tok_it;
    std::vector<std::string> column_tokens;
    boost::split(column_tokens, column_name, boost::is_any_of("."));
    if (column_tokens.size() == 2) {
      // If the column name is qualified, take user's word.
      uc_column_table_qualifiers.insert(to_upper(column_tokens.front()));
    } else {
      uc_column_names.insert(to_upper(column_name));
    }
  }
  return {uc_column_names, uc_column_table_qualifiers};
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive	(	const std::string &	archive_path,
		const import_export::CopyParams &	copy_params
	)

Definition at line 4291 of file DBHandler.cpp.

References import_export::Importer::gdalGetAllFilesInArchive(), logger::INFO, is_a_supported_geo_file(), LOG, and remove_vsi_prefixes().

Referenced by DBHandler::detect_column_types(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                                                                       {
  // get the recursive list of all files in the archive
  std::vector<std::string> files =
      import_export::Importer::gdalGetAllFilesInArchive(archive_path, copy_params);

  // report the list
  LOG(INFO) << "Found " << files.size() << " files in Archive "
            << remove_vsi_prefixes(archive_path);
  for (const auto& file : files) {
    LOG(INFO) << "  " << file;
  }

  // scan the list for the first candidate file
  bool found_suitable_file = false;
  std::string file_name;
  for (const auto& file : files) {
    if (is_a_supported_geo_file(file)) {
      file_name = file;
      found_suitable_file = true;
      break;
    }
  }

  // if we didn't find anything
  if (!found_suitable_file) {
    LOG(INFO) << "Failed to find any supported geo files in Archive: " +
                     remove_vsi_prefixes(archive_path);
    file_name.clear();
  }

  // done
  return file_name;
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t anonymous_namespace{DBHandler.cpp}::get_column_size

(

const TColumn &

column

)

Definition at line 3454 of file DBHandler.cpp.

Referenced by DBHandler::load_table_binary_columnar().

                                              {
  if (!column.nulls.empty()) {
    return column.nulls.size();
  } else {
    // it is a very bold estimate but later we check it against REAL data
    // and if this function returns a wrong result (e.g. both int and string
    // vectors are filled with values), we get an error
    return column.data.int_col.size() + column.data.arr_col.size() +
           column.data.real_col.size() + column.data.str_col.size();
  }
}

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::get_import_tag	(	const std::string &	import_tag,
		const std::string &	table_name,
		const std::string &	file_path
	)

Definition at line 3315 of file DBHandler.cpp.

Referenced by DBHandler::import_table(), and DBHandler::importGeoTableSingle().

                                                       {
  std::ostringstream oss;
  oss << import_tag << "(" << table_name << ", file_path:" << file_path << ")";
  return oss.str();
}

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::get_load_tag	(	const std::string &	load_tag,
		const std::string &	table_name
	)

Definition at line 3309 of file DBHandler.cpp.

Referenced by DBHandler::load_table(), DBHandler::load_table_binary(), DBHandler::load_table_binary_arrow(), and DBHandler::load_table_binary_columnar().

                                                                               {
  std::ostringstream oss;
  oss << load_tag << "(" << table_name << ")";
  return oss.str();
}

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::get_mismatch_attr_warning_text	(	const std::string &	table_name,
		const std::string &	file_path,
		const std::string &	column_name,
		const std::string &	attr,
		const std::string &	got,
		const std::string &	expected
	)

Definition at line 5385 of file DBHandler.cpp.

Referenced by DBHandler::importGeoTableSingle().

                                                                      {
  return "Issue encountered in geo/raster file '" + file_path +
         "' while appending to table '" + table_name + "'. Column '" + column_name +
         "' " + attr + " mismatch (got '" + got + "', expected '" + expected + "')";
}

Here is the caller graph for this function:

TTableRefreshInfo anonymous_namespace{DBHandler.cpp}::get_refresh_info

(

const TableDescriptor *

td

)

Definition at line 2463 of file DBHandler.cpp.

References QueryRunner::ALL, foreign_storage::ForeignTable::ALL_REFRESH_UPDATE_TYPE, foreign_storage::ForeignTable::APPEND_REFRESH_UPDATE_TYPE, CHECK, shared::convert_temporal_to_iso_format(), TableDescriptor::isForeignTable(), kTIMESTAMP, foreign_storage::ForeignTable::MANUAL_REFRESH_TIMING_TYPE, foreign_storage::ForeignTable::NULL_REFRESH_TIME, foreign_storage::ForeignTable::REFRESH_INTERVAL_KEY, foreign_storage::ForeignTable::REFRESH_START_DATE_TIME_KEY, foreign_storage::ForeignTable::REFRESH_TIMING_TYPE_KEY, foreign_storage::ForeignTable::REFRESH_UPDATE_TYPE_KEY, foreign_storage::ForeignTable::SCHEDULE_REFRESH_TIMING_TYPE, and UNREACHABLE.

Referenced by DBHandler::get_table_details_impl().

                                                              {
  CHECK(td->isForeignTable());
  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
  CHECK(foreign_table);
  TTableRefreshInfo refresh_info;
  const auto& update_type =
      foreign_table->getOption(foreign_storage::ForeignTable::REFRESH_UPDATE_TYPE_KEY);
  CHECK(update_type.has_value());
  if (update_type.value() == foreign_storage::ForeignTable::ALL_REFRESH_UPDATE_TYPE) {
    refresh_info.update_type = TTableRefreshUpdateType::ALL;
  } else if (update_type.value() ==
             foreign_storage::ForeignTable::APPEND_REFRESH_UPDATE_TYPE) {
    refresh_info.update_type = TTableRefreshUpdateType::APPEND;
  } else {
    UNREACHABLE() << "Unexpected refresh update type: " << update_type.value();
  }

  const auto& timing_type =
      foreign_table->getOption(foreign_storage::ForeignTable::REFRESH_TIMING_TYPE_KEY);
  CHECK(timing_type.has_value());
  if (timing_type.value() == foreign_storage::ForeignTable::MANUAL_REFRESH_TIMING_TYPE) {
    refresh_info.timing_type = TTableRefreshTimingType::MANUAL;
    refresh_info.interval_count = -1;
  } else if (timing_type.value() ==
             foreign_storage::ForeignTable::SCHEDULE_REFRESH_TIMING_TYPE) {
    refresh_info.timing_type = TTableRefreshTimingType::SCHEDULED;
    const auto& start_date_time = foreign_table->getOption(
        foreign_storage::ForeignTable::REFRESH_START_DATE_TIME_KEY);
    CHECK(start_date_time.has_value());
    auto start_date_time_epoch = dateTimeParse<kTIMESTAMP>(start_date_time.value(), 0);
    refresh_info.start_date_time =
        shared::convert_temporal_to_iso_format({kTIMESTAMP}, start_date_time_epoch);
    const auto& interval =
        foreign_table->getOption(foreign_storage::ForeignTable::REFRESH_INTERVAL_KEY);
    CHECK(interval.has_value());
    const auto& interval_str = interval.value();
    refresh_info.interval_count =
        std::stoi(interval_str.substr(0, interval_str.length() - 1));
    auto interval_type = std::toupper(interval_str[interval_str.length() - 1]);
    if (interval_type == 'H') {
      refresh_info.interval_type = TTableRefreshIntervalType::HOUR;
    } else if (interval_type == 'D') {
      refresh_info.interval_type = TTableRefreshIntervalType::DAY;
    } else if (interval_type == 'S') {
      // This use case is for development only.
      refresh_info.interval_type = TTableRefreshIntervalType::NONE;
    } else {
      UNREACHABLE() << "Unexpected interval type: " << interval_str;
    }
  } else {
    UNREACHABLE() << "Unexpected refresh timing type: " << timing_type.value();
  }
  if (foreign_table->last_refresh_time !=
      foreign_storage::ForeignTable::NULL_REFRESH_TIME) {
    refresh_info.last_refresh_time = shared::convert_temporal_to_iso_format(
        {kTIMESTAMP}, foreign_table->last_refresh_time);
  }
  if (foreign_table->next_refresh_time !=
      foreign_storage::ForeignTable::NULL_REFRESH_TIME) {
    refresh_info.next_refresh_time = shared::convert_temporal_to_iso_format(
        {kTIMESTAMP}, foreign_table->next_refresh_time);
  }
  return refresh_info;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file

(

const std::string &

path

)

Definition at line 4278 of file DBHandler.cpp.

References path_has_valid_filename().

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                                        {
  if (!path_has_valid_filename(path)) {
    return false;
  }
  if (boost::iends_with(path, ".zip") && !boost::iends_with(path, ".gdb.zip")) {
    return true;
  } else if (boost::iends_with(path, ".tar") || boost::iends_with(path, ".tgz") ||
             boost::iends_with(path, ".tar.gz")) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_a_supported_geo_file

(

const std::string &

path

)

Definition at line 4262 of file DBHandler.cpp.

References path_has_valid_filename().

Referenced by find_first_geo_file_in_archive().

                                                    {
  if (!path_has_valid_filename(path)) {
    return false;
  }
  // this is now just for files that we want to recognize
  // as geo when inside an archive (see below)
  // @TODO(se) make this more flexible?
  if (boost::iends_with(path, ".shp") || boost::iends_with(path, ".geojson") ||
      boost::iends_with(path, ".json") || boost::iends_with(path, ".kml") ||
      boost::iends_with(path, ".kmz") || boost::iends_with(path, ".gdb") ||
      boost::iends_with(path, ".gdb.zip") || boost::iends_with(path, ".fgb")) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_local_file

(

const std::string &

file_path

)

Definition at line 4326 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), and validate_import_file_path_if_local().

                                               {
  return (!boost::istarts_with(file_path, "s3://") &&
          !boost::istarts_with(file_path, "http://") &&
          !boost::istarts_with(file_path, "https://"));
}

Here is the caller graph for this function:

RecordBatchVector anonymous_namespace{DBHandler.cpp}::loadArrowStream

(

const std::string &

stream

)

Definition at line 3576 of file DBHandler.cpp.

References ARROW_ASSIGN_OR_THROW, ARROW_THRIFT_THROW_NOT_OK, logger::ERROR, and LOG.

Referenced by DBHandler::load_table_binary_arrow().

                                                           {
  RecordBatchVector batches;
  try {
    // TODO(wesm): Make this simpler in general, see ARROW-1600
    auto stream_buffer =
        std::make_shared<arrow::Buffer>(reinterpret_cast<const uint8_t*>(stream.c_str()),
                                        static_cast<int64_t>(stream.size()));

    arrow::io::BufferReader buf_reader(stream_buffer);
    std::shared_ptr<arrow::RecordBatchReader> batch_reader;
    ARROW_ASSIGN_OR_THROW(batch_reader,
                          arrow::ipc::RecordBatchStreamReader::Open(&buf_reader));

    while (true) {
      std::shared_ptr<arrow::RecordBatch> batch;
      // Read batch (zero-copy) from the stream
      ARROW_THRIFT_THROW_NOT_OK(batch_reader->ReadNext(&batch));
      if (batch == nullptr) {
        break;
      }
      batches.emplace_back(std::move(batch));
    }
  } catch (const std::exception& e) {
    LOG(ERROR) << "Error parsing Arrow stream: " << e.what() << ". Import aborted";
  }
  return batches;
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::log_cache_size

(

const Catalog_Namespace::Catalog &

cat

)

Definition at line 3167 of file DBHandler.cpp.

References BASELINE_HT, BBOX_INTERSECT_AUTO_TUNER_PARAM, BBOX_INTERSECT_HT, CHUNK_METADATA, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER, BoundingBoxIntersectJoinHashTable::getBoundingBoxIntersectTuningParamCache(), DataRecycler< CACHED_ITEM_TYPE, META_INFO_TYPE >::getCurrentCacheSizeForDevice(), Executor::getExecutor(), BoundingBoxIntersectJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), QueryEngine::getInstance(), Catalog_Namespace::Catalog::getTotalMemorySizeForDictionariesForDatabase(), logger::INFO, LOG, PERFECT_HT, and Executor::UNITARY_EXECUTOR_ID.

Referenced by log_system_cpu_memory_status().

                                                         {
  std::ostringstream oss;
  oss << "Cache size information {";
  auto executor = Executor::getExecutor(Executor::UNITARY_EXECUTOR_ID);
  // 1. Data recycler
  // 1.a Resultset Recycler
  auto resultset_cache_size =
      executor->getResultSetRecyclerHolder()
          .getResultSetRecycler()
          ->getResultSetRecyclerMetricTracker()
          .getCurrentCacheSize(DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  if (resultset_cache_size) {
    oss << "\"query_resultset\": " << *resultset_cache_size << " bytes, ";
  }

  // 1.b Join Hash Table Recycler
  auto perfect_join_ht_cache_size =
      PerfectJoinHashTable::getHashTableCache()->getCurrentCacheSizeForDevice(
          CacheItemType::PERFECT_HT, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  auto baseline_join_ht_cache_size =
      BaselineJoinHashTable::getHashTableCache()->getCurrentCacheSizeForDevice(
          CacheItemType::BASELINE_HT, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  auto bbox_intersect_ht_cache_size =
      BoundingBoxIntersectJoinHashTable::getHashTableCache()
          ->getCurrentCacheSizeForDevice(CacheItemType::BBOX_INTERSECT_HT,
                                         DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  auto bbox_intersect_ht_tuner_cache_size =
      BoundingBoxIntersectJoinHashTable::getBoundingBoxIntersectTuningParamCache()
          ->getCurrentCacheSizeForDevice(CacheItemType::BBOX_INTERSECT_AUTO_TUNER_PARAM,
                                         DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  auto sum_hash_table_cache_size =
      perfect_join_ht_cache_size + baseline_join_ht_cache_size +
      bbox_intersect_ht_cache_size + bbox_intersect_ht_tuner_cache_size;
  oss << "\"hash_tables\": " << sum_hash_table_cache_size << " bytes, ";

  // 1.c Chunk Metadata Recycler
  auto chunk_metadata_cache_size =
      executor->getResultSetRecyclerHolder()
          .getChunkMetadataRecycler()
          ->getCurrentCacheSizeForDevice(CacheItemType::CHUNK_METADATA,
                                         DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
  oss << "\"chunk_metadata\": " << chunk_metadata_cache_size << " bytes, ";

  // 2. Query Plan Dag
  auto query_plan_dag_cache_size =
      executor->getQueryPlanDagCache().getCurrentNodeMapSize();
  oss << "\"query_plan_dag\": " << query_plan_dag_cache_size << " bytes, ";

  // 3. Compiled (GPU) Code
  oss << "\"compiled_GPU code\": "
      << QueryEngine::getInstance()->gpu_code_accessor->getCacheSize() << " bytes, ";

  // 4. String Dictionary
  oss << "\"string_dictionary\": " << cat.getTotalMemorySizeForDictionariesForDatabase()
      << " bytes";
  oss << "}";
  LOG(INFO) << oss.str();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::log_system_cpu_memory_status	(	std::string const &	query,
		const Catalog_Namespace::Catalog &	cat
	)

Definition at line 3226 of file DBHandler.cpp.

References g_allow_memory_status_log, Catalog_Namespace::Catalog::getDataMgr(), Data_Namespace::DataMgr::getSystemMemoryUsage(), logger::INFO, LOG, and log_cache_size().

Referenced by DBHandler::import_table(), DBHandler::importGeoTableSingle(), DBHandler::load_table(), DBHandler::load_table_binary(), DBHandler::load_table_binary_arrow(), DBHandler::load_table_binary_columnar(), and DBHandler::sql_execute_impl().

                                                                       {
  if (g_allow_memory_status_log) {
    std::ostringstream oss;
    oss << query << "\n" << cat.getDataMgr().getSystemMemoryUsage();
    LOG(INFO) << oss.str();
    log_cache_size(cat);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::path_has_valid_filename

(

const std::string &

path

)

Definition at line 4254 of file DBHandler.cpp.

References nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename().

Referenced by is_a_supported_archive_file(), and is_a_supported_geo_file().

                                                    {
  auto filename = boost::filesystem::path(path).filename().string();
  if (filename.size() == 0 || filename[0] == '.' || filename[0] == '/') {
    return false;
  }
  return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::path_is_relative

(

const std::string &

path

)

Definition at line 4246 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::importGeoTableSingle().

                                             {
  if (boost::istarts_with(path, "s3://") || boost::istarts_with(path, "http://") ||
      boost::istarts_with(path, "https://")) {
    return false;
  }
  return !boost::filesystem::path(path).is_absolute();
}

Here is the caller graph for this function:

ScopeGuard anonymous_namespace{DBHandler.cpp}::pause_and_resume_executor_queue

(

)

Definition at line 2873 of file DBHandler.cpp.

References g_enable_executor_resource_mgr, Executor::pause_executor_queue(), and Executor::resume_executor_queue().

Referenced by DBHandler::clear_cpu_memory(), DBHandler::clear_gpu_memory(), and DBHandler::clearRenderMemory().

                                             {
  if (g_enable_executor_resource_mgr) {
    Executor::pause_executor_queue();
    return [] {
      // we need to resume erm queue if we throw any exception
      // that heavydb server can handle w/o shutting it down
      Executor::resume_executor_queue();
    };
  }
  return [] {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::remove_vsi_prefixes

(

const std::string &

path_in

)

Definition at line 4224 of file DBHandler.cpp.

Referenced by find_first_geo_file_in_archive().

                                                        {
  std::string path(path_in);

  // these will be first
  if (boost::istarts_with(path, "/vsizip/")) {
    boost::replace_first(path, "/vsizip/", "");
  } else if (boost::istarts_with(path, "/vsitar/")) {
    boost::replace_first(path, "/vsitar/", "");
  } else if (boost::istarts_with(path, "/vsigzip/")) {
    boost::replace_first(path, "/vsigzip/", "");
  }

  // then these
  if (boost::istarts_with(path, "/vsicurl/")) {
    boost::replace_first(path, "/vsicurl/", "");
  } else if (boost::istarts_with(path, "/vsis3/")) {
    boost::replace_first(path, "/vsis3/", "s3://");
  }

  return path;
}

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::TTypeInfo_IsGeo

(

const TDatumType::type &

t

)

Definition at line 5373 of file DBHandler.cpp.

Referenced by DBHandler::importGeoTableSingle().

                                              {
  return (t == TDatumType::POLYGON || t == TDatumType::MULTIPOLYGON ||
          t == TDatumType::LINESTRING || t == TDatumType::MULTILINESTRING ||
          t == TDatumType::POINT || t == TDatumType::MULTIPOINT);
}

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::TTypeInfo_TypeToString

(

const TDatumType::type &

t

)

Definition at line 5379 of file DBHandler.cpp.

Referenced by DBHandler::importGeoTableSingle().

                                                          {
  std::stringstream ss;
  ss << t;
  return ss.str();
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local

(

const std::string &

file_path

)

Definition at line 4332 of file DBHandler.cpp.

References ddl_utils::IMPORT, is_local_file(), and ddl_utils::validate_allowed_file_path().

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), DBHandler::import_table(), and DBHandler::importGeoTableSingle().

                                                                    {
  if (is_local_file(file_path)) {
    ddl_utils::validate_allowed_file_path(
        file_path, ddl_utils::DataTransferType::IMPORT, true);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function: