import_export::anonymous_namespace{ForeignDataImporter.cpp} Namespace Reference

Functions
int32_t	get_proxy_foreign_table_fragment_size (const size_t maximum_num_fragments_buffered, const size_t max_import_batch_row_count, const Catalog_Namespace::SessionInfo &parent_session_info, const int32_t table_id)

Function Documentation

int32_t import_export::anonymous_namespace{ForeignDataImporter.cpp}::get_proxy_foreign_table_fragment_size	(	const size_t	maximum_num_fragments_buffered,
		const size_t	max_import_batch_row_count,
		const Catalog_Namespace::SessionInfo &	parent_session_info,
		const int32_t	table_id
	)

Definition at line 469 of file ForeignDataImporter.cpp.

References DEFAULT_FRAGMENT_ROWS, Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::SessionInfo::getCatalog(), import_export::ForeignDataImporter::proxy_foreign_table_fragment_size_, and run_benchmark_import::type.

Referenced by import_export::ForeignDataImporter::importGeneral().

                            {
  if (ForeignDataImporter::proxy_foreign_table_fragment_size_ != 0) {
    return ForeignDataImporter::proxy_foreign_table_fragment_size_;
  }

  if (max_import_batch_row_count != 0) {
    return max_import_batch_row_count;
  }

  // This number is chosen as a reasonable default value to reserve for
  // intermediate buffering during import, it is about 2GB of memory. Note,
  // depending on the acutal size of var len values, this heuristic target may
  // be off. NOTE: `maximum_num_fragments_buffered` scales the allowed buffer
  // size with the assumption that in the worst case all buffers may be
  // buffered at once.
  const size_t max_buffer_byte_size =
      2 * 1024UL * 1024UL * 1024UL / maximum_num_fragments_buffered;

  auto& catalog = parent_session_info.getCatalog();

  auto logical_columns =
      catalog.getAllColumnMetadataForTable(table_id, false, false, false);

  size_t row_byte_size = 0;
  for (const auto& column_descriptor : logical_columns) {
    auto type = column_descriptor->columnType;
    size_t field_byte_length = 0;
    if (type.is_varlen_indeed()) {
      // use a heuristic where varlen types are assumed to be 256 bytes in length
      field_byte_length = 256;
    } else {
      field_byte_length = type.get_size();
    }
    row_byte_size += field_byte_length;
  }

  return std::min<size_t>((max_buffer_byte_size + row_byte_size - 1) / row_byte_size,
                          DEFAULT_FRAGMENT_ROWS);
}

Here is the call graph for this function:

Here is the caller graph for this function: