import_export::anonymous_namespace{Importer.cpp} Namespace Reference

Functions
SQLTypes	get_type_for_datum (const SQLTypeInfo &ti)
std::tuple< int, SQLTypes, std::string >	explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
int64_t	explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
int64_t	int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
float	float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
double	double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
OGRLayer &	getLayerWithSpecifiedName (const std::string &geo_layer_name, const Geospatial::GDAL::DataSourceUqPtr &poDS, const std::string &file_name)
std::pair< SQLTypes, bool >	ogr_to_type (const OGRFieldType &ogr_type)
SQLTypes	ogr_to_type (const OGRwkbGeometryType &ogr_type)
RasterImporter::PointType	convert_raster_point_type (const import_export::RasterPointType raster_point_type)
RasterImporter::PointTransform	convert_raster_point_transform (const import_export::RasterPointTransform raster_point_transform)

Function Documentation

RasterImporter::PointTransform import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_transform

(

const import_export::RasterPointTransform

raster_point_transform

)

Definition at line 4801 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kFile, import_export::RasterImporter::kFile, import_export::kNone, import_export::RasterImporter::kNone, import_export::kWorld, import_export::RasterImporter::kWorld, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

                                                                    {
  switch (raster_point_transform) {
    case import_export::RasterPointTransform::kNone:
      return RasterImporter::PointTransform::kNone;
    case import_export::RasterPointTransform::kAuto:
      return RasterImporter::PointTransform::kAuto;
    case import_export::RasterPointTransform::kFile:
      return RasterImporter::PointTransform::kFile;
    case import_export::RasterPointTransform::kWorld:
      return RasterImporter::PointTransform::kWorld;
  }
  UNREACHABLE();
  return RasterImporter::PointTransform::kNone;
}

Here is the caller graph for this function:

RasterImporter::PointType import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_type

(

const import_export::RasterPointType

raster_point_type

)

Definition at line 4779 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kDouble, import_export::RasterImporter::kDouble, import_export::kFloat, import_export::RasterImporter::kFloat, import_export::kInt, import_export::RasterImporter::kInt, import_export::kNone, import_export::RasterImporter::kNone, import_export::kPoint, import_export::RasterImporter::kPoint, import_export::kSmallInt, import_export::RasterImporter::kSmallInt, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

                                                          {
  switch (raster_point_type) {
    case import_export::RasterPointType::kNone:
      return RasterImporter::PointType::kNone;
    case import_export::RasterPointType::kAuto:
      return RasterImporter::PointType::kAuto;
    case import_export::RasterPointType::kSmallInt:
      return RasterImporter::PointType::kSmallInt;
    case import_export::RasterPointType::kInt:
      return RasterImporter::PointType::kInt;
    case import_export::RasterPointType::kFloat:
      return RasterImporter::PointType::kFloat;
    case import_export::RasterPointType::kDouble:
      return RasterImporter::PointType::kDouble;
    case import_export::RasterPointType::kPoint:
      return RasterImporter::PointType::kPoint;
  }
  UNREACHABLE();
  return RasterImporter::PointType::kNone;
}

Here is the caller graph for this function:

double import_export::anonymous_namespace{Importer.cpp}::double_value_at	(	const TypedImportBuffer &	import_buffer,
		const size_t	index
	)

Definition at line 2730 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by import_export::Loader::fillShardRow().

                                                                                   {
  const auto& ti = import_buffer.getTypeInfo();
  CHECK_EQ(kDOUBLE, ti.get_type());
  const auto values_buffer = import_buffer.getAsBytes();
  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple<int, SQLTypes, std::string> import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1

(

const std::list< const ColumnDescriptor * > &

col_descs

)

Definition at line 1862 of file Importer.cpp.

References kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

                                                       {
  // validate the columns
  // for now we can only explode into a single destination column
  // which must be of the child type (POLYGON, LINESTRING, POINT)
  int collection_col_idx = -1;
  int col_idx = 0;
  std::string collection_col_name;
  SQLTypes collection_child_type = kNULLT;
  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
    auto const& cd = *cd_it;
    auto const col_type = cd->columnType.get_type();
    if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
      if (collection_col_idx >= 0) {
        throw std::runtime_error(
            "Explode Collections: Found more than one destination column");
      }
      collection_col_idx = col_idx;
      collection_child_type = col_type;
      collection_col_name = cd->columnName;
    }
    for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
      ++cd_it;
    }
    col_idx++;
  }
  if (collection_col_idx < 0) {
    throw std::runtime_error(
        "Explode Collections: Failed to find a supported column type to explode "
        "into");
  }
  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
}

Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2	(	OGRGeometry *	ogr_geometry,
		SQLTypes	collection_child_type,
		const std::string &	collection_col_name,
		size_t	row_or_feature_idx,
		std::function< void(OGRGeometry *)>	execute_import_lambda
	)

Definition at line 1896 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

                                                           {
  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
  bool is_collection = false;
  switch (collection_child_type) {
    case kPOINT:
      switch (ogr_geometry_type) {
        case wkbMultiPoint:
          is_collection = true;
          break;
        case wkbPoint:
          break;
        default:
          throw std::runtime_error(
              "Explode Collections: Source geo type must be MULTIPOINT or POINT");
      }
      break;
    case kLINESTRING:
      switch (ogr_geometry_type) {
        case wkbMultiLineString:
          is_collection = true;
          break;
        case wkbLineString:
          break;
        default:
          throw std::runtime_error(
              "Explode Collections: Source geo type must be MULTILINESTRING or "
              "LINESTRING");
      }
      break;
    case kPOLYGON:
      switch (ogr_geometry_type) {
        case wkbMultiPolygon:
          is_collection = true;
          break;
        case wkbPolygon:
          break;
        default:
          throw std::runtime_error(
              "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
      }
      break;
    default:
      CHECK(false) << "Unsupported geo child type " << collection_child_type;
  }

  int64_t us = 0LL;

  // explode or just import
  if (is_collection) {
    // cast to collection
    OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
    CHECK(collection_geometry);

#if LOG_EXPLODE_COLLECTIONS
    // log number of children
    LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
              << explode_col_name << "' into " << collection_geometry->getNumGeometries()
              << " child rows";
#endif

    // loop over children
    uint32_t child_geometry_count = 0;
    auto child_geometry_it = collection_geometry->begin();
    while (child_geometry_it != collection_geometry->end()) {
      // get and import this child
      OGRGeometry* import_geometry = *child_geometry_it;
      us += measure<std::chrono::microseconds>::execution(
          [&] { execute_import_lambda(import_geometry); });

      // next child
      child_geometry_it++;
      child_geometry_count++;
    }
  } else {
    // import non-collection row just once
    us = measure<std::chrono::microseconds>::execution(
        [&] { execute_import_lambda(ogr_geometry); });
  }

  // done
  return us;
}

Here is the call graph for this function:

Here is the caller graph for this function:

float import_export::anonymous_namespace{Importer.cpp}::float_value_at	(	const TypedImportBuffer &	import_buffer,
		const size_t	index
	)

Definition at line 2723 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by import_export::Loader::fillShardRow().

                                                                                 {
  const auto& ti = import_buffer.getTypeInfo();
  CHECK_EQ(kFLOAT, ti.get_type());
  const auto values_buffer = import_buffer.getAsBytes();
  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::get_type_for_datum ( const SQLTypeInfo &  ti )

inline

Definition at line 260 of file Importer.cpp.

References decimal_to_int_type(), SQLTypeInfo::get_type(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_dict_encoded_string(), string_dict_to_int_type(), and run_benchmark_import::type.

Referenced by import_export::NullArrayDatum(), and NullDatum().

                                                          {
  SQLTypes type;
  if (ti.is_decimal()) {
    type = decimal_to_int_type(ti);
  } else if (ti.is_dict_encoded_string()) {
    type = string_dict_to_int_type(ti);
  } else {
    type = ti.get_type();
  }
  return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

OGRLayer& import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName	(	const std::string &	geo_layer_name,
		const Geospatial::GDAL::DataSourceUqPtr &	poDS,
		const std::string &	file_name
	)

Definition at line 4602 of file Importer.cpp.

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo(), import_export::Importer::importGDALGeo(), and import_export::Importer::readMetadataSampleGDAL().

                                                                {
  // get layer with specified name, or default to first layer
  OGRLayer* poLayer = nullptr;
  if (geo_layer_name.size()) {
    poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
    if (poLayer == nullptr) {
      throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
                               file_name);
    }
  } else {
    poLayer = poDS->GetLayer(0);
    if (poLayer == nullptr) {
      throw std::runtime_error("No layers found in " + file_name);
    }
  }
  return *poLayer;
}

Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at	(	const TypedImportBuffer &	import_buffer,
		const size_t	index
	)

Definition at line 2692 of file Importer.cpp.

References CHECK, CHECK_EQ, logger::FATAL, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getStringDictBuffer(), import_export::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by import_export::Loader::distributeToShardsExistingColumns(), and import_export::Loader::fillShardRow().

                                                                                 {
  const auto& ti = import_buffer.getTypeInfo();
  const int8_t* values_buffer{nullptr};
  if (ti.is_string()) {
    CHECK_EQ(kENCODING_DICT, ti.get_compression());
    values_buffer = import_buffer.getStringDictBuffer();
  } else {
    values_buffer = import_buffer.getAsBytes();
  }
  CHECK(values_buffer);
  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
  switch (logical_size) {
    case 1: {
      return values_buffer[index];
    }
    case 2: {
      return reinterpret_cast<const int16_t*>(values_buffer)[index];
    }
    case 4: {
      return reinterpret_cast<const int32_t*>(values_buffer)[index];
    }
    case 8: {
      return reinterpret_cast<const int64_t*>(values_buffer)[index];
    }
    default:
      LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
  }
  UNREACHABLE();
  return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::anonymous_namespace{Importer.cpp}::ogr_to_type

(

const OGRFieldType &

ogr_type

)

Definition at line 4721 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo().

                                                                  {
  switch (ogr_type) {
    case OFTInteger:
      return std::make_pair(kINT, false);
    case OFTIntegerList:
      return std::make_pair(kINT, true);
#if GDAL_VERSION_MAJOR > 1
    case OFTInteger64:
      return std::make_pair(kBIGINT, false);
    case OFTInteger64List:
      return std::make_pair(kBIGINT, true);
#endif
    case OFTReal:
      return std::make_pair(kDOUBLE, false);
    case OFTRealList:
      return std::make_pair(kDOUBLE, true);
    case OFTString:
      return std::make_pair(kTEXT, false);
    case OFTStringList:
      return std::make_pair(kTEXT, true);
    case OFTDate:
      return std::make_pair(kDATE, false);
    case OFTTime:
      return std::make_pair(kTIME, false);
    case OFTDateTime:
      return std::make_pair(kTIMESTAMP, false);
    case OFTBinary:
      // Interpret binary blobs as byte arrays here
      // but actual import will store NULL as GDAL will not
      // extract the blob (OGRFeature::GetFieldAsString will
      // result in the import buffers having an empty string)
      return std::make_pair(kTINYINT, true);
    default:
      break;
  }
  throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::ogr_to_type

(

const OGRwkbGeometryType &

ogr_type

)

Definition at line 4759 of file Importer.cpp.

References kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

                                                         {
  switch (ogr_type) {
    case wkbPoint:
      return kPOINT;
    case wkbMultiPoint:
      return kMULTIPOINT;
    case wkbLineString:
      return kLINESTRING;
    case wkbMultiLineString:
      return kMULTILINESTRING;
    case wkbPolygon:
      return kPOLYGON;
    case wkbMultiPolygon:
      return kMULTIPOLYGON;
    default:
      break;
  }
  throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
}

Here is the call graph for this function: