import_export::RasterImporter Class Reference

#include <RasterImporter.h>

Classes
struct	ImportBandInfo

Public Types
enum	PointType {   PointType::kNone, PointType::kAuto, PointType::kSmallInt, PointType::kInt,   PointType::kFloat, PointType::kDouble, PointType::kPoint }
enum	PointTransform { PointTransform::kNone, PointTransform::kAuto, PointTransform::kFile, PointTransform::kWorld }
using	NamesAndSQLTypes = std::vector< std::pair< std::string, SQLTypes >>
using	RawPixels = std::vector< std::byte >
using	NullValue = std::pair< double, bool >
using	Coords = std::vector< std::tuple< double, double, float >>

Public Member Functions
	RasterImporter ()=default
	~RasterImporter ()=default
void	detect (const std::string &file_name, const std::string &specified_band_names, const std::string &specified_band_dimensions, const PointType point_type, const PointTransform point_transform, const bool point_compute_angle, const bool throw_on_error, const MetadataColumnInfos &metadata_column_infos)
void	import (size_t &max_threads, const bool max_threads_using_default)
const uint32_t	getNumBands () const
const PointTransform	getPointTransform () const
const NamesAndSQLTypes	getPointNamesAndSQLTypes () const
const NamesAndSQLTypes	getBandNamesAndSQLTypes () const
const int	getBandsWidth () const
const int	getBandsHeight () const
const NullValue	getBandNullValue (const int band_idx) const
const Coords	getProjectedPixelCoords (const uint32_t thread_idx, const int y) const
void	getRawPixels (const uint32_t thread_idx, const uint32_t band_idx, const int y_start, const int num_rows, const SQLTypes column_sql_type, RawPixels &raw_pixel_bytes)

Private Member Functions
void	getRawBandNamesForFormat (const Geospatial::GDAL::DataSourceUqPtr &datasource)
void	initializeFiltering (const std::string &specified_band_names, const std::string &specified_band_dimensions, const MetadataColumnInfos &metadata_column_infos)
bool	shouldImportBandWithName (const std::string &name)
bool	shouldImportBandWithDimensions (const int width, const int height)
std::string	getBandName (const uint32_t datasource_idx, const int band_idx)
void	checkSpecifiedBandNamesFound () const

Private Attributes
std::vector< std::string >	datasource_names_
std::vector< std::vector < std::string > >	raw_band_names_
std::map< std::string, bool >	specified_band_names_map_
std::map< std::string, int >	column_name_repeats_map_
std::vector< ImportBandInfo >	import_band_infos_
std::array< double, 6 >	affine_transform_matrix_
std::vector< std::vector < Geospatial::GDAL::DataSourceUqPtr > >	datasource_handles_
int	specified_band_width_ {-1}
int	specified_band_height_ {-1}
int	bands_width_ {-1}
int	bands_height_ {-1}
PointType	point_type_ {PointType::kNone}
PointTransform	point_transform_ {PointTransform::kNone}
std::vector < Geospatial::GDAL::CoordinateTransformationUqPtr >	coordinate_transformations_
std::vector< std::unique_ptr < GCPTransformer > >	gcp_transformers_
bool	point_compute_angle_ {false}

Detailed Description

Definition at line 55 of file RasterImporter.h.

Member Typedef Documentation

using import_export::RasterImporter::Coords = std::vector<std::tuple<double, double, float>>

Definition at line 77 of file RasterImporter.h.

using import_export::RasterImporter::NamesAndSQLTypes = std::vector<std::pair<std::string, SQLTypes>>

Definition at line 74 of file RasterImporter.h.

using import_export::RasterImporter::NullValue = std::pair<double, bool>

Definition at line 76 of file RasterImporter.h.

using import_export::RasterImporter::RawPixels = std::vector<std::byte>

Definition at line 75 of file RasterImporter.h.

Member Enumeration Documentation

enum import_export::RasterImporter::PointTransform

strong

Enumerator
kNone
kAuto
kFile
kWorld

Definition at line 61 of file RasterImporter.h.

{ kNone, kAuto, kFile, kWorld };

enum import_export::RasterImporter::PointType

strong

Enumerator
kNone
kAuto
kSmallInt
kInt
kFloat
kDouble
kPoint

Definition at line 60 of file RasterImporter.h.

{ kNone, kAuto, kSmallInt, kInt, kFloat, kDouble, kPoint };

Constructor & Destructor Documentation

import_export::RasterImporter::RasterImporter ( )

default

import_export::RasterImporter::~RasterImporter ( )

default

Member Function Documentation

void import_export::RasterImporter::checkSpecifiedBandNamesFound ( ) const

private

Definition at line 1051 of file RasterImporter.cpp.

References specified_band_names_map_.

Referenced by detect().

                                                        {
  for (auto const& itr : specified_band_names_map_) {
    if (!itr.second) {
      throw std::runtime_error("Specified import band name '" + itr.first +
                               "' was not found in the input raster file");
    }
  }
}

Here is the caller graph for this function:

void import_export::RasterImporter::detect	(	const std::string &	file_name,
		const std::string &	specified_band_names,
		const std::string &	specified_band_dimensions,
		const PointType	point_type,
		const PointTransform	point_transform,
		const bool	point_compute_angle,
		const bool	throw_on_error,
		const MetadataColumnInfos &	metadata_column_infos
	)

Definition at line 276 of file RasterImporter.cpp.

References bands_height_, bands_width_, CHECK, checkSpecifiedBandNamesFound(), datasource_names_, DEBUG_RASTER_IMPORT, logger::ERROR, import_export::anonymous_namespace{RasterImporter.cpp}::gdal_data_type_to_sql_type(), getBandName(), getRawBandNamesForFormat(), import_band_infos_, logger::INFO, initializeFiltering(), kAuto, kDouble, kInt, kNone, kPoint, import_export::kRasterFile, kSmallInt, kWorld, LOG, LOG_IF, Geospatial::GDAL::logMetadata(), Geospatial::GDAL::openDataSource(), point_compute_angle_, point_transform_, point_type_, shouldImportBandWithDimensions(), shouldImportBandWithName(), specified_band_names_map_, to_string(), Geospatial::GDAL::unpackMetadata(), and logger::WARNING.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

                                                                              {
  // open base file to check for subdatasources
  bool has_spatial_reference{false};
  {
    // prepare to open
    // open the file
    Geospatial::GDAL::DataSourceUqPtr datasource = Geospatial::GDAL::openDataSource(
        file_name, import_export::SourceType::kRasterFile);
    if (datasource == nullptr) {
      throw std::runtime_error("Raster Importer: Unable to open raster file " +
                               file_name);
    }

#if DEBUG_RASTER_IMPORT
    // log all its metadata
    Geospatial::GDAL::logMetadata(datasource);
#endif

    // get and add subdatasource datasource names
    auto const subdatasources =
        Geospatial::GDAL::unpackMetadata(datasource->GetMetadata("SUBDATASETS"));
    for (auto const& subdatasource : subdatasources) {
      auto const name_equals = subdatasource.find("_NAME=");
      if (name_equals != std::string::npos) {
        auto subdatasource_name =
            subdatasource.substr(name_equals + 6, std::string::npos);
        LOG_IF(INFO, DEBUG_RASTER_IMPORT)
            << "DEBUG: Found subdatasource '" << subdatasource_name << "'";
        datasource_names_.push_back(subdatasource_name);
      }
    }

    // note if it has a spatial reference (of either type)
    if (datasource->GetSpatialRef()) {
      has_spatial_reference = true;
      LOG_IF(INFO, DEBUG_RASTER_IMPORT) << "DEBUG: Found regular Spatial Reference";
    } else if (datasource->GetGCPSpatialRef()) {
      auto const num_points = datasource->GetGCPCount();
      LOG_IF(INFO, DEBUG_RASTER_IMPORT)
          << "DEBUG: Found GCP Spatial Reference with " << num_points << " GCPs";
      has_spatial_reference = (num_points > 0);
    }

    // fetch
    getRawBandNamesForFormat(datasource);
  }

  // if we didn't find any subdatasources, just use the base file
  if (datasource_names_.size() == 0) {
    LOG_IF(INFO, DEBUG_RASTER_IMPORT)
        << "DEBUG: No sub-datasources found, just using '" << file_name << "'";
    datasource_names_.push_back(file_name);
  }

  // auto point transform
  if (point_transform == PointTransform::kAuto) {
    point_transform_ =
        has_spatial_reference ? PointTransform::kWorld : PointTransform::kNone;
  } else {
    point_transform_ = point_transform;
  }

  // auto point type
  bool optimize_to_smallint = false;
  if (point_type == PointType::kAuto) {
    if (point_transform_ == PointTransform::kNone) {
      point_type_ = PointType::kInt;
      optimize_to_smallint = true;
    } else {
      point_type_ = PointType::kDouble;
    }
  } else {
    point_type_ = point_type;
  }

  std::set<std::pair<int, int>> found_dimensions;

  // lambda to process a datasource
  auto process_datasource = [&](const Geospatial::GDAL::DataSourceUqPtr& datasource,
                                const uint32_t datasource_idx) {
    auto raster_count = datasource->GetRasterCount();
    if (raster_count == 0) {
      throw std::runtime_error("Raster Importer: Raster file " + file_name +
                               " has no rasters");
    }

    // for each band (1-based index)
    for (int i = 1; i <= raster_count; i++) {
      // get band name
      auto band_name = getBandName(datasource_idx, i);

      // if there are specified band names, and this isn't one of them, skip
      if (!shouldImportBandWithName(band_name)) {
        continue;
      }

      // get the band
      auto band = datasource->GetRasterBand(i);
      CHECK(band);

      // get band dimensions
      auto const band_width = band->GetXSize();
      auto const band_height = band->GetYSize();
      int block_size_x, block_size_y;
      band->GetBlockSize(&block_size_x, &block_size_y);

      // report
      LOG(INFO) << "Raster Importer: Found Band '" << band_name << "', with dimensions "
                << band_width << "x" << band_height;

      // if there are specified band dimensions, and this band doesn't match, skip
      if (!shouldImportBandWithDimensions(band_width, band_height)) {
        continue;
      }

      // add to found dimensions
      found_dimensions.insert({band_width, band_height});

      // get SQL type
      auto sql_type = gdal_data_type_to_sql_type(band->GetRasterDataType());

      // get null value and validity
      int null_value_valid{0};
      auto null_value = band->GetNoDataValue(&null_value_valid);

      // store info
      if (specified_band_names_map_.size()) {
        // find and finalize existing info for this band
        auto itr =
            std::find_if(import_band_infos_.begin(),
                         import_band_infos_.end(),
                         [&](ImportBandInfo& info) { return info.name == band_name; });
        CHECK(itr != import_band_infos_.end());
        itr->datasource_idx = datasource_idx;
        itr->band_idx = i;
        itr->sql_type = sql_type;
        itr->null_value = null_value;
        itr->null_value_valid = (null_value_valid != 0);
      } else {
        // import all bands
        import_band_infos_.push_back({band_name,
                                      band_name,
                                      sql_type,
                                      datasource_idx,
                                      i,
                                      null_value,
                                      null_value_valid != 0});
      }
    }
  };

  // initialize filtering
  initializeFiltering(
      specified_band_names, specified_band_dimensions, metadata_column_infos);

  // process datasources
  uint32_t datasource_idx{0u};
  std::vector<std::string> valid_datasource_names;
  for (auto const& datasource_name : datasource_names_) {
    // open it
    Geospatial::GDAL::DataSourceUqPtr datasource_handle =
        Geospatial::GDAL::openDataSource(datasource_name,
                                         import_export::SourceType::kRasterFile);

    // did it open?
    if (datasource_handle == nullptr) {
      continue;
    } else {
      valid_datasource_names.push_back(datasource_name);
    }

    // process it
    process_datasource(datasource_handle, datasource_idx++);
  }

  // check dimensions
  if (found_dimensions.size() > 1u) {
    // report
    LOG(WARNING) << "Raster Importer: Dimensions found as follows:";
    for (auto const& dimension : found_dimensions) {
      LOG(WARNING) << "Raster Importer:   " << dimension.first << "x" << dimension.second;
    }
    if (throw_on_error) {
      throw std::runtime_error("Raster Importer: Raster file '" + file_name +
                               "' datasource/band dimensions are inconsistent. This file "
                               "cannot be imported into a single table.");
    } else {
      LOG(WARNING) << "Raster Importer: Raster file '" << file_name
                   << "' datasource/band dimensions are inconsistent. This file "
                      "cannot be imported into a single table.";
    }
  } else if (found_dimensions.size() == 1u) {
    bands_width_ = found_dimensions.begin()->first;
    bands_height_ = found_dimensions.begin()->second;
    LOG(INFO) << "Raster Importer: Importing dimension " << bands_width_ << "x"
              << bands_height_;
  }

  // report if we found nothing
  if (import_band_infos_.size() == 0 || found_dimensions.size() == 0u) {
    if (throw_on_error) {
      throw std::runtime_error("Raster Importer: Raster file " + file_name +
                               " has no importable bands");
    } else {
      LOG(ERROR) << "Raster Importer: Raster file " << file_name
                 << " has no importable bands";
    }
  }

  // report any invalid datasources and keep only the valid ones
  // the datasource indices stored in the infos will match the valid ones
  auto const failed_datasource_count =
      datasource_names_.size() - valid_datasource_names.size();
  if (failed_datasource_count) {
    LOG(WARNING) << "Raster Importer: Failed to open " << failed_datasource_count
                 << " out of " << std::to_string(datasource_names_.size())
                 << " datasources";
  }
  datasource_names_ = valid_datasource_names;

  // fail if any specified import band names were not found
  checkSpecifiedBandNamesFound();

  // optimize point_type for small rasters
  if (optimize_to_smallint && (bands_width_ <= std::numeric_limits<int16_t>::max() &&
                               bands_height_ <= std::numeric_limits<int16_t>::max())) {
    point_type_ = PointType::kSmallInt;
  }

  // capture this
  point_compute_angle_ = point_compute_angle;

  // validate final point type/transform
  if (!has_spatial_reference && point_transform_ == PointTransform::kWorld) {
    throw std::runtime_error(
        "Raster Importer: Raster file has no geo-spatial reference metadata, unable to "
        "transform points to World Space. Use raster_point_transform='none|file' "
        "instead.");
  }
  if (point_type_ == PointType::kSmallInt || point_type_ == PointType::kInt) {
    if (point_transform_ == PointTransform::kWorld) {
      throw std::runtime_error(
          "Raster Importer: Cannot do World Transform with SMALLINT/INT Point type");
    }
  } else if (point_type_ == PointType::kPoint) {
    if (point_transform_ != PointTransform::kWorld) {
      throw std::runtime_error(
          "Raster Importer: Must do World Transform with POINT Point type");
    }
  }
  if (point_type_ == PointType::kSmallInt &&
      (bands_width_ > std::numeric_limits<int16_t>::max() ||
       bands_height_ > std::numeric_limits<int16_t>::max())) {
    throw std::runtime_error(
        "Raster Importer: Raster file '" + file_name +
        "' has band dimensions too large for 'SMALLINT' raster_point_type (" +
        std::to_string(bands_width_) + "x" + std::to_string(bands_height_) + ")");
  }
  if (point_compute_angle_ && point_transform_ != PointTransform::kWorld) {
    throw std::runtime_error(
        "Raster Importer: Must do World Transform with raster_point_compute_angle");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string import_export::RasterImporter::getBandName ( const uint32_t  datasource_idx, const int  band_idx  )

private

Definition at line 1019 of file RasterImporter.cpp.

References column_name_repeats_map_, raw_band_names_, ImportHelpers::sanitize_name(), spatial_type::suffix(), and to_string().

Referenced by detect().

                                                            {
  std::string band_name;

  // format-specific name fetching
  if (datasource_idx < raw_band_names_.size()) {
    if (band_idx > 0 &&
        band_idx <= static_cast<int>(raw_band_names_[datasource_idx].size())) {
      band_name = raw_band_names_[datasource_idx][band_idx - 1];
    }
  }

  // if we didn't get a format-specific name, use a default name
  if (band_name.length() == 0) {
    band_name =
        "band_" + std::to_string(datasource_idx + 1) + "_" + std::to_string(band_idx);
  }

  // additional suffix if not unique
  auto band_name_lower = boost::algorithm::to_lower_copy(band_name);
  auto itr = column_name_repeats_map_.find(band_name_lower);
  if (itr != column_name_repeats_map_.end()) {
    auto const suffix = ++(itr->second);
    band_name += "_" + std::to_string(suffix);
  } else {
    column_name_repeats_map_.emplace(band_name_lower, 1);
  }

  // sanitize and return
  return ImportHelpers::sanitize_name(band_name, /*underscore=*/true);
}

Here is the call graph for this function:

Here is the caller graph for this function:

const RasterImporter::NamesAndSQLTypes import_export::RasterImporter::getBandNamesAndSQLTypes

(

)

const

Definition at line 696 of file RasterImporter.cpp.

References import_band_infos_.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

                                                                                   {
  NamesAndSQLTypes names_and_sql_types;
  // return alt names
  for (auto const& info : import_band_infos_) {
    names_and_sql_types.emplace_back(info.alt_name, info.sql_type);
  }
  return names_and_sql_types;
}

Here is the caller graph for this function:

const RasterImporter::NullValue import_export::RasterImporter::getBandNullValue

(

const int

band_idx

)

const

Definition at line 705 of file RasterImporter.cpp.

References CHECK_LT, and import_band_infos_.

Referenced by import_export::Importer::importGDALRaster().

                              {
  CHECK_LT(static_cast<uint32_t>(band_idx), import_band_infos_.size());
  auto const& info = import_band_infos_[band_idx];
  return {info.null_value, info.null_value_valid};
}

Here is the caller graph for this function:

const int import_export::RasterImporter::getBandsHeight ( ) const

inline

Definition at line 84 of file RasterImporter.h.

References bands_height_.

Referenced by import_export::Importer::importGDALRaster().

{ return bands_height_; }

Here is the caller graph for this function:

const int import_export::RasterImporter::getBandsWidth ( ) const

inline

Definition at line 83 of file RasterImporter.h.

References bands_width_.

Referenced by import_export::Importer::importGDALRaster().

{ return bands_width_; }

Here is the caller graph for this function:

const uint32_t import_export::RasterImporter::getNumBands

(

)

const

Definition at line 666 of file RasterImporter.cpp.

References import_band_infos_.

Referenced by import_export::Importer::importGDALRaster().

                                                 {
  return import_band_infos_.size();
}

Here is the caller graph for this function:

const RasterImporter::NamesAndSQLTypes import_export::RasterImporter::getPointNamesAndSQLTypes

(

)

const

Definition at line 670 of file RasterImporter.cpp.

References kFLOAT, kNone, kPoint, kWorld, point_compute_angle_, point_transform_, point_type_, and import_export::anonymous_namespace{RasterImporter.cpp}::point_type_to_sql_type().

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), import_export::Importer::importGDALRaster(), and initializeFiltering().

                                                                                    {
  NamesAndSQLTypes names_and_sql_types;
  if (point_type_ != PointType::kNone) {
    auto sql_type = point_type_to_sql_type(point_type_);
    if (point_transform_ == PointTransform::kWorld) {
      if (point_type_ == PointType::kPoint) {
        names_and_sql_types.emplace_back("raster_point", sql_type);
      } else {
        names_and_sql_types.emplace_back("raster_lon", sql_type);
        names_and_sql_types.emplace_back("raster_lat", sql_type);
      }
      if (point_compute_angle_) {
        names_and_sql_types.emplace_back("raster_angle", kFLOAT);
      }
    } else {
      names_and_sql_types.emplace_back("raster_x", sql_type);
      names_and_sql_types.emplace_back("raster_y", sql_type);
    }
  }
  return names_and_sql_types;
}

Here is the call graph for this function:

Here is the caller graph for this function:

const RasterImporter::PointTransform import_export::RasterImporter::getPointTransform

(

)

const

Definition at line 692 of file RasterImporter.cpp.

References point_transform_.

                                                                           {
  return point_transform_;
}

const RasterImporter::Coords import_export::RasterImporter::getProjectedPixelCoords	(	const uint32_t	thread_idx,
		const int	y
	)		const

Definition at line 712 of file RasterImporter.cpp.

References affine_transform_matrix_, bands_width_, conv_4326_900913_x(), conv_4326_900913_y(), coordinate_transformations_, gcp_transformers_, kNone, kWorld, M_PI, point_compute_angle_, and point_transform_.

Referenced by import_export::Importer::importGDALRaster().

                       {
  Coords coords(bands_width_);

  double prev_mx{0.0}, prev_my{0.0};

  for (int x = 0; x < bands_width_; x++) {
    // start with the pixel coord
    double dx = double(x);
    double dy = double(y);

    // transforms
    if (point_transform_ != PointTransform::kNone) {
      // affine transform to the file coordinate space
      double fdx = affine_transform_matrix_[0] + (dx * affine_transform_matrix_[1]) +
                   (dy * affine_transform_matrix_[2]);
      double fdy = affine_transform_matrix_[3] + (dx * affine_transform_matrix_[4]) +
                   (dy * affine_transform_matrix_[5]);
      dx = fdx;
      dy = fdy;

      // do geo-spatial transform if we can (otherwise leave alone)
      if (point_transform_ == PointTransform::kWorld) {
        // first any GCP transformation
        if (thread_idx < gcp_transformers_.size()) {
          gcp_transformers_[thread_idx]->transform(dx, dy);
        }
        // then any additional transformation to world space
        if (thread_idx < coordinate_transformations_.size()) {
          int success{0};
          coordinate_transformations_[thread_idx]->Transform(
              1, &dx, &dy, nullptr, &success);
          if (!success) {
            throw std::runtime_error("Failed OGRCoordinateTransform");
          }
        }
      }
    }

    // store
    coords[x] = {dx, dy, 0.0f};

    // compute and overwrite angle?
    if (point_compute_angle_) {
      if (x == 0) {
        // capture first pixel's Mercator coords
        prev_mx = conv_4326_900913_x(dx);
        prev_my = conv_4326_900913_y(dy);
      } else {
        // compute angle between this pixel's Mercator coords and the previous
        // expressed as clockwise degrees for symbol rotation
        auto const mx = conv_4326_900913_x(dx);
        auto const my = conv_4326_900913_y(dy);
        auto const angle = atan2f(my - prev_my, mx - prev_mx) * (-180.0f / M_PI);
        prev_mx = mx;
        prev_my = my;

        // overwrite this angle (and that of the first pixel, if this is the second)
        std::get<2>(coords[x]) = angle;
        if (x == 1) {
          std::get<2>(coords[0]) = angle;
        }
      }
    }
  }

  // done
  return coords;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void import_export::RasterImporter::getRawBandNamesForFormat ( const Geospatial::GDAL::DataSourceUqPtr &  datasource )

private

Definition at line 832 of file RasterImporter.cpp.

References CHECK, datasource_names_, import_export::anonymous_namespace{RasterImporter.cpp}::get_ome_tiff_band_names(), Geospatial::GDAL::getMetadataString(), logger::INFO, LOG, setup::name, raw_band_names_, split(), and logger::WARNING.

Referenced by detect().

                                                     {
  // get the name of the driver that GDAL picked to open the file
  std::string driver_name = datasource->GetDriverName();

  LOG(INFO) << "Raster Importer: Using Raster Driver '" << driver_name << "'";

  // logic is different for each format
  if (driver_name == "GTiff") {
    //
    // TIFF
    // Could be an OME TIFF or a GeoTIFF
    //
    auto const tifftag_imagedescription = Geospatial::GDAL::getMetadataString(
        datasource->GetMetadata(), "TIFFTAG_IMAGEDESCRIPTION");
    if (tifftag_imagedescription.length()) {
      //
      // OME TIFF
      // one datasource per band
      // names are in a JSON blob
      //
      auto const names = get_ome_tiff_band_names(tifftag_imagedescription);
      for (auto const& name : names) {
        raw_band_names_.push_back({name});
      }
    } else {
      //
      // Some other GeoTIFF variant
      // single datasource
      // names in band descriptions?
      //
      std::vector<std::string> names;
      auto const raster_count = datasource->GetRasterCount();
      for (int i = 1; i <= raster_count; i++) {
        auto* band = datasource->GetRasterBand(i);
        CHECK(band);
        auto const* description = band->GetDescription();
        if (!description || strlen(description) == 0) {
          raw_band_names_.clear();
          return;
        }
        names.emplace_back(description);
      }
      raw_band_names_.emplace_back(std::move(names));
    }
  } else if (driver_name == "netCDF" || driver_name == "Zarr") {
    //
    // for these formats the band names are in the datasource names
    // that we already obtained, of the format:
    // <FORMAT>:"<filename>":<bandname>
    // one band per datasource
    //
    for (auto const& datasource_name : datasource_names_) {
      std::vector<std::string> tokens;
      boost::split(tokens, datasource_name, boost::is_any_of(":"));
      if (tokens.size() < 3 || tokens[2].length() == 0u) {
        LOG(WARNING) << "Raster Importer: Failed to parse band name from datasource name";
        raw_band_names_.clear();
        return;
      }
      raw_band_names_.push_back({tokens[2]});
    }
  } else if (driver_name == "GRIB") {
    //
    // GRIB/GRIB2
    // one datasource
    // names are in the per-band metadata
    //
    std::vector<std::string> names;
    auto const raster_count = datasource->GetRasterCount();
    for (int i = 1; i <= raster_count; i++) {
      auto* band = datasource->GetRasterBand(i);
      CHECK(band);
      auto const grib_comment =
          Geospatial::GDAL::getMetadataString(band->GetMetadata(), "GRIB_COMMENT");
      if (grib_comment.length() == 0) {
        LOG(WARNING) << "Raster Importer: Failed to parse band name from GRIB_COMMENT";
        raw_band_names_.clear();
        return;
      }
      names.push_back(grib_comment);
    }
    raw_band_names_.emplace_back(std::move(names));
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void import_export::RasterImporter::getRawPixels	(	const uint32_t	thread_idx,
		const uint32_t	band_idx,
		const int	y_start,
		const int	num_rows,
		const SQLTypes	column_sql_type,
		RawPixels &	raw_pixel_bytes
	)

Definition at line 783 of file RasterImporter.cpp.

References bands_width_, CHECK, CHECK_LT, datasource_handles_, import_band_infos_, run_benchmark_import::result, import_export::anonymous_namespace{RasterImporter.cpp}::sql_type_to_gdal_data_type(), and to_string().

Referenced by import_export::Importer::importGDALRaster().

                                                              {
  // get the band info
  CHECK_LT(band_idx, import_band_infos_.size());
  auto const band_info = import_band_infos_[band_idx];

  // get the dataset handle for this dataset and thread
  CHECK_LT(band_info.datasource_idx, datasource_handles_.size());
  auto const& datasource_handles_per_thread =
      datasource_handles_[band_info.datasource_idx];
  CHECK_LT(thread_idx, datasource_handles_per_thread.size());
  auto const& datasource_handle = datasource_handles_per_thread[thread_idx];
  CHECK(datasource_handle);

  // get the band
  auto* band = datasource_handle->GetRasterBand(band_info.band_idx);
  CHECK(band);

  // translate the requested data type
  auto const gdal_data_type = sql_type_to_gdal_data_type(column_sql_type);

  // read the scanlines
  auto result = band->RasterIO(GF_Read,
                               0,
                               y_start,
                               bands_width_,
                               num_rows,
                               raw_pixel_bytes.data(),
                               bands_width_,
                               num_rows,
                               gdal_data_type,
                               0,
                               0,
                               nullptr);
  if (result != CE_None) {
    throw std::runtime_error("Failed to read raster pixels, rows " +
                             std::to_string(y_start) + " to " +
                             std::to_string(y_start + num_rows));
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void import_export::RasterImporter::import	(	size_t &	max_threads,
		const bool	max_threads_using_default
	)

Definition at line 547 of file RasterImporter.cpp.

References affine_transform_matrix_, CHECK, CHECK_GE, coordinate_transformations_, datasource_handles_, datasource_names_, import_export::anonymous_namespace{RasterImporter.cpp}::datasource_requires_libhdf(), gcp_transformers_, import_export::anonymous_namespace{RasterImporter.cpp}::get_datasource_driver_name(), shared::get_from_map(), import_band_infos_, kAuto, kDouble, kFloat, kInt, kNone, kPoint, import_export::kRasterFile, kSmallInt, kWorld, Geospatial::GDAL::openDataSource(), point_transform_, point_type_, and to_string().

Referenced by import_export::Importer::importGDALRaster().

                                                                                     {
  // validate
  if (import_band_infos_.size() == 0) {
    throw std::runtime_error("Raster Import aborted. No bands to import.");
  }

  // validate
  CHECK_GE(max_threads, 1u);

  // open all datasources on all threads
  std::map<std::string, std::vector<Geospatial::GDAL::DataSourceUqPtr>>
      datasource_thread_handles_map;
  for (uint32_t i = 0; i < max_threads; i++) {
    for (auto const& datasource_name : datasource_names_) {
      auto& datasource_thread_handles = datasource_thread_handles_map[datasource_name];
      auto datasource_handle = Geospatial::GDAL::openDataSource(
          datasource_name, import_export::SourceType::kRasterFile);
      if (datasource_handle == nullptr) {
        throw std::runtime_error("Raster Importer: Unable to open raster file " +
                                 datasource_name);
      }
      if (i == 0) {  // check for HDF5 data sources in first loop
        if (datasource_requires_libhdf(datasource_handle.get()) && max_threads > 1) {
          if (!max_threads_using_default) {
            throw std::runtime_error("Raster Importer: Unable to import raster file " +
                                     datasource_name + ": GDAL driver " +
                                     get_datasource_driver_name(datasource_handle.get()) +
                                     " requires use of HDF5 library which is "
                                     "incompatible with multithreading.");
          } else {
            max_threads = 1;  // force use of one thread
          }
        }
      }
      datasource_thread_handles.emplace_back(std::move(datasource_handle));
    }
  }

  for (auto const& datasource_name : datasource_names_) {
    datasource_handles_.emplace_back(
        std::move(shared::get_from_map(datasource_thread_handles_map, datasource_name)));
  }

  // use handle for the first datasource from the first thread to read the globals
  auto const& global_datasource_handle = datasource_handles_[0][0];

  // get the raster affine transform
  // this converts the points from pixel space to the file coordinate system space
  global_datasource_handle->GetGeoTransform(affine_transform_matrix_.data());

  // transform logic
  // @TODO(se) discuss!
  // if the raster_point_type is SMALLINT or INT, we just store pixel X/Y
  // if the raster_point_type is FLOAT, DOUBLE, or POINT, we store projected X/Y
  // this will either be in the file coordinate space (raster_point_transform='affine')
  // or in world space (raster_point_transform='default')
  // yeah, that's a mess, but I need to sleep on it

  // determine the final desired SRID
  // the first column will either be a scalar of some type or a POINT
  // FLOAT or DOUBLE can support world space coords (so assume 4326)
  // POINT can be anything and can define an SRID (but assume 4326 for now)
  int srid{0};
  switch (point_type_) {
    case PointType::kNone:
    case PointType::kSmallInt:
    case PointType::kInt:
      break;
    case PointType::kFloat:
    case PointType::kDouble:
    case PointType::kPoint: {
      srid = 4326;
    } break;
    case PointType::kAuto:
      CHECK(false);
  }

  // create a world-space coordinate transformation for the points?
  if (srid != 0 && point_transform_ == PointTransform::kWorld) {
    // get the file's spatial reference, if it has one (only geo files will)
    // also determine if we need to do additional GCP transformations
    bool need_gcp_transformers{false};
    auto const* spatial_reference = global_datasource_handle->GetSpatialRef();
    if (!spatial_reference) {
      spatial_reference = global_datasource_handle->GetGCPSpatialRef();
      if (spatial_reference) {
        need_gcp_transformers = true;
      }
    }
    if (spatial_reference) {
      // if it's valid, create a transformation to use on the points
      // make a spatial reference for the desired SRID
      OGRSpatialReference sr_geometry;
      auto import_err = sr_geometry.importFromEPSG(srid);
      if (import_err != OGRERR_NONE) {
        throw std::runtime_error("Failed to create spatial reference for EPSG " +
                                 std::to_string(srid));
      }
#if GDAL_VERSION_MAJOR >= 3
      sr_geometry.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
#endif

      for (uint32_t i = 0; i < max_threads; i++) {
        coordinate_transformations_.emplace_back(
            OGRCreateCoordinateTransformation(spatial_reference, &sr_geometry));
        if (!coordinate_transformations_.back()) {
          throw std::runtime_error(
              "Failed to create coordinate system transformation to EPSG " +
              std::to_string(srid));
        }
        if (need_gcp_transformers) {
          gcp_transformers_.emplace_back(
              new GCPTransformer(global_datasource_handle.get()));
        }
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void import_export::RasterImporter::initializeFiltering ( const std::string &  specified_band_names, const std::string &  specified_band_dimensions, const MetadataColumnInfos &  metadata_column_infos  )

private

Definition at line 918 of file RasterImporter.cpp.

References column_name_repeats_map_, getPointNamesAndSQLTypes(), import_band_infos_, kNULLT, setup::name, specified_band_height_, specified_band_names_map_, specified_band_width_, split(), and strip().

Referenced by detect().

                                                      {
  // specified names?
  if (specified_band_names.length()) {
    // tokenize names
    std::vector<std::string> names_and_alt_names;
    boost::split(names_and_alt_names, specified_band_names, boost::is_any_of(","));

    // pre-populate infos (in given order) and found map
    for (auto const& name_and_alt_name : names_and_alt_names) {
      // parse for optional alt name
      std::vector<std::string> tokens;
      boost::split(tokens, name_and_alt_name, boost::is_any_of("="));
      if (tokens.size() < 1 || tokens.size() > 2) {
        throw std::runtime_error("Failed to parse specified name '" + name_and_alt_name +
                                 "'");
      }
      auto const& name = strip(tokens[0]);
      auto const& alt_name = strip(tokens[tokens.size() == 2 ? 1 : 0]);
      import_band_infos_.push_back({name, alt_name, kNULLT, 0u, 0, 0.0, false});
      specified_band_names_map_.emplace(name, false);
    }
  }

  column_name_repeats_map_.clear();

  // initialize repeats map with point column names(s)
  // in case there are bands with the same name
  auto const names_and_sql_types = getPointNamesAndSQLTypes();
  for (auto const& name_and_sql_type : names_and_sql_types) {
    column_name_repeats_map_.emplace(
        boost::algorithm::to_lower_copy(name_and_sql_type.first), 1);
  }

  // specified band dimensions?
  if (specified_band_dimensions.length()) {
    // tokenize dimension values
    std::vector<std::string> values;
    boost::split(values, specified_band_dimensions, boost::is_any_of(",x "));
    if (values.size() != 2u) {
      throw std::invalid_argument("failed to parse width/height values from '" +
                                  specified_band_dimensions + "'");
    }
    try {
      size_t num_chars_w{0u}, num_chars_h{0u};
      specified_band_width_ = std::stoi(values[0], &num_chars_w);
      specified_band_height_ = std::stoi(values[1], &num_chars_h);
      if (num_chars_w == 0u || num_chars_h == 0u) {
        throw std::invalid_argument("empty width/height value");
      }
      if (specified_band_width_ < 0 || specified_band_height_ < 0) {
        throw std::invalid_argument("negative width/height value");
      }
    } catch (std::invalid_argument& e) {
      throw std::runtime_error("Raster Importer: Invalid specified dimensions (" +
                               std::string(e.what()) + ")");
    } catch (std::out_of_range& e) {
      throw std::runtime_error("Raster Importer: Out-of-range specified dimensions (" +
                               std::string(e.what()) + ")");
    }
  }

  // also any metadata column names
  for (auto const& mci : metadata_column_infos) {
    auto column_name_lower =
        boost::algorithm::to_lower_copy(mci.column_descriptor.columnName);
    if (column_name_repeats_map_.find(column_name_lower) !=
        column_name_repeats_map_.end()) {
      throw std::runtime_error("Invalid metadata column name '" +
                               mci.column_descriptor.columnName +
                               "' (clashes with existing column name)");
    }
    column_name_repeats_map_.emplace(column_name_lower, 1);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool import_export::RasterImporter::shouldImportBandWithDimensions ( const int  width, const int  height  )

private

Definition at line 1010 of file RasterImporter.cpp.

References specified_band_height_, and specified_band_width_.

Referenced by detect().

                                                                                     {
  // if no specified dimensions, import everything
  if (specified_band_width_ < 0 && specified_band_height_ < 0) {
    return true;
  }
  // import only if dimensions match
  return (width == specified_band_width_) && (height == specified_band_height_);
}

Here is the caller graph for this function:

bool import_export::RasterImporter::shouldImportBandWithName ( const std::string &  name )

private

Definition at line 996 of file RasterImporter.cpp.

References specified_band_names_map_.

Referenced by detect().

                                                                   {
  // if no specified band names, import everything
  if (specified_band_names_map_.size() == 0u) {
    return true;
  }
  // find it, and mark as found
  auto itr = specified_band_names_map_.find(name);
  if (itr != specified_band_names_map_.end()) {
    itr->second = true;
    return true;
  }
  return false;
}

Here is the caller graph for this function:

Member Data Documentation

std::array<double, 6> import_export::RasterImporter::affine_transform_matrix_

private

Definition at line 110 of file RasterImporter.h.

Referenced by getProjectedPixelCoords(), and import().

int import_export::RasterImporter::bands_height_ {-1}

private

Definition at line 116 of file RasterImporter.h.

Referenced by detect(), and getBandsHeight().

int import_export::RasterImporter::bands_width_ {-1}

private

Definition at line 115 of file RasterImporter.h.

Referenced by detect(), getBandsWidth(), getProjectedPixelCoords(), and getRawPixels().

std::map<std::string, int> import_export::RasterImporter::column_name_repeats_map_

private

Definition at line 108 of file RasterImporter.h.

Referenced by getBandName(), and initializeFiltering().

std::vector<Geospatial::GDAL::CoordinateTransformationUqPtr> import_export::RasterImporter::coordinate_transformations_

private

Definition at line 120 of file RasterImporter.h.

Referenced by getProjectedPixelCoords(), and import().

std::vector<std::vector<Geospatial::GDAL::DataSourceUqPtr> > import_export::RasterImporter::datasource_handles_

private

Definition at line 111 of file RasterImporter.h.

Referenced by getRawPixels(), and import().

std::vector<std::string> import_export::RasterImporter::datasource_names_

private

Definition at line 105 of file RasterImporter.h.

Referenced by detect(), getRawBandNamesForFormat(), and import().

std::vector<std::unique_ptr<GCPTransformer> > import_export::RasterImporter::gcp_transformers_

private

Definition at line 121 of file RasterImporter.h.

Referenced by getProjectedPixelCoords(), and import().

std::vector<ImportBandInfo> import_export::RasterImporter::import_band_infos_

private

Definition at line 109 of file RasterImporter.h.

Referenced by detect(), getBandNamesAndSQLTypes(), getBandNullValue(), getNumBands(), getRawPixels(), import(), and initializeFiltering().

bool import_export::RasterImporter::point_compute_angle_ {false}

private

Definition at line 122 of file RasterImporter.h.

Referenced by detect(), getPointNamesAndSQLTypes(), and getProjectedPixelCoords().

PointTransform import_export::RasterImporter::point_transform_ {PointTransform::kNone}

private

Definition at line 118 of file RasterImporter.h.

Referenced by detect(), getPointNamesAndSQLTypes(), getPointTransform(), getProjectedPixelCoords(), and import().

PointType import_export::RasterImporter::point_type_ {PointType::kNone}

private

Definition at line 117 of file RasterImporter.h.

Referenced by detect(), getPointNamesAndSQLTypes(), and import().

std::vector<std::vector<std::string> > import_export::RasterImporter::raw_band_names_

private

Definition at line 106 of file RasterImporter.h.

Referenced by getBandName(), and getRawBandNamesForFormat().

int import_export::RasterImporter::specified_band_height_ {-1}

private

Definition at line 113 of file RasterImporter.h.

Referenced by initializeFiltering(), and shouldImportBandWithDimensions().

std::map<std::string, bool> import_export::RasterImporter::specified_band_names_map_

private

Definition at line 107 of file RasterImporter.h.

Referenced by checkSpecifiedBandNamesFound(), detect(), initializeFiltering(), and shouldImportBandWithName().

int import_export::RasterImporter::specified_band_width_ {-1}

private

Definition at line 112 of file RasterImporter.h.

Referenced by initializeFiltering(), and shouldImportBandWithDimensions().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/RasterImporter.h
• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/RasterImporter.cpp