AlterTableAlterColumnCommandRecoveryMgr Class Reference

#include <AlterColumnRecovery.h>

Collaboration diagram for AlterTableAlterColumnCommandRecoveryMgr:

Classes
struct	ColumnAltered
struct	RecoveryInfo
struct	RecoveryParamFilepathInfo

Public Types
using	TypePairs = alter_column_shared::TypePairs

Public Member Functions
	AlterTableAlterColumnCommandRecoveryMgr (Catalog_Namespace::Catalog &catalog)
void	rollback (const TableDescriptor *td, const RecoveryInfo &param)
void	cleanup (const TableDescriptor *td, const TypePairs &src_dst_cds)
void	checkpoint (const TableDescriptor *td, const TypePairs &src_dst_cds)
RecoveryInfo	deserializeRecoveryInformation (const std::string &filename)
std::string	serializeRecoveryInformation (const RecoveryInfo &param)
void	writeSerializedRecoveryInformation (const RecoveryInfo &param, const RecoveryParamFilepathInfo &filepath_info)
void	readSerializedRecoveryInformation (RecoveryInfo &param, const RecoveryParamFilepathInfo &filepath_info)
std::string	recoveryFilepath (const RecoveryParamFilepathInfo &filepath_info)
RecoveryParamFilepathInfo	getRecoveryFilepathInfo (const int32_t table_id=-1)

Static Public Member Functions
static void	resolveIncompleteAlterColumnCommandsForAllCatalogs ()

Static Public Attributes
static const std::string	kRecoveryDirectoryName = "crash_recovery"

Private Member Functions
void	cleanupDeleteDictionaries (const TableDescriptor *td, const TypePairs &src_dst_cds)
void	cleanupClearChunk (const ChunkKey &key, const MemoryLevel mem_level)
void	cleanupClearChunk (const ChunkKey &key)
void	cleanupClearRemainingChunks (const TableDescriptor *td, const TypePairs &src_dst_cds)
void	cleanupDropSourceGeoColumns (const TableDescriptor *td, const TypePairs &src_dst_cds)
std::list< std::pair < ColumnDescriptor, ColumnDescriptor > >	toPairedCds (const std::list< ColumnAltered > &altered_columns)
void	recoverAlterTableAlterColumnFromFile (const std::string &filename)
std::list< std::filesystem::path >	getRecoveryFiles ()
TypePairs	getSrcDstCds (int table_id, std::list< std::pair< ColumnDescriptor, ColumnDescriptor >> &pairs_list)
void	resolveIncompleteAlterColumnCommands ()

Static Private Member Functions
static std::list< ColumnAltered >	fromPairedCds (const std::list< std::pair< ColumnDescriptor, ColumnDescriptor >> &altered_columns)
static std::filesystem::path	getRecoveryPrefix (const std::string &base_path)
static std::map< std::string, AlterTableAlterColumnCommandRecoveryMgr >	createRecoveryManagersForCatalogs ()

Private Attributes
Catalog_Namespace::Catalog &	catalog_

Detailed Description

Definition at line 22 of file AlterColumnRecovery.h.

Member Typedef Documentation

using AlterTableAlterColumnCommandRecoveryMgr::TypePairs = alter_column_shared::TypePairs

Definition at line 26 of file AlterColumnRecovery.h.

Constructor & Destructor Documentation

AlterTableAlterColumnCommandRecoveryMgr::AlterTableAlterColumnCommandRecoveryMgr

(

Catalog_Namespace::Catalog &

catalog

)

Definition at line 27 of file AlterColumnRecovery.cpp.

    : catalog_(catalog) {}

Member Function Documentation

void AlterTableAlterColumnCommandRecoveryMgr::checkpoint	(	const TableDescriptor *	td,
		const TypePairs &	src_dst_cds
	)

Definition at line 348 of file AlterColumnRecovery.cpp.

References catalog_, Catalog_Namespace::Catalog::checkpointWithAutoRollback(), ddl_utils::alter_column_utils::compare_column_descriptors(), Catalog_Namespace::Catalog::getMetadataForDict(), ddl_utils::alter_column_utils::CompareResult::sql_types_match, and TableDescriptor::tableId.

Referenced by AlterTableAlterColumnCommand::alterColumnTypes(), and cleanup().

                                                                                       {
  for (auto& [src_cd, dst_cd] : src_dst_cds) {
    if (!dst_cd->columnType.is_dict_encoded_type()) {
      continue;
    }
    if (ddl_utils::alter_column_utils::compare_column_descriptors(src_cd, dst_cd)
            .sql_types_match) {
      continue;
    }
    auto string_dictionary =
        catalog_.getMetadataForDict(dst_cd->columnType.get_comp_param(), true)
            ->stringDict.get();
    if (!string_dictionary->checkpoint()) {
      throw std::runtime_error("Failed to checkpoint dictionary while altering column " +
                               dst_cd->columnName + ".");
    }
  }
  catalog_.checkpointWithAutoRollback(td->tableId);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanup	(	const TableDescriptor *	td,
		const TypePairs &	src_dst_cds
	)

Definition at line 317 of file AlterColumnRecovery.cpp.

References catalog_, checkpoint(), cleanupClearRemainingChunks(), cleanupDeleteDictionaries(), cleanupDropSourceGeoColumns(), LOG, Catalog_Namespace::Catalog::resetTableEpochFloor(), TableDescriptor::tableId, and logger::WARNING.

Referenced by AlterTableAlterColumnCommand::alterColumnTypes(), and recoverAlterTableAlterColumnFromFile().

                                                                                    {
  try {
    cleanupDeleteDictionaries(td, src_dst_cds);
  } catch (std::exception& except) {
    LOG(WARNING) << "Alter column type: failed to clear source dictionaries: "
                 << except.what();
    throw;
  }

  try {
    cleanupClearRemainingChunks(td, src_dst_cds);
  } catch (std::exception& except) {
    LOG(WARNING) << "Alter column type: failed to clear remaining chunks: "
                 << except.what();
    throw;
  }

  try {
    cleanupDropSourceGeoColumns(td, src_dst_cds);
  } catch (std::exception& except) {
    LOG(WARNING) << "Alter column type: failed to remove geo's source column : "
                 << except.what();
    throw;
  }

  // Data is removed data, rollback no longer possible
  checkpoint(td, src_dst_cds);
  catalog_.resetTableEpochFloor(td->tableId);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanupClearChunk ( const ChunkKey &  key, const MemoryLevel  mem_level  )

private

Definition at line 115 of file AlterColumnRecovery.cpp.

References catalog_, Data_Namespace::DataMgr::deleteChunk(), and Catalog_Namespace::Catalog::getDataMgr().

Referenced by cleanupClearChunk(), and cleanupClearRemainingChunks().

                                 {
  auto& data_mgr = catalog_.getDataMgr();
  if (mem_level >= data_mgr.levelSizes_.size()) {
    return;
  }
  for (int device = 0; device < data_mgr.levelSizes_[mem_level]; ++device) {
    if (data_mgr.isBufferOnDevice(key, mem_level, device)) {
      data_mgr.deleteChunk(key, mem_level, device);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanupClearChunk ( const ChunkKey &  key )

private

Definition at line 129 of file AlterColumnRecovery.cpp.

References cleanupClearChunk(), Data_Namespace::CPU_LEVEL, Data_Namespace::DISK_LEVEL, and Data_Namespace::GPU_LEVEL.

                                                                                   {
  cleanupClearChunk(key, MemoryLevel::GPU_LEVEL);
  cleanupClearChunk(key, MemoryLevel::CPU_LEVEL);
  cleanupClearChunk(key, MemoryLevel::DISK_LEVEL);
}

Here is the call graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanupClearRemainingChunks ( const TableDescriptor *  td, const TypePairs &  src_dst_cds  )

private

Definition at line 135 of file AlterColumnRecovery.cpp.

References catalog_, CHUNK_KEY_FRAGMENT_IDX, cleanupClearChunk(), Data_Namespace::DataMgr::getChunkMetadataVecForKeyPrefix(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), and TableDescriptor::tableId.

Referenced by cleanup().

                                  {
  // for (non-geo) cases where the chunk keys change, chunks that remain with old chunk
  // key must be removed
  for (auto& [src_cd, dst_cd] : src_dst_cds) {
    if (src_cd->columnType.is_varlen_indeed() != dst_cd->columnType.is_varlen_indeed()) {
      ChunkMetadataVector chunk_metadata;
      catalog_.getDataMgr().getChunkMetadataVecForKeyPrefix(
          chunk_metadata, {catalog_.getDatabaseId(), td->tableId, dst_cd->columnId});
      std::set<int> fragment_ids;
      for (const auto& [key, _] : chunk_metadata) {
        fragment_ids.insert(key[CHUNK_KEY_FRAGMENT_IDX]);
      }
      for (const auto& frag_id : fragment_ids) {
        ChunkKey key = {catalog_.getDatabaseId(), td->tableId, src_cd->columnId, frag_id};
        if (src_cd->columnType.is_varlen_indeed()) {
          auto data_key = key;
          data_key.push_back(1);
          cleanupClearChunk(data_key);
          auto index_key = key;
          index_key.push_back(2);
          cleanupClearChunk(index_key);
        } else {  // no varlen case
          cleanupClearChunk(key);
        }
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanupDeleteDictionaries ( const TableDescriptor *  td, const TypePairs &  src_dst_cds  )

private

Definition at line 101 of file AlterColumnRecovery.cpp.

References catalog_, ddl_utils::alter_column_utils::compare_column_descriptors(), Catalog_Namespace::Catalog::delDictionaryTransactional(), and ddl_utils::alter_column_utils::CompareResult::sql_types_match.

Referenced by cleanup().

                                  {
  for (auto& [src_cd, dst_cd] : src_dst_cds) {
    if (!src_cd->columnType.is_dict_encoded_type()) {
      continue;
    }
    if (!ddl_utils::alter_column_utils::compare_column_descriptors(src_cd, dst_cd)
             .sql_types_match) {
      catalog_.delDictionaryTransactional(*src_cd);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns ( const TableDescriptor *  td, const TypePairs &  src_dst_cds  )

private

Definition at line 166 of file AlterColumnRecovery.cpp.

References catalog_, Data_Namespace::CPU_LEVEL, Catalog_Namespace::DBMetadata::dbId, Data_Namespace::DataMgr::deleteChunksWithPrefix(), Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::dropColumnTransactional(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), Data_Namespace::GPU_LEVEL, and TableDescriptor::tableId.

Referenced by cleanup().

                                  {
  for (auto& [src_cd, dst_cd] : src_dst_cds) {
    if (!dst_cd->columnType.is_geometry()) {
      continue;
    }
    auto catalog_cd = catalog_.getMetadataForColumn(src_cd->tableId, src_cd->columnId);
    catalog_.dropColumnTransactional(*td, *catalog_cd);
    ChunkKey col_key{catalog_.getCurrentDB().dbId, td->tableId, src_cd->columnId};
    auto& data_mgr = catalog_.getDataMgr();
    data_mgr.deleteChunksWithPrefix(col_key, MemoryLevel::GPU_LEVEL);
    data_mgr.deleteChunksWithPrefix(col_key, MemoryLevel::CPU_LEVEL);
    data_mgr.deleteChunksWithPrefix(col_key, MemoryLevel::DISK_LEVEL);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::map< std::string, AlterTableAlterColumnCommandRecoveryMgr > AlterTableAlterColumnCommandRecoveryMgr::createRecoveryManagersForCatalogs ( )

staticprivate

Definition at line 509 of file AlterColumnRecovery.cpp.

References CHECK, getRecoveryPrefix(), Catalog_Namespace::SysCatalog::instance(), and run_benchmark_import::result.

Referenced by resolveIncompleteAlterColumnCommandsForAllCatalogs().

                                                                           {
  std::map<std::string, AlterTableAlterColumnCommandRecoveryMgr> result;

  auto& syscat = Catalog_Namespace::SysCatalog::instance();
  auto base_path = syscat.getCatalogBasePath();
  auto prefix = getRecoveryPrefix(base_path);
  if (!std::filesystem::exists(prefix)) {
    return {};
  }

  auto catalog_metadata = syscat.getAllDBMetadata();

  for (const auto& entry : std::filesystem::directory_iterator(prefix)) {
    auto entry_path = entry.path().string();

    for (const auto& db_metadata : catalog_metadata) {
      if (result.count(db_metadata.dbName)) {
        continue;
      }
      auto match_db = entry_path.find("alter_column_recovery_db_" + db_metadata.dbName);
      if (match_db == std::string::npos) {
        continue;
      }
      auto catalog = syscat.getCatalog(db_metadata.dbName);
      CHECK(catalog.get());
      result.emplace(db_metadata.dbName, *catalog);
    }
  }

  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo AlterTableAlterColumnCommandRecoveryMgr::deserializeRecoveryInformation

(

const std::string &

filename

)

Definition at line 204 of file AlterColumnRecovery.cpp.

References AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::added_columns, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::altered_columns, CHECK, fromPairedCds(), json_utils::get_value_from_object(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::is_vacuumed, json_utils::read_from_file(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::renamed_columns, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::src_dst_cds, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::table_epoch, and AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::updated_dict_cds.

Referenced by readSerializedRecoveryInformation(), and recoverAlterTableAlterColumnFromFile().

                               {
  RecoveryInfo param;
  auto d = json_utils::read_from_file(filename);
  CHECK(d.IsObject());

  json_utils::get_value_from_object(d, param.added_columns, "added_columns");

  std::list<std::pair<ColumnDescriptor, ColumnDescriptor>> altered_column_pairs;
  json_utils::get_value_from_object(d, altered_column_pairs, "altered_columns");
  param.altered_columns = fromPairedCds(altered_column_pairs);

  json_utils::get_value_from_object(d, param.renamed_columns, "renamed_columns");

  json_utils::get_value_from_object(d, param.updated_dict_cds, "updated_dict_cds");

  json_utils::get_value_from_object(d, param.table_epoch, "table_epoch");
  json_utils::get_value_from_object(d, param.is_vacuumed, "is_vacuumed");

  json_utils::get_value_from_object(d, param.src_dst_cds, "src_dst_cds");

  return param;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::list< AlterTableAlterColumnCommandRecoveryMgr::ColumnAltered > AlterTableAlterColumnCommandRecoveryMgr::fromPairedCds ( const std::list< std::pair< ColumnDescriptor, ColumnDescriptor >> &  altered_columns )

staticprivate

Definition at line 184 of file AlterColumnRecovery.cpp.

Referenced by deserializeRecoveryInformation().

                                                                                 {
  std::list<ColumnAltered> retval;
  for (const auto& [old_cd, new_cd] : altered_columns) {
    retval.emplace_back(old_cd, new_cd);
  }
  return retval;
}

Here is the caller graph for this function:

AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFilepathInfo

(

const int32_t

table_id = -1

)

Definition at line 248 of file AlterColumnRecovery.cpp.

References AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::base_path, catalog_, AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::db_name, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::Catalog::getCatalogBasePath(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getTableName(), and AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::table_name.

Referenced by AlterTableAlterColumnCommand::cleanupRecoveryInfo(), getRecoveryFiles(), and AlterTableAlterColumnCommand::populateAndWriteRecoveryInfo().

                                                                                       {
  RecoveryParamFilepathInfo path_info;
  path_info.base_path = catalog_.getCatalogBasePath();
  path_info.db_name = catalog_.getCurrentDB().dbName;
  path_info.table_name =
      table_id >= 0 ? catalog_.getTableName(table_id).value() : std::string{};
  return path_info;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::list< std::filesystem::path > AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFiles ( )

private

Definition at line 370 of file AlterColumnRecovery.cpp.

References catalog_, getRecoveryFilepathInfo(), logger::INFO, LOG, Catalog_Namespace::Catalog::name(), recoveryFilepath(), and run_benchmark_import::result.

Referenced by resolveIncompleteAlterColumnCommands().

                                                          {
  std::list<std::filesystem::path> result;
  std::string path = recoveryFilepath(getRecoveryFilepathInfo());

  if (!std::filesystem::exists(path)) {
    return {};
  }

  for (const auto& entry : std::filesystem::directory_iterator(path)) {
    auto entry_path = entry.path().string();
    if (entry_path.find("alter_column_recovery_db_" + catalog_.name() + "_table_") !=
            std::string::npos &&
        entry_path.find(".json") != std::string::npos) {
      if (entry_path.find(".tmp") != std::string::npos &&
          entry_path.find(".tmp") == entry_path.size() - std::string{".tmp"}.size()) {
        if (!std::filesystem::remove(entry_path)) {
          throw std::runtime_error("Failed to remove incomplete recovery file: " +
                                   entry_path);
        } else {
          LOG(INFO) << "Removing incomplete ALTER COLUMN recovery file: " + entry_path;
        }
      } else {
        result.emplace_back(entry.path());
      }
    }
  }
  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::filesystem::path AlterTableAlterColumnCommandRecoveryMgr::getRecoveryPrefix ( const std::string &  base_path )

staticprivate

Definition at line 228 of file AlterColumnRecovery.cpp.

References kRecoveryDirectoryName.

Referenced by createRecoveryManagersForCatalogs(), recoveryFilepath(), and writeSerializedRecoveryInformation().

                                {
  return std::filesystem::path(base_path) / std::filesystem::path(kRecoveryDirectoryName);
}

Here is the caller graph for this function:

AlterTableAlterColumnCommandRecoveryMgr::TypePairs AlterTableAlterColumnCommandRecoveryMgr::getSrcDstCds ( int  table_id, std::list< std::pair< ColumnDescriptor, ColumnDescriptor >> &  pairs_list  )

private

Definition at line 400 of file AlterColumnRecovery.cpp.

References catalog_, Catalog_Namespace::Catalog::getMetadataForColumn(), and run_benchmark_import::result.

Referenced by recoverAlterTableAlterColumnFromFile().

                                                                      {
  // Source columns must be obtained from catalog to ensure correctness/consistency
  std::list<std::pair<const ColumnDescriptor*, ColumnDescriptor*>> result;
  for (auto& [src, dst] : pairs_list) {
    auto catalog_cd = catalog_.getMetadataForColumn(table_id, src.columnId);
    if (!catalog_cd) {
      // If column is missing in catalog, operate under the assumption it was
      // already successfully removed in cleanup, along with all related
      // components such as dictionaries
      continue;
    }
    result.emplace_back(catalog_cd, &dst);
  }
  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::readSerializedRecoveryInformation	(	RecoveryInfo &	param,
		const RecoveryParamFilepathInfo &	filepath_info
	)

Definition at line 257 of file AlterColumnRecovery.cpp.

References deserializeRecoveryInformation(), nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), and recoveryFilepath().

                                                    {
  auto filename = recoveryFilepath(filepath_info);
  std::ifstream ifs(filename);
  if (!ifs) {
    throw std::runtime_error{"Error trying to read file \"" + filename +
                             "\". The error was: " + std::strerror(errno)};
  }
  std::string json_string;
  ifs >> json_string;
  param = deserializeRecoveryInformation(json_string);
}

Here is the call graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile ( const std::string &  filename )

private

Definition at line 418 of file AlterColumnRecovery.cpp.

References catalog_, CHECK, CHECK_EQ, CHECK_GT, cleanup(), Data_Namespace::CPU_LEVEL, Catalog_Namespace::DBMetadata::dbId, Data_Namespace::DataMgr::deleteChunksWithPrefix(), deserializeRecoveryInformation(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), getSrcDstCds(), Catalog_Namespace::Catalog::getTableEpochs(), Catalog_Namespace::Catalog::getTableName(), Data_Namespace::GPU_LEVEL, hash_value(), logger::INFO, CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCachesByTable(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::is_vacuumed, LOG, rollback(), Catalog_Namespace::Catalog::setTableEpoch(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::src_dst_cds, and AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::table_epoch.

Referenced by resolveIncompleteAlterColumnCommands().

                               {
  AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo recovery_param;

  recovery_param = deserializeRecoveryInformation(filename);

  CHECK_GT(recovery_param.src_dst_cds.size(), 0UL);

  auto table_id = recovery_param.src_dst_cds.begin()->first.tableId;
  auto table_name_opt = catalog_.getTableName(table_id);
  CHECK(table_name_opt.has_value());
  auto table_name = table_name_opt.value();

  const auto td_with_lock =
      lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
          catalog_, table_name, false);
  const auto td = td_with_lock();

  CHECK(td);
  LOG(INFO) << "Starting crash recovery for table: " << td->tableName;

  auto table_epochs = catalog_.getTableEpochs(catalog_.getDatabaseId(), td->tableId);
  CHECK_GT(table_epochs.size(), 0UL);
  auto current_first_epoch = table_epochs[0].table_epoch;

  // The following semantics apply to the three epochs referenced below.
  //
  // alter_catalog_checkpoint_epoch: This refers to the checkpoint prior to any
  // data modification, only operations that have modified the catalog are
  // expected to have happened.
  //
  // alter_data_checkpoint_epoch: This refers to the checkpoint immediately
  // after data conversion is complete.
  //
  // completed_alter_column_checkpoint_epoch: This refers to the checkpoint
  // after all cleanup operations complete for the alter table command.
  //
  // NOTE: If the table may have deleted but unvacuumed elements, there may be
  // an additional epoch introduced at the start requiring an additional offset.
  int alter_catalog_checkpoint_epoch = recovery_param.table_epoch;

  if (recovery_param.is_vacuumed) {
    alter_catalog_checkpoint_epoch++;
  }
  int alter_data_checkpoint_epoch = alter_catalog_checkpoint_epoch + 1;
  int completed_alter_column_checkpoint_epoch = alter_data_checkpoint_epoch + 1;

  if (current_first_epoch == alter_catalog_checkpoint_epoch) {
    rollback(td, recovery_param);
    // If a vacuum operation was performed prior to alter column, revert the
    // operation.
    if (recovery_param.is_vacuumed) {
      catalog_.setTableEpoch(
          catalog_.getDatabaseId(), td->tableId, recovery_param.table_epoch);
    }
  } else if (current_first_epoch == alter_data_checkpoint_epoch) {
    auto src_dst_cds = getSrcDstCds(table_id, recovery_param.src_dst_cds);
    try {
      cleanup(td, src_dst_cds);
    } catch (std::exception& except) {
      throw std::runtime_error("Alter column recovery error during cleanup: " +
                               std::string(except.what()));
    }
    ChunkKey table_key{catalog_.getCurrentDB().dbId, td->tableId};
    UpdateTriggeredCacheInvalidator::invalidateCachesByTable(
        boost::hash_value(table_key));
    catalog_.getDataMgr().deleteChunksWithPrefix(table_key, MemoryLevel::GPU_LEVEL);
    catalog_.getDataMgr().deleteChunksWithPrefix(table_key, MemoryLevel::CPU_LEVEL);

  } else {
    CHECK_EQ(current_first_epoch, completed_alter_column_checkpoint_epoch);
    // no-op, last checkpoint reached in processing
  }
  LOG(INFO) << "Completed crash recovery for table: " << td->tableName;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string AlterTableAlterColumnCommandRecoveryMgr::recoveryFilepath

(

const RecoveryParamFilepathInfo &

filepath_info

)

Definition at line 233 of file AlterColumnRecovery.cpp.

References AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::base_path, AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::db_name, getRecoveryPrefix(), and AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::table_name.

Referenced by AlterTableAlterColumnCommand::cleanupRecoveryInfo(), getRecoveryFiles(), AlterTableAlterColumnCommand::populateAndWriteRecoveryInfo(), readSerializedRecoveryInformation(), and writeSerializedRecoveryInformation().

                                                    {
  auto prefix = getRecoveryPrefix(filepath_info.base_path);

  if (filepath_info.table_name.empty()) {
    return prefix.string();
  }

  return (prefix /
          std::filesystem::path("alter_column_recovery_db_" + filepath_info.db_name +
                                "_table_" + filepath_info.table_name + ".json"))
      .string();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::resolveIncompleteAlterColumnCommands ( )

private

Definition at line 494 of file AlterColumnRecovery.cpp.

References catalog_, getRecoveryFiles(), logger::INFO, LOG, Catalog_Namespace::Catalog::name(), and recoverAlterTableAlterColumnFromFile().

                                                                                   {
  auto recovery_files = getRecoveryFiles();
  if (recovery_files.empty()) {
    return;
  }

  LOG(INFO) << "Starting crash recovery for tables in catalog: " << catalog_.name();
  for (const auto& filepath : recovery_files) {
    recoverAlterTableAlterColumnFromFile(filepath.string());
    std::filesystem::remove(filepath);
  }
  LOG(INFO) << "Completed crash recovery for tables in catalog: " << catalog_.name();
}

Here is the call graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::resolveIncompleteAlterColumnCommandsForAllCatalogs ( )

static

Definition at line 542 of file AlterColumnRecovery.cpp.

References createRecoveryManagersForCatalogs().

Referenced by startHeavyDBServer().

                                                         {
  auto recovery_mgrs = createRecoveryManagersForCatalogs();

  for (auto& [dbname, recovery_mgr] : recovery_mgrs) {
    recovery_mgr.resolveIncompleteAlterColumnCommands();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::rollback	(	const TableDescriptor *	td,
		const RecoveryInfo &	param
	)

Definition at line 31 of file AlterColumnRecovery.cpp.

References AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::added_columns, Catalog_Namespace::Catalog::alterColumnTypeTransactional(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::altered_columns, catalog_, Data_Namespace::CPU_LEVEL, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::Catalog::delDictionaryTransactional(), Data_Namespace::DataMgr::deleteChunksWithPrefix(), Catalog_Namespace::Catalog::dropColumnTransactional(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getAllDictionariesWithColumnInName(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::Catalog::renameColumn(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::renamed_columns, TableDescriptor::tableId, and AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::updated_dict_cds.

Referenced by AlterTableAlterColumnCommand::alterColumnTypes(), and recoverAlterTableAlterColumnFromFile().

                                                                                  {
  auto cds = catalog_.getAllColumnMetadataForTable(td->tableId, false, false, true);

  // Drop any columns that were added
  for (auto& added_column : param.added_columns) {
    auto cd_it =
        std::find_if(cds.begin(), cds.end(), [&added_column](const ColumnDescriptor* cd) {
          return added_column.columnId == cd->columnId;
        });
    auto cd = *cd_it;
    ChunkKey col_key{catalog_.getCurrentDB().dbId, td->tableId, cd->columnId};
    catalog_.dropColumnTransactional(*td, *cd);
    auto& data_mgr = catalog_.getDataMgr();
    data_mgr.deleteChunksWithPrefix(col_key, MemoryLevel::GPU_LEVEL);
    data_mgr.deleteChunksWithPrefix(col_key, MemoryLevel::CPU_LEVEL);
    cds.erase(cd_it);
  }

  // Rename any columns back to original name
  for (auto& renamed_column : param.renamed_columns) {
    auto cd_it = std::find_if(
        cds.begin(), cds.end(), [&renamed_column](const ColumnDescriptor* cd) {
          return renamed_column.columnId == cd->columnId;
        });
    if (cd_it != cds.end()) {
      auto cd = *cd_it;
      auto old_name = renamed_column.columnName;
      if (cd->columnName != old_name) {
        catalog_.renameColumn(td, cd, old_name);
      }
    }
  }

  // Remove any added dictionary
  for (auto& added_dict : param.updated_dict_cds) {
    auto cd_it =
        std::find_if(cds.begin(), cds.end(), [&added_dict](const ColumnDescriptor* cd) {
          return added_dict.columnId == cd->columnId;
        });
    if (cd_it != cds.end()) {
      auto cd = *cd_it;

      // Find all dictionaries, delete dictionaries which are defunct
      auto dds = catalog_.getAllDictionariesWithColumnInName(cd);
      for (const auto& dd : dds) {
        if (!added_dict.columnType.is_dict_encoded_type() ||
            dd->dictRef.dictId != added_dict.columnType.get_comp_param()) {
          auto temp_cd = *cd;
          temp_cd.columnType.set_comp_param(dd->dictRef.dictId);
          temp_cd.columnType.setStringDictKey(
              {catalog_.getDatabaseId(), dd->dictRef.dictId});
          catalog_.delDictionaryTransactional(temp_cd);
        }
      }
    }
  }

  // Undo any altered column
  for (auto& altered_column : param.altered_columns) {
    auto cd_it = std::find_if(
        cds.begin(), cds.end(), [&altered_column](const ColumnDescriptor* cd) {
          return altered_column.new_cd.columnId == cd->columnId;
        });
    if (cd_it != cds.end()) {
      catalog_.alterColumnTypeTransactional(altered_column.old_cd);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string AlterTableAlterColumnCommandRecoveryMgr::serializeRecoveryInformation

(

const RecoveryInfo &

param

)

Definition at line 271 of file AlterColumnRecovery.cpp.

References json_utils::add_value_to_object(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::added_columns, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::altered_columns, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::is_vacuumed, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::renamed_columns, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::src_dst_cds, AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::table_epoch, toPairedCds(), AlterTableAlterColumnCommandRecoveryMgr::RecoveryInfo::updated_dict_cds, and json_utils::write_to_string().

Referenced by writeSerializedRecoveryInformation().

                               {
  rapidjson::Document d;
  d.SetObject();

  json_utils::add_value_to_object(
      d, param.added_columns, "added_columns", d.GetAllocator());
  json_utils::add_value_to_object(
      d, toPairedCds(param.altered_columns), "altered_columns", d.GetAllocator());
  json_utils::add_value_to_object(
      d, param.renamed_columns, "renamed_columns", d.GetAllocator());
  json_utils::add_value_to_object(
      d, param.updated_dict_cds, "updated_dict_cds", d.GetAllocator());
  json_utils::add_value_to_object(d, param.table_epoch, "table_epoch", d.GetAllocator());
  json_utils::add_value_to_object(d, param.is_vacuumed, "is_vacuumed", d.GetAllocator());

  json_utils::add_value_to_object(d, param.src_dst_cds, "src_dst_cds", d.GetAllocator());

  return json_utils::write_to_string(d);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::list< std::pair< ColumnDescriptor, ColumnDescriptor > > AlterTableAlterColumnCommandRecoveryMgr::toPairedCds ( const std::list< ColumnAltered > &  altered_columns )

private

Definition at line 194 of file AlterColumnRecovery.cpp.

Referenced by serializeRecoveryInformation().

                                                   {
  std::list<std::pair<ColumnDescriptor, ColumnDescriptor>> retval;
  for (const auto& [old_cd, new_cd] : altered_columns) {
    retval.emplace_back(old_cd, new_cd);
  }
  return retval;
}

Here is the caller graph for this function:

void AlterTableAlterColumnCommandRecoveryMgr::writeSerializedRecoveryInformation	(	const RecoveryInfo &	param,
		const RecoveryParamFilepathInfo &	filepath_info
	)

Definition at line 292 of file AlterColumnRecovery.cpp.

References AlterTableAlterColumnCommandRecoveryMgr::RecoveryParamFilepathInfo::base_path, nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), getRecoveryPrefix(), recoveryFilepath(), and serializeRecoveryInformation().

Referenced by AlterTableAlterColumnCommand::populateAndWriteRecoveryInfo().

                                                    {
  auto filename = recoveryFilepath(filepath_info);

  // Create crash recovery directory if non-existent
  auto prefix = getRecoveryPrefix(filepath_info.base_path).string();
  if (!std::filesystem::exists(prefix)) {
    if (!std::filesystem::create_directory(prefix)) {
      throw std::runtime_error{"Error trying to create crash recovery directory \"" +
                               prefix + "\". The error was: " + std::strerror(errno)};
    }
  }

  // Use a temporary file name to indicate file has not been written yet
  std::ofstream ofs(filename + ".tmp");
  if (!ofs) {
    throw std::runtime_error{"Error trying to create file \"" + filename +
                             "\". The error was: " + std::strerror(errno)};
  }
  ofs << serializeRecoveryInformation(param);
  // Rename to target filename to indicate file was successfully written
  std::filesystem::rename(filename + ".tmp", filename);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Catalog_Namespace::Catalog& AlterTableAlterColumnCommandRecoveryMgr::catalog_

private

Definition at line 112 of file AlterColumnRecovery.h.

Referenced by checkpoint(), cleanup(), cleanupClearChunk(), cleanupClearRemainingChunks(), cleanupDeleteDictionaries(), cleanupDropSourceGeoColumns(), getRecoveryFilepathInfo(), getRecoveryFiles(), getSrcDstCds(), recoverAlterTableAlterColumnFromFile(), resolveIncompleteAlterColumnCommands(), and rollback().

const std::string AlterTableAlterColumnCommandRecoveryMgr::kRecoveryDirectoryName = "crash_recovery"

inlinestatic

Definition at line 76 of file AlterColumnRecovery.h.

Referenced by getRecoveryPrefix().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Catalog/AlterColumnRecovery.h
• /home/jenkins-slave/workspace/core-os-doxygen/Catalog/AlterColumnRecovery.cpp