Fragmenter.h

Go to the documentation of this file.

/*
 * Copyright 2022 HEAVY.AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef FRAGMENTER_H
#define FRAGMENTER_H

#include <deque>
#include <list>
#include <map>
#include <mutex>
#include "../Catalog/ColumnDescriptor.h"
#include "../DataMgr/Chunk/Chunk.h"
#include "../DataMgr/ChunkMetadata.h"
#include "../Shared/heavyai_shared_mutex.h"
#include "../Shared/types.h"

namespace Data_Namespace {
class AbstractBuffer;
}

class ResultSet;

namespace Fragmenter_Namespace {
class InsertOrderFragmenter;

enum FragmenterType {
  INSERT_ORDER = 0  // these values persist in catalog.  make explicit
};

struct InsertChunks {
  const int table_id;
  const int db_id;
  std::map</*column_id=*/int, std::shared_ptr<Chunk_NS::Chunk> > chunks;
  std::vector<size_t> valid_row_indices; /* specifies which row indices in chunk are valid
                                            for insertion */
};

struct InsertData {
  int databaseId;  
  int tableId;     
  std::vector<int> columnIds;  
  size_t numRows;              
  std::vector<DataBlockPtr> data;  
  std::vector<bool> is_default;    
};

class FragmentInfo {
 public:
  FragmentInfo()
      : fragmentId(-1)
      , shadowNumTuples(0)
      , physicalTableId(-1)
      , shard(-1)
      , resultSet(nullptr)
      , numTuples(0)
      , synthesizedNumTuplesIsValid(false)
      , synthesizedMetadataIsValid(false) {}

  void setChunkMetadataMap(const ChunkMetadataMap& chunk_metadata_map) {
    this->chunkMetadataMap = chunk_metadata_map;
  }

  void setChunkMetadata(const int col, std::shared_ptr<ChunkMetadata> chunkMetadata) {
    chunkMetadataMap[col] = chunkMetadata;
  }

  const ChunkMetadataMap& getChunkMetadataMap() const;

  const ChunkMetadataMap& getChunkMetadataMapPhysical() const { return chunkMetadataMap; }

  ChunkMetadataMap getChunkMetadataMapPhysicalCopy() const;

  size_t getNumTuples() const;

  size_t getPhysicalNumTuples() const { return numTuples; }

  bool isEmptyPhysicalFragment() const { return physicalTableId >= 0 && !numTuples; }

  void setPhysicalNumTuples(const size_t physNumTuples) { numTuples = physNumTuples; }

  void invalidateChunkMetadataMap() const { synthesizedMetadataIsValid = false; };
  void invalidateNumTuples() const { synthesizedNumTuplesIsValid = false; }

  // for unit tests
  static void setUnconditionalVacuum(const double unconditionalVacuum) {
    unconditionalVacuum_ = unconditionalVacuum;
  }

  int fragmentId;
  size_t shadowNumTuples;
  std::vector<int> deviceIds;
  int physicalTableId;
  int shard;
  ChunkMetadataMap shadowChunkMetadataMap;
  mutable ResultSet* resultSet;
  mutable std::shared_ptr<std::mutex> resultSetMutex;

 private:
  mutable size_t numTuples;
  mutable ChunkMetadataMap chunkMetadataMap;
  mutable bool synthesizedNumTuplesIsValid;
  mutable bool synthesizedMetadataIsValid;

  friend class InsertOrderFragmenter;
  static bool unconditionalVacuum_;
};

class TableInfo {
 public:
  TableInfo() : numTuples(0) {}

  size_t getNumTuples() const;

  size_t getNumTuplesUpperBound() const;

  size_t getPhysicalNumTuples() const { return numTuples; }

  void setPhysicalNumTuples(const size_t physNumTuples) { numTuples = physNumTuples; }

  size_t getFragmentNumTuplesUpperBound() const;

  std::vector<int> chunkKeyPrefix;
  std::vector<FragmentInfo> fragments;

 private:
  mutable size_t numTuples;
};

}  // namespace Fragmenter_Namespace

#endif  // FRAGMENTER_H