FileBuffer.h

Go to the documentation of this file.

/*
 * Copyright 2022 HEAVY.AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include "DataMgr/AbstractBuffer.h"
#include "DataMgr/FileMgr/Page.h"

#include <iostream>
#include <stdexcept>

#include "Logger/Logger.h"

using namespace Data_Namespace;

#define NUM_METADATA 10
#define METADATA_VERSION 0

namespace File_Namespace {

// forward declarations
class FileMgr;
class CachingFileMgr;

class FileBuffer : public AbstractBuffer {
  friend class FileMgr;
  friend class CachingFileMgr;

 public:
  FileBuffer(FileMgr* fm,
             const size_t pageSize,
             const ChunkKey& chunkKey,
             const size_t initialSize = 0);

  FileBuffer(FileMgr* fm,
             const size_t pageSize,
             const ChunkKey& chunkKey,
             const SQLTypeInfo sqlType,
             const size_t initialSize = 0);

  FileBuffer(FileMgr* fm,
             /* const size_t pageSize,*/ const ChunkKey& chunkKey,
             const std::vector<HeaderInfo>::const_iterator& headerStartIt,
             const std::vector<HeaderInfo>::const_iterator& headerEndIt);

  ~FileBuffer() override;

  Page addNewMultiPage(const int32_t epoch);

  void reserve(const size_t numBytes) override;

  size_t freeMetadataPages();
  size_t freeChunkPages();
  size_t freePages();
  void freePagesBeforeEpoch(const int32_t targetEpoch);

  void read(int8_t* const dst,
            const size_t numBytes = 0,
            const size_t offset = 0,
            const MemoryLevel dstMemoryLevel = CPU_LEVEL,
            const int32_t deviceId = -1) override;

  void write(int8_t* src,
             const size_t numBytes,
             const size_t offset = 0,
             const MemoryLevel srcMemoryLevel = CPU_LEVEL,
             const int32_t deviceId = -1) override;

  void append(int8_t* src,
              const size_t numBytes,
              const MemoryLevel srcMemoryLevel = CPU_LEVEL,
              const int32_t deviceId = -1) override;
  void copyPage(Page& srcPage,
                Page& destPage,
                const size_t numBytes,
                const size_t offset = 0);
  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }

  int8_t* getMemoryPtr() override {
    LOG(FATAL) << "Operation not supported.";
    return nullptr;  // satisfy return-type warning
  }

  inline size_t pageCount() const override { return multiPages_.size(); }

  inline bool hasDataPages() const { return pageCount() > 0; }

  inline size_t pageSize() const override { return pageSize_; }

  inline virtual size_t pageDataSize() const { return pageDataSize_; }

  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }

  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
  inline MultiPage getMetadataPage() const { return metadataPages_; }

  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }

  // inline virtual size_t used() const {

  inline size_t numMetadataPages() const { return metadataPages_.pageVersions.size(); };

  bool isMissingPages() const;
  size_t numChunkPages() const;
  std::string dump() const;

  static size_t getMinPageSize();

  // Used for testing
  void freePage(const Page& page);

  static constexpr size_t kHeaderBufferOffset{32};

 private:
  // FileBuffer(const FileBuffer&);      // private copy constructor
  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator

  // headerSize(numBytes), ChunkKey, pageId, version epoch
  // void writeHeader(Page &page, const int32_t pageId, const int32_t epoch, const bool
  // writeSize = false);
  void writeHeader(Page& page,
                   const int32_t pageId,
                   const int32_t epoch,
                   const bool writeMetadata = false);
  void writeMetadata(const int32_t epoch);
  void readMetadata(const Page& page);
  void setBufferHeaderSize();

  void freePage(const Page& page, const bool isRolloff);
  void freePagesBeforeEpochForMultiPage(MultiPage& multiPage,
                                        const int32_t targetEpoch,
                                        const int32_t currentEpoch);
  void initMetadataAndPageDataSize();
  int32_t getFileMgrEpoch();

  FileMgr* fm_;  // a reference to FileMgr is needed for writing to new pages in available
                 // files
  // pageSize_ is non-const because it can be read from a metadata file to create a
  // non-standard page size. metadataPageSize_ is const because we need to know what the
  // metadata page size is in order to know which files are metadata files (and therefore
  // determine page size).  This is set earlier in FileMgr construction.
  const size_t metadataPageSize_;
  MultiPage metadataPages_;
  std::vector<MultiPage> multiPages_;
  size_t pageSize_;
  size_t pageDataSize_;
  size_t reservedHeaderSize_;  // lets make this a constant now for simplicity - 128 bytes
  ChunkKey chunkKey_;
};

}  // namespace File_Namespace