StringDictionaryProxy.h

Go to the documentation of this file.

/*
 * Copyright 2022 HEAVY.AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef STRINGDICTIONARY_STRINGDICTIONARYPROXY_H
#define STRINGDICTIONARY_STRINGDICTIONARYPROXY_H

#include "Logger/Logger.h"  // For CHECK macros
#include "Shared/misc.h"
#include "StringDictionary.h"

#include "ThirdParty/robin_hood/robin_hood.h"

#include <optional>
#include <ostream>
#include <shared_mutex>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>

namespace StringOps_Namespace {
struct StringOpInfo;
}

// used to access a StringDictionary when transient strings are involved
class StringDictionaryProxy {
 public:
  StringDictionaryProxy(StringDictionaryProxy const&) = delete;
  StringDictionaryProxy const& operator=(StringDictionaryProxy const&) = delete;
  StringDictionaryProxy(std::shared_ptr<StringDictionary> sd,
                        const shared::StringDictKey& string_dict_key,
                        const int64_t generation);

  const shared::StringDictKey& getDictKey() const noexcept { return string_dict_key_; };

  bool operator==(StringDictionaryProxy const&) const;
  bool operator!=(StringDictionaryProxy const&) const;

  int32_t getOrAdd(const std::string& str) noexcept;
  StringDictionary* getDictionary() const noexcept;
  int64_t getGeneration() const noexcept;

  std::vector<int32_t> getTransientBulk(const std::vector<std::string>& strings) const;
  int32_t getOrAddTransient(const std::string&);
  int32_t getOrAddTransient(const std::string_view);
  // Not currently used
  std::vector<int32_t> getOrAddTransientBulk(const std::vector<std::string>& strings);
  int32_t getIdOfString(const std::string& str) const;
  int32_t getIdOfStringNoGeneration(
      const std::string& str) const;  // disregard generation, only used by QueryRenderer
  std::string getString(int32_t string_id) const;
  std::vector<std::string> getStrings(const std::vector<int32_t>& string_ids) const;
  std::pair<const char*, size_t> getStringBytes(int32_t string_id) const noexcept;

  template <typename T>
  class TranslationMap {
    size_t const offset_;
    std::vector<T> vector_map_;
    int64_t num_untranslated_strings_{-1};
    T range_start_{0};
    T range_end_{0};

   public:
    // +1 is added to skip string_id=-1 reserved for INVALID_STR_ID. id_map[-1]==-1.
    TranslationMap(uint32_t const tran_size, uint32_t const dict_size)
        : offset_(tran_size + 1), vector_map_(offset_ + dict_size) {}
    TranslationMap(uint32_t const tran_size, uint32_t const dict_size, const T& init_val)
        : offset_(tran_size + 1), vector_map_(offset_ + dict_size, init_val) {}
    TranslationMap(TranslationMap const&) = delete;
    TranslationMap(TranslationMap&&) = default;
    bool empty() const { return vector_map_.size() == 1; }
    inline size_t getIndex(int32_t const id) const { return offset_ + id; }
    std::vector<T> const& getVectorMap() const { return vector_map_; }
    size_t size() const { return vector_map_.size(); }
    size_t numTransients() const { return offset_ - 1; }
    size_t numNonTransients() const { return vector_map_.size() - offset_; }
    T* data() { return vector_map_.data(); }
    T const* data() const { return vector_map_.data(); }
    int32_t domainStart() const { return -static_cast<int32_t>(offset_); }
    int32_t domainEnd() const { return static_cast<int32_t>(numNonTransients()); }
    void setRangeStart(const int32_t range_start) { range_start_ = range_start; }
    void setRangeEnd(const int32_t range_end) { range_end_ = range_end; }
    T rangeStart() const { return range_start_; }
    T rangeEnd() const { return range_end_; }

    // Next two methods are currently used by buildUnionTranslationMapToOtherProxy to
    // short circuit iteration over ids after intersection translation if all
    // ids translated. Currently the private num_untranslated_strings_ is initialized
    // to a -1 sentinel to signify that the value has not been calculated, which we
    // CHECK against in the getter numUntranslatedStrings() method
    // to represent that the num_untranslated_strings_ field has been uninitialized
    size_t numUntranslatedStrings() const {
      CHECK_GE(num_untranslated_strings_, 0L);
      return static_cast<size_t>(num_untranslated_strings_);
    }
    void setNumUntranslatedStrings(const size_t num_untranslated_strings) {
      num_untranslated_strings_ = static_cast<int64_t>(num_untranslated_strings);
    }
    T* storageData() { return vector_map_.data() + offset_; }
    T& operator[](int32_t const id) { return vector_map_[getIndex(id)]; }
    T operator[](int32_t const id) const { return vector_map_[getIndex(id)]; }
    friend std::ostream& operator<<(std::ostream& os, TranslationMap<T> const& sdp_map) {
      return os << "IdMap(offset_(" << sdp_map.offset_ << ") vector_map_"
                << shared::printContainer(sdp_map.vector_map_) << ')';
    }
  };

  using IdMap = TranslationMap<int32_t>;

  IdMap initIdMap() const {
    return IdMap(
        transient_string_vec_.size(), generation_, StringDictionary::INVALID_STR_ID);
  }

  TranslationMap<Datum> buildNumericTranslationMap(
      const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) const;

  IdMap buildIntersectionTranslationMapToOtherProxy(
      const StringDictionaryProxy* dest_proxy,
      const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) const;

  IdMap buildUnionTranslationMapToOtherProxy(
      StringDictionaryProxy* dest_proxy,
      const std::vector<StringOps_Namespace::StringOpInfo>& string_op_types) const;

  size_t storageEntryCount() const;

  size_t transientEntryCount() const;

  size_t entryCount() const;

  void updateGeneration(const int64_t generation) noexcept;

  template <typename T>
  std::vector<T> getLike(const std::string& pattern,
                         const bool icase,
                         const bool is_simple,
                         const char escape) const;

  std::vector<int32_t> getCompare(const std::string& pattern,
                                  const std::string& comp_operator) const;

  std::vector<int32_t> getRegexpLike(const std::string& pattern, const char escape) const;

  struct HeterogeneousStringHash {
    using is_transparent = void;  // Used by robin_hood to activate heterogenous hashing
    // std::string and char const* are implicitly cast to std::string_view.
    size_t operator()(std::string_view const key) const {
      return robin_hood::hash_bytes(key.data(), key.size());
    }
  };
  struct HeterogeneousStringEqual {
    using is_transparent = void;  // Used by robin_hood to activate heterogenous equal
    // std::string and char const* are implicitly cast to std::string_view.
    bool operator()(std::string_view const lhs, std::string_view const rhs) const {
      return lhs == rhs;
    }
  };

  // The std::string must live in the map, and std::string const* in the vector. As
  // desirable as it might be to have it the other way, string addresses won't change
  // in the robin_hood::unordered_node_map when new strings are added, but may change
  // in a std::vector (and robin_hood::unordered_flat_map).
  using TransientMap = robin_hood::unordered_node_map<std::string,
                                                      int32_t,
                                                      HeterogeneousStringHash,
                                                      HeterogeneousStringEqual>;

  const std::vector<std::string const*>& getTransientVector() const {
    return transient_string_vec_;
  }

  // INVALID_STR_ID = -1 is reserved for invalid string_ids.
  // Thus the greatest valid transient string_id is -2.
  static unsigned transientIdToIndex(int32_t const id) {
    constexpr int max_transient_string_id = -2;
    return static_cast<unsigned>(max_transient_string_id - id);
  }

  static int32_t transientIndexToId(unsigned const index) {
    constexpr int max_transient_string_id = -2;
    return static_cast<int32_t>(max_transient_string_id - index);
  }

  // Iterate over transient strings, then non-transients.
  void eachStringSerially(StringDictionary::StringCallback&) const;

  // Union strings from both StringDictionaryProxies into *this as transients.
  // Return map of old string_ids to new string_ids.
  IdMap transientUnion(StringDictionaryProxy const&);

 private:
  std::string getStringUnlocked(const int32_t string_id) const;
  size_t transientEntryCountUnlocked() const;
  size_t entryCountUnlocked() const;
  size_t persistedC() const;
  template <typename String>
  int32_t getOrAddTransientImpl(String);
  template <typename String>
  int32_t lookupTransientStringUnlocked(const String& lookup_string) const;
  size_t getTransientBulkImpl(const std::vector<std::string>& strings,
                              int32_t* string_ids,
                              const bool take_read_lock) const;
  template <typename String>
  size_t transientLookupBulk(const std::vector<String>& lookup_strings,
                             int32_t* string_ids,
                             const bool take_read_lock) const;
  template <typename String>
  size_t transientLookupBulkUnlocked(const std::vector<String>& lookup_strings,
                                     int32_t* string_ids) const;
  template <typename String>
  size_t transientLookupBulkParallelUnlocked(const std::vector<String>& lookup_strings,
                                             int32_t* string_ids) const;

  IdMap buildIntersectionTranslationMapToOtherProxyUnlocked(
      const StringDictionaryProxy* dest_proxy,
      const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) const;

  std::shared_ptr<StringDictionary> string_dict_;
  const shared::StringDictKey string_dict_key_;
  TransientMap transient_str_to_int_;
  // Holds pointers into transient_str_to_int_
  std::vector<std::string const*> transient_string_vec_;
  int64_t generation_;
  mutable std::shared_mutex rw_mutex_;

  // Return INVALID_STR_ID if not found on string_dict_. Don't lock or check transients.
  template <typename String>
  int32_t getIdOfStringFromClient(String const&) const;
  template <typename String>
  int32_t getOrAddTransientUnlocked(String const&);

  friend class StringLocalCallback;
  friend class StringNetworkCallback;
};
#endif  // STRINGDICTIONARY_STRINGDICTIONARYPROXY_H