ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE > Struct Template Reference

Collaboration diagram for ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >:

Public Types
using	BufferIteratorType = BUFFER_ITERATOR_TYPE

Public Member Functions
	ResultSetComparator (const std::list< Analyzer::OrderEntry > &order_entries, const ResultSet *result_set, const PermutationView permutation, const Executor *executor, const bool single_threaded)
void	materializeCountDistinctColumns ()
ApproxQuantileBuffers	materializeApproxQuantileColumns () const
ModeBuffers	materializeModeColumns () const
std::vector< int64_t >	materializeCountDistinctColumn (const Analyzer::OrderEntry &order_entry) const
ApproxQuantileBuffers::value_type	materializeApproxQuantileColumn (const Analyzer::OrderEntry &order_entry) const
ModeBuffers::value_type	materializeModeColumn (const Analyzer::OrderEntry &order_entry) const
bool	operator() (const PermutationIdx lhs, const PermutationIdx rhs) const

Public Attributes
const std::list < Analyzer::OrderEntry > &	order_entries_
const ResultSet *	result_set_
const PermutationView	permutation_
const BufferIteratorType	buffer_itr_
const Executor *	executor_
const bool	single_threaded_
std::vector< std::vector < int64_t > >	count_distinct_materialized_buffers_
const ApproxQuantileBuffers	approx_quantile_materialized_buffers_
const ModeBuffers	mode_buffers_

Detailed Description

template<typename BUFFER_ITERATOR_TYPE>
struct ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >

Definition at line 833 of file ResultSet.h.

Member Typedef Documentation

template<typename BUFFER_ITERATOR_TYPE>

using ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::BufferIteratorType = BUFFER_ITERATOR_TYPE

Definition at line 834 of file ResultSet.h.

Constructor & Destructor Documentation

template<typename BUFFER_ITERATOR_TYPE>

ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::ResultSetComparator ( const std::list< Analyzer::OrderEntry > &  order_entries, const ResultSet *  result_set, const PermutationView  permutation, const Executor *  executor, const bool  single_threaded  )

inline

Definition at line 836 of file ResultSet.h.

References ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumns().

        : order_entries_(order_entries)
        , result_set_(result_set)
        , permutation_(permutation)
        , buffer_itr_(result_set)
        , executor_(executor)
        , single_threaded_(single_threaded)
        , approx_quantile_materialized_buffers_(materializeApproxQuantileColumns())
        , mode_buffers_(materializeModeColumns()) {
      materializeCountDistinctColumns();
    }

Here is the call graph for this function:

Member Function Documentation

template<typename BUFFER_ITERATOR_TYPE >

ResultSet::ApproxQuantileBuffers::value_type ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxQuantileColumn

(

const Analyzer::OrderEntry &

order_entry

)

const

Definition at line 1057 of file ResultSet.cpp.

References cpu_threads(), NULL_DOUBLE, threading_std::task_group::run(), logger::thread_local_ids(), Analyzer::OrderEntry::tle_no, and threading_std::task_group::wait().

                                                 {
  ResultSet::ApproxQuantileBuffers::value_type materialized_buffer(
      result_set_->query_mem_desc_.getEntryCount());
  const size_t size = permutation_.size();
  const auto work = [&, parent_thread_local_ids = logger::thread_local_ids()](
                        const size_t start, const size_t end) {
    logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
    for (size_t i = start; i < end; ++i) {
      const PermutationIdx permuted_idx = permutation_[i];
      const auto storage_lookup_result = result_set_->findStorage(permuted_idx);
      const auto storage = storage_lookup_result.storage_ptr;
      const auto off = storage_lookup_result.fixedup_entry_idx;
      const auto value = buffer_itr_.getColumnInternal(
          storage->buff_, off, order_entry.tle_no - 1, storage_lookup_result);
      materialized_buffer[permuted_idx] =
          value.i1 ? calculateQuantile(reinterpret_cast<quantile::TDigest*>(value.i1))
                   : NULL_DOUBLE;
    }
  };
  if (single_threaded_) {
    work(0, size);
  } else {
    threading::task_group thread_pool;
    for (auto interval : makeIntervals<size_t>(0, size, cpu_threads())) {
      thread_pool.run([=] { work(interval.begin, interval.end); });
    }
    thread_pool.wait();
  }
  return materialized_buffer;
}

Here is the call graph for this function:

template<typename BUFFER_ITERATOR_TYPE >

ResultSet::ApproxQuantileBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxQuantileColumns

(

)

const

Definition at line 983 of file ResultSet.cpp.

References kAPPROX_QUANTILE.

                                                                  {
  ResultSet::ApproxQuantileBuffers approx_quantile_materialized_buffers;
  for (const auto& order_entry : order_entries_) {
    if (result_set_->targets_[order_entry.tle_no - 1].agg_kind == kAPPROX_QUANTILE) {
      approx_quantile_materialized_buffers.emplace_back(
          materializeApproxQuantileColumn(order_entry));
    }
  }
  return approx_quantile_materialized_buffers;
}

template<typename BUFFER_ITERATOR_TYPE >

std::vector< int64_t > ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumn

(

const Analyzer::OrderEntry &

order_entry

)

const

Definition at line 1011 of file ResultSet.cpp.

References count_distinct_set_size(), cpu_threads(), threading_std::task_group::run(), logger::thread_local_ids(), Analyzer::OrderEntry::tle_no, and threading_std::task_group::wait().

                                                 {
  const size_t num_storage_entries = result_set_->query_mem_desc_.getEntryCount();
  std::vector<int64_t> count_distinct_materialized_buffer(num_storage_entries);
  const CountDistinctDescriptor count_distinct_descriptor =
      result_set_->query_mem_desc_.getCountDistinctDescriptor(order_entry.tle_no - 1);
  const size_t num_non_empty_entries = permutation_.size();

  const auto work = [&, parent_thread_local_ids = logger::thread_local_ids()](
                        const size_t start, const size_t end) {
    logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
    for (size_t i = start; i < end; ++i) {
      const PermutationIdx permuted_idx = permutation_[i];
      const auto storage_lookup_result = result_set_->findStorage(permuted_idx);
      const auto storage = storage_lookup_result.storage_ptr;
      const auto off = storage_lookup_result.fixedup_entry_idx;
      const auto value = buffer_itr_.getColumnInternal(
          storage->buff_, off, order_entry.tle_no - 1, storage_lookup_result);
      count_distinct_materialized_buffer[permuted_idx] =
          count_distinct_set_size(value.i1, count_distinct_descriptor);
    }
  };
  // TODO(tlm): Allow use of tbb after we determine how to easily encapsulate the choice
  // between thread pool types
  if (single_threaded_) {
    work(0, num_non_empty_entries);
  } else {
    threading::task_group thread_pool;
    for (auto interval : makeIntervals<size_t>(0, num_non_empty_entries, cpu_threads())) {
      thread_pool.run([=] { work(interval.begin, interval.end); });
    }
    thread_pool.wait();
  }
  return count_distinct_materialized_buffer;
}

Here is the call graph for this function:

template<typename BUFFER_ITERATOR_TYPE >

void ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumns

(

)

Definition at line 960 of file ResultSet.cpp.

References is_distinct_target().

Referenced by ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::ResultSetComparator().

                                                           {
  for (const auto& order_entry : order_entries_) {
    if (is_distinct_target(result_set_->targets_[order_entry.tle_no - 1])) {
      count_distinct_materialized_buffers_.emplace_back(
          materializeCountDistinctColumn(order_entry));
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename BUFFER_ITERATOR_TYPE >

ResultSet::ModeBuffers::value_type ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeModeColumn

(

const Analyzer::OrderEntry &

order_entry

)

const

Definition at line 1126 of file ResultSet.cpp.

References threading_serial::parallel_for(), and logger::thread_local_ids().

                                                 {
  ResultSet::ModeBuffers::value_type materialized_buffer(
      result_set_->query_mem_desc_.getEntryCount());
  ModeScatter mode_scatter{
      logger::thread_local_ids(), this, order_entry, materialized_buffer};
  if (single_threaded_) {
    mode_scatter(ModeBlockedRange(0, permutation_.size()));  // Still has new thread_id.
  } else {
    tbb::parallel_for(ModeBlockedRange(0, permutation_.size()), mode_scatter);
  }
  return materialized_buffer;
}

Here is the call graph for this function:

template<typename BUFFER_ITERATOR_TYPE >

ResultSet::ModeBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeModeColumns

(

)

const

Definition at line 996 of file ResultSet.cpp.

References kMODE.

                                                                                 {
  ResultSet::ModeBuffers mode_buffers;
  IsAggKind const is_mode(result_set_->targets_, kMODE);
  mode_buffers.reserve(
      std::count_if(order_entries_.begin(), order_entries_.end(), is_mode));
  for (auto const& order_entry : order_entries_) {
    if (is_mode(order_entry)) {
      mode_buffers.emplace_back(materializeModeColumn(order_entry));
    }
  }
  return mode_buffers;
}

template<typename BUFFER_ITERATOR_TYPE >

bool ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::operator()	(	const PermutationIdx	lhs,
		const PermutationIdx	rhs
	)		const

Definition at line 1141 of file ResultSet.cpp.

References CHECK, CHECK_EQ, CHECK_GE, CHECK_LT, executor_(), get_compact_type(), is_distinct_target(), kAPPROX_QUANTILE, kENCODING_DICT, kFLOAT, kMODE, LIKELY, NULL_BIGINT, NULL_DOUBLE, pair_to_double(), takes_float_argument(), and UNLIKELY.

                                    {
  // NB: The compare function must define a strict weak ordering, otherwise
  // std::sort will trigger a segmentation fault (or corrupt memory).
  const auto lhs_storage_lookup_result = result_set_->findStorage(lhs);
  const auto rhs_storage_lookup_result = result_set_->findStorage(rhs);
  const auto lhs_storage = lhs_storage_lookup_result.storage_ptr;
  const auto rhs_storage = rhs_storage_lookup_result.storage_ptr;
  const auto fixedup_lhs = lhs_storage_lookup_result.fixedup_entry_idx;
  const auto fixedup_rhs = rhs_storage_lookup_result.fixedup_entry_idx;
  size_t materialized_count_distinct_buffer_idx{0};
  size_t materialized_approx_quantile_buffer_idx{0};
  size_t materialized_mode_buffer_idx{0};

  for (const auto& order_entry : order_entries_) {
    CHECK_GE(order_entry.tle_no, 1);
    // lhs_entry_ti and rhs_entry_ti can differ on comp_param w/ UNION of string dicts.
    const auto& lhs_agg_info = lhs_storage->targets_[order_entry.tle_no - 1];
    const auto& rhs_agg_info = rhs_storage->targets_[order_entry.tle_no - 1];
    const auto lhs_entry_ti = get_compact_type(lhs_agg_info);
    const auto rhs_entry_ti = get_compact_type(rhs_agg_info);
    // When lhs vs rhs doesn't matter, the lhs is used. For example:
    bool float_argument_input = takes_float_argument(lhs_agg_info);
    // Need to determine if the float value has been stored as float
    // or if it has been compacted to a different (often larger 8 bytes)
    // in distributed case the floats are actually 4 bytes
    // TODO the above takes_float_argument() is widely used wonder if this problem
    // exists elsewhere
    if (lhs_entry_ti.get_type() == kFLOAT) {
      const auto is_col_lazy =
          !result_set_->lazy_fetch_info_.empty() &&
          result_set_->lazy_fetch_info_[order_entry.tle_no - 1].is_lazily_fetched;
      if (result_set_->query_mem_desc_.getPaddedSlotWidthBytes(order_entry.tle_no - 1) ==
          sizeof(float)) {
        float_argument_input =
            result_set_->query_mem_desc_.didOutputColumnar() ? !is_col_lazy : true;
      }
    }

    if (UNLIKELY(is_distinct_target(lhs_agg_info))) {
      CHECK_LT(materialized_count_distinct_buffer_idx,
               count_distinct_materialized_buffers_.size());

      const auto& count_distinct_materialized_buffer =
          count_distinct_materialized_buffers_[materialized_count_distinct_buffer_idx];
      const auto lhs_sz = count_distinct_materialized_buffer[lhs];
      const auto rhs_sz = count_distinct_materialized_buffer[rhs];
      ++materialized_count_distinct_buffer_idx;
      if (lhs_sz == rhs_sz) {
        continue;
      }
      return (lhs_sz < rhs_sz) != order_entry.is_desc;
    } else if (UNLIKELY(lhs_agg_info.agg_kind == kAPPROX_QUANTILE)) {
      CHECK_LT(materialized_approx_quantile_buffer_idx,
               approx_quantile_materialized_buffers_.size());
      const auto& approx_quantile_materialized_buffer =
          approx_quantile_materialized_buffers_[materialized_approx_quantile_buffer_idx];
      const auto lhs_value = approx_quantile_materialized_buffer[lhs];
      const auto rhs_value = approx_quantile_materialized_buffer[rhs];
      ++materialized_approx_quantile_buffer_idx;
      if (lhs_value == rhs_value) {
        continue;
      } else if (!lhs_entry_ti.get_notnull()) {
        if (lhs_value == NULL_DOUBLE) {
          return order_entry.nulls_first;
        } else if (rhs_value == NULL_DOUBLE) {
          return !order_entry.nulls_first;
        }
      }
      return (lhs_value < rhs_value) != order_entry.is_desc;
    } else if (UNLIKELY(lhs_agg_info.agg_kind == kMODE)) {
      CHECK_LT(materialized_mode_buffer_idx, mode_buffers_.size());
      auto const& mode_buffer = mode_buffers_[materialized_mode_buffer_idx++];
      int64_t const lhs_value = mode_buffer[lhs];
      int64_t const rhs_value = mode_buffer[rhs];
      if (lhs_value == rhs_value) {
        continue;
        // MODE(x) can only be NULL when the group is empty, since it skips null values.
      } else if (lhs_value == NULL_BIGINT) {  // NULL_BIGINT from materializeMode()
        return order_entry.nulls_first;
      } else if (rhs_value == NULL_BIGINT) {
        return !order_entry.nulls_first;
      } else {
        return result_set_->isLessThan(lhs_entry_ti, lhs_value, rhs_value) !=
               order_entry.is_desc;
      }
    }

    const auto lhs_v = buffer_itr_.getColumnInternal(lhs_storage->buff_,
                                                     fixedup_lhs,
                                                     order_entry.tle_no - 1,
                                                     lhs_storage_lookup_result);
    const auto rhs_v = buffer_itr_.getColumnInternal(rhs_storage->buff_,
                                                     fixedup_rhs,
                                                     order_entry.tle_no - 1,
                                                     rhs_storage_lookup_result);

    if (UNLIKELY(isNull(lhs_entry_ti, lhs_v, float_argument_input) &&
                 isNull(rhs_entry_ti, rhs_v, float_argument_input))) {
      continue;
    }
    if (UNLIKELY(isNull(lhs_entry_ti, lhs_v, float_argument_input) &&
                 !isNull(rhs_entry_ti, rhs_v, float_argument_input))) {
      return order_entry.nulls_first;
    }
    if (UNLIKELY(isNull(rhs_entry_ti, rhs_v, float_argument_input) &&
                 !isNull(lhs_entry_ti, lhs_v, float_argument_input))) {
      return !order_entry.nulls_first;
    }

    if (LIKELY(lhs_v.isInt())) {
      CHECK(rhs_v.isInt());
      if (UNLIKELY(lhs_entry_ti.is_string() &&
                   lhs_entry_ti.get_compression() == kENCODING_DICT)) {
        CHECK_EQ(4, lhs_entry_ti.get_logical_size());
        CHECK(executor_);
        const auto lhs_string_dict_proxy = executor_->getStringDictionaryProxy(
            lhs_entry_ti.getStringDictKey(), result_set_->row_set_mem_owner_, false);
        const auto rhs_string_dict_proxy = executor_->getStringDictionaryProxy(
            rhs_entry_ti.getStringDictKey(), result_set_->row_set_mem_owner_, false);
        const auto lhs_str = lhs_string_dict_proxy->getString(lhs_v.i1);
        const auto rhs_str = rhs_string_dict_proxy->getString(rhs_v.i1);
        if (lhs_str == rhs_str) {
          continue;
        }
        return (lhs_str < rhs_str) != order_entry.is_desc;
      }

      if (lhs_v.i1 == rhs_v.i1) {
        continue;
      }
      if (lhs_entry_ti.is_fp()) {
        if (float_argument_input) {
          const auto lhs_dval = *reinterpret_cast<const float*>(may_alias_ptr(&lhs_v.i1));
          const auto rhs_dval = *reinterpret_cast<const float*>(may_alias_ptr(&rhs_v.i1));
          return (lhs_dval < rhs_dval) != order_entry.is_desc;
        } else {
          const auto lhs_dval =
              *reinterpret_cast<const double*>(may_alias_ptr(&lhs_v.i1));
          const auto rhs_dval =
              *reinterpret_cast<const double*>(may_alias_ptr(&rhs_v.i1));
          return (lhs_dval < rhs_dval) != order_entry.is_desc;
        }
      }
      return (lhs_v.i1 < rhs_v.i1) != order_entry.is_desc;
    } else {
      if (lhs_v.isPair()) {
        CHECK(rhs_v.isPair());
        const auto lhs =
            pair_to_double({lhs_v.i1, lhs_v.i2}, lhs_entry_ti, float_argument_input);
        const auto rhs =
            pair_to_double({rhs_v.i1, rhs_v.i2}, rhs_entry_ti, float_argument_input);
        if (lhs == rhs) {
          continue;
        }
        return (lhs < rhs) != order_entry.is_desc;
      } else {
        CHECK(lhs_v.isStr() && rhs_v.isStr());
        const auto lhs = lhs_v.strVal();
        const auto rhs = rhs_v.strVal();
        if (lhs == rhs) {
          continue;
        }
        return (lhs < rhs) != order_entry.is_desc;
      }
    }
  }
  return false;
}

Here is the call graph for this function:

Member Data Documentation

template<typename BUFFER_ITERATOR_TYPE>

const ApproxQuantileBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::approx_quantile_materialized_buffers_

Definition at line 872 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const BufferIteratorType ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::buffer_itr_

Definition at line 868 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

std::vector<std::vector<int64_t> > ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::count_distinct_materialized_buffers_

Definition at line 871 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const Executor* ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::executor_

Definition at line 869 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const ModeBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::mode_buffers_

Definition at line 873 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const std::list<Analyzer::OrderEntry>& ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::order_entries_

Definition at line 865 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const PermutationView ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::permutation_

Definition at line 867 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const ResultSet* ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::result_set_

Definition at line 866 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>

const bool ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::single_threaded_

Definition at line 870 of file ResultSet.h.

---

The documentation for this struct was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/ResultSet.h
• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/ResultSet.cpp