TableFunctionsStats.hpp File Reference

#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>
#include <rapidjson/document.h>
#include "QueryEngine/heavydbTypes.h"

Include dependency graph for TableFunctionsStats.hpp:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	StatsRequestPredicate
struct	ColumnStats< T >
struct	StatsRequest

Enumerations
enum	StatsRequestPredicateOp { StatsRequestPredicateOp::NONE, StatsRequestPredicateOp::LT, StatsRequestPredicateOp::GT }
enum	StatsRequestAggType {   StatsRequestAggType::COUNT, StatsRequestAggType::MIN, StatsRequestAggType::MAX, StatsRequestAggType::SUM,   StatsRequestAggType::AVG }

Functions
std::vector< StatsRequest >	parse_stats_requests_json (const std::string &stats_requests_json_str, const int64_t num_attrs)
template<typename TA >
ColumnStats< TA >	get_column_stats (const ColumnList< TA > &attrs, StatsRequest &stats_request, std::unordered_map< std::string, ColumnStats< TA >> &stats_map)
template<typename TA >
void	compute_stats_requests (const ColumnList< TA > &attrs, std::vector< StatsRequest > &stats_requests)
template<typename TA >
void	populate_output_stats_cols (Column< TextEncodingDict > &stat_names, Column< TA > &stat_vals, const std::vector< StatsRequest > &stats_requests)
template<typename T >
NEVER_INLINE HOST ColumnStats< T >	get_column_stats (const T *data, const int64_t num_rows, const StatsRequestPredicate &predicate=StatsRequestPredicate())
template<typename T >
NEVER_INLINE HOST ColumnStats< T >	get_column_stats (const Column< T > &col, const StatsRequestPredicate &predicate=StatsRequestPredicate())

Enumeration Type Documentation

enum StatsRequestAggType

strong

Enumerator
COUNT
MIN
MAX
SUM
AVG

Definition at line 104 of file TableFunctionsStats.hpp.

                               {
  COUNT,
  MIN,
  MAX,
  SUM,
  AVG,
};

enum StatsRequestPredicateOp

strong

Enumerator
NONE
LT
GT

Definition at line 30 of file TableFunctionsStats.hpp.

{ NONE, LT, GT };

Function Documentation

template<typename TA >

void compute_stats_requests	(	const ColumnList< TA > &	attrs,
		std::vector< StatsRequest > &	stats_requests
	)

Definition at line 143 of file TableFunctionsStats.hpp.

References AVG, COUNT, get_column_stats(), MAX, MIN, and SUM.

                                                                     {
  std::unordered_map<std::string, ColumnStats<TA>> stats_map;

  for (auto& stats_request : stats_requests) {
    const auto column_stats = get_column_stats(attrs, stats_request, stats_map);
    switch (stats_request.agg_type) {
      case StatsRequestAggType::COUNT: {
        stats_request.result = column_stats.non_null_or_filtered_count;
        break;
      }
      case StatsRequestAggType::MIN: {
        stats_request.result = column_stats.min;
        break;
      }
      case StatsRequestAggType::MAX: {
        stats_request.result = column_stats.max;
        break;
      }
      case StatsRequestAggType::SUM: {
        stats_request.result = column_stats.sum;
        break;
      }
      case StatsRequestAggType::AVG: {
        stats_request.result = column_stats.mean;
        break;
      }
    }
  }
}

Here is the call graph for this function:

template<typename TA >

ColumnStats<TA> get_column_stats	(	const ColumnList< TA > &	attrs,
		StatsRequest &	stats_request,
		std::unordered_map< std::string, ColumnStats< TA >> &	stats_map
	)

Definition at line 126 of file TableFunctionsStats.hpp.

References StatsRequest::attr_id, StatsRequest::filter_type, StatsRequest::filter_val, get_column_stats(), StatsRequestPredicate::to_string(), and to_string().

                                                             {
  StatsRequestPredicate predicate(stats_request.filter_type, stats_request.filter_val);
  const std::string request_str_key =
      std::to_string(stats_request.attr_id) + "||" + predicate.to_string();
  auto stats_map_itr = stats_map.find(request_str_key);
  if (stats_map_itr != stats_map.end()) {
    return stats_map_itr->second;
  }
  const auto column_stats = get_column_stats(attrs[stats_request.attr_id], predicate);
  stats_map[request_str_key] = column_stats;
  return column_stats;
}

Here is the call graph for this function:

template<typename T >

NEVER_INLINE HOST ColumnStats<T> get_column_stats	(	const T *	data,
		const int64_t	num_rows,
		const StatsRequestPredicate &	predicate = StatsRequestPredicate()
	)

Definition at line 22 of file TableFunctionsStats.cpp.

References ColumnStats< T >::max, max_inputs_per_thread, ColumnStats< T >::mean, ColumnStats< T >::min, ColumnStats< T >::non_null_or_filtered_count, threading_serial::parallel_for(), ColumnStats< T >::sum, heavydb.dtypes::T, and ColumnStats< T >::total_count.

Referenced by compute_stats_requests(), and get_column_stats().

                                            {
  // const int64_t num_rows = col.size();
  const size_t max_thread_count = std::thread::hardware_concurrency();
  const size_t max_inputs_per_thread = 20000;
  const size_t num_threads = std::min(
      max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));

  std::vector<T> local_col_mins(num_threads, std::numeric_limits<T>::max());
  std::vector<T> local_col_maxes(num_threads, std::numeric_limits<T>::lowest());
  std::vector<double> local_col_sums(num_threads, 0.);
  std::vector<int64_t> local_col_non_null_or_filtered_counts(num_threads, 0L);
  tbb::task_arena limited_arena(num_threads);
  limited_arena.execute([&] {
    tbb::parallel_for(
        tbb::blocked_range<int64_t>(0, num_rows),
        [&](const tbb::blocked_range<int64_t>& r) {
          const int64_t start_idx = r.begin();
          const int64_t end_idx = r.end();
          T local_col_min = std::numeric_limits<T>::max();
          T local_col_max = std::numeric_limits<T>::lowest();
          double local_col_sum = 0.;
          int64_t local_col_non_null_or_filtered_count = 0;
          for (int64_t r = start_idx; r < end_idx; ++r) {
            const T val = data[r];
            if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double>) {
              if (std::isnan(val) || std::isinf(val)) {
                continue;
              }
            }
            if (val == inline_null_value<T>()) {
              continue;
            }
            if (!predicate(val)) {
              continue;
            }
            if (val < local_col_min) {
              local_col_min = val;
            }
            if (val > local_col_max) {
              local_col_max = val;
            }
            local_col_sum += data[r];
            local_col_non_null_or_filtered_count++;
          }
          size_t thread_idx = tbb::this_task_arena::current_thread_index();
          if (local_col_min < local_col_mins[thread_idx]) {
            local_col_mins[thread_idx] = local_col_min;
          }
          if (local_col_max > local_col_maxes[thread_idx]) {
            local_col_maxes[thread_idx] = local_col_max;
          }
          local_col_sums[thread_idx] += local_col_sum;
          local_col_non_null_or_filtered_counts[thread_idx] +=
              local_col_non_null_or_filtered_count;
        });
  });

  ColumnStats<T> column_stats;
  // Use separate double col_sum instead of column_stats.sum to avoid fp imprecision if T
  // is float
  double col_sum = 0.0;
  column_stats.total_count = num_rows;

  for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
    if (local_col_mins[thread_idx] < column_stats.min) {
      column_stats.min = local_col_mins[thread_idx];
    }
    if (local_col_maxes[thread_idx] > column_stats.max) {
      column_stats.max = local_col_maxes[thread_idx];
    }
    col_sum += local_col_sums[thread_idx];
    column_stats.non_null_or_filtered_count +=
        local_col_non_null_or_filtered_counts[thread_idx];
  }

  if (column_stats.non_null_or_filtered_count > 0) {
    column_stats.sum = col_sum;
    column_stats.mean = col_sum / column_stats.non_null_or_filtered_count;
  }
  return column_stats;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

NEVER_INLINE HOST ColumnStats<T> get_column_stats	(	const Column< T > &	col,
		const StatsRequestPredicate &	predicate = StatsRequestPredicate()
	)

Definition at line 133 of file TableFunctionsStats.cpp.

References get_column_stats(), Column< T >::getPtr(), and Column< T >::size().

                                            {
  return get_column_stats(col.getPtr(), col.size(), predicate);
}

Here is the call graph for this function:

std::vector<StatsRequest> parse_stats_requests_json	(	const std::string &	stats_requests_json_str,
		const int64_t	num_attrs
	)

Definition at line 211 of file TableFunctionsStats.cpp.

References StatsRequest::agg_type, StatsRequest::attr_id, convert_string_to_stats_request_agg_type(), convert_string_to_stats_request_predicate_op(), StatsRequest::filter_type, StatsRequest::filter_val, StatsRequest::name, NONE, replace_substrings(), to_string(), and shared::transform().

                             {
  std::vector<StatsRequest> stats_requests;
  rapidjson::Document doc;

  // remove double double quotes our parser introduces
  const auto fixed_stats_requests_json_str =
      replace_substrings(stats_requests_json_str, "\"\"", "\"");

  if (doc.Parse(fixed_stats_requests_json_str.c_str()).HasParseError()) {
    // Not valid JSON
    std::cout << "DEBUG: Failed JSON: " << fixed_stats_requests_json_str << std::endl;
    throw std::runtime_error("Could not parse Stats Requests JSON.");
  }
  // Todo (todd): Enforce Schema
  if (!doc.IsArray()) {
    throw std::runtime_error("Stats Request JSON did not contain valid root Array.");
  }
  const std::vector<std::string> required_keys = {
      "name", "attr_id", "agg_type", "filter_type"};

  for (const auto& stat_request_obj : doc.GetArray()) {
    for (const auto& required_key : required_keys) {
      if (!stat_request_obj.HasMember(required_key)) {
        throw std::runtime_error("Stats Request JSON missing key " + required_key + ".");
      }
      if (required_key == "attr_id") {
        if (!stat_request_obj[required_key].IsUint()) {
          throw std::runtime_error(required_key + " must be int type");
        }
      } else {
        if (!stat_request_obj[required_key].IsString()) {
          throw std::runtime_error(required_key + " must be string type");
        }
      }
    }
    StatsRequest stats_request;
    stats_request.name = stat_request_obj["name"].GetString();
    stats_request.attr_id = stat_request_obj["attr_id"].GetInt() - 1;
    if (stats_request.attr_id < 0 || stats_request.attr_id >= num_attrs) {
      throw std::runtime_error("Invalid attr_id: " +
                               std::to_string(stats_request.attr_id));
    }

    std::string agg_type_str = stat_request_obj["agg_type"].GetString();
    std::transform(
        agg_type_str.begin(), agg_type_str.end(), agg_type_str.begin(), ::toupper);
    stats_request.agg_type = convert_string_to_stats_request_agg_type(agg_type_str);

    std::string filter_type_str = stat_request_obj["filter_type"].GetString();
    std::transform(filter_type_str.begin(),
                   filter_type_str.end(),
                   filter_type_str.begin(),
                   ::toupper);
    stats_request.filter_type =
        convert_string_to_stats_request_predicate_op(filter_type_str);
    if (stats_request.filter_type != StatsRequestPredicateOp::NONE) {
      if (!stat_request_obj.HasMember("filter_val")) {
        throw std::runtime_error("Stats Request JSON missing expected filter_val");
      }
      if (!stat_request_obj["filter_val"].IsNumber()) {
        throw std::runtime_error("Stats Request JSON filter_val should be numeric.");
      }
      stats_request.filter_val = stat_request_obj["filter_val"].GetDouble();
    }
    stats_requests.emplace_back(stats_request);
  }
  return stats_requests;
}

Here is the call graph for this function:

template<typename TA >

void populate_output_stats_cols	(	Column< TextEncodingDict > &	stat_names,
		Column< TA > &	stat_vals,
		const std::vector< StatsRequest > &	stats_requests
	)

Definition at line 175 of file TableFunctionsStats.hpp.

References StringDictionaryProxy::getOrAddTransient(), setup::name, logger::request_id(), and Column< TextEncodingDict >::string_dict_proxy_.

                                                                               {
  const int64_t num_requests = static_cast<int64_t>(stats_requests.size());
  for (int64_t request_id = 0; request_id < num_requests; ++request_id) {
    stat_names[request_id] =
        stat_names.string_dict_proxy_->getOrAddTransient(stats_requests[request_id].name);
    stat_vals[request_id] = stats_requests[request_id].result;
  }
}

Here is the call graph for this function: