UtilityTableFunctions.h File Reference

#include "QueryEngine/heavydbTypes.h"

Include dependency graph for UtilityTableFunctions.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
template<typename T , typename K >
NEVER_INLINE HOST int32_t	generate_series__cpu_template (TableFunctionManager &mgr, const T start, const T stop, const K step, Column< T > &series_output)
template<typename T >
NEVER_INLINE HOST int32_t	generate_series__cpu_template (TableFunctionManager &mgr, const T start, const T stop, Column< T > &series_output)
EXTENSION_NOINLINE_HOST int32_t	generate_random_strings__cpu_ (TableFunctionManager &mgr, const int64_t num_strings, const int64_t string_length, Column< int64_t > &output_id, Column< TextEncodingDict > &output_strings)

Function Documentation

EXTENSION_NOINLINE_HOST int32_t generate_random_strings__cpu_	(	TableFunctionManager &	mgr,
		const int64_t	num_strings,
		const int64_t	string_length,
		Column< int64_t > &	output_id,
		Column< TextEncodingDict > &	output_strings
	)

Definition at line 121 of file UtilityTableFunctions.cpp.

References CHECK_LE, DEBUG_TIMER, gen_random_str(), ThreadInfo::num_elems_per_thread, ThreadInfo::num_threads, threading_serial::parallel_for(), and TableFunctionManager::set_output_row_size().

                                                                                {
  auto timer = DEBUG_TIMER(__func__);
  // Check for out-of-range errors for the input parameters
  // in the function instead of with require due to issue encountered
  // with require over multiple variables
  constexpr int64_t max_strings{10000000L};
  constexpr int64_t max_str_len{10000L};
  if (num_strings > max_strings) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: num_strings must be between 0 and 10,000,000.");
  }
  if (string_length > max_str_len) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: string_length must be between 1 and 10,000.");
  }
  if (num_strings == 0L) {
    // Bail early as there is no work to be done
    return 0;
  }

  mgr.set_output_row_size(num_strings);
  constexpr int64_t target_strings_per_thread{5000};
  const ThreadInfo thread_info(
      std::thread::hardware_concurrency(), num_strings, target_strings_per_thread);
  std::vector<std::mt19937> per_thread_rand_generators;
  per_thread_rand_generators.reserve(thread_info.num_threads);
  for (int64_t thread_idx = 0; thread_idx < thread_info.num_threads; ++thread_idx) {
    const uint64_t seed = std::chrono::duration_cast<std::chrono::nanoseconds>(
                              std::chrono::system_clock::now().time_since_epoch())
                              .count() +
                          thread_idx * 971;
    per_thread_rand_generators.emplace_back(seed);
  }
  std::vector<std::string> rand_strings(num_strings);
  tbb::task_arena limited_arena(thread_info.num_threads);
  limited_arena.execute([&] {
    CHECK_LE(tbb::this_task_arena::max_concurrency(), thread_info.num_threads);
    tbb::parallel_for(
        tbb::blocked_range<int64_t>(0, num_strings, thread_info.num_elems_per_thread),
        [&](const tbb::blocked_range<int64_t>& r) {
          const int64_t tbb_thread_idx = tbb::this_task_arena::current_thread_index();
          const int64_t start_out_idx = r.begin();
          const int64_t end_out_idx = r.end();
          for (int64_t out_idx = start_out_idx; out_idx != end_out_idx; ++out_idx) {
            rand_strings[out_idx] =
                gen_random_str(per_thread_rand_generators[tbb_thread_idx], string_length);
          }
        },
        tbb::simple_partitioner());
  });
  const std::vector<int32_t> rand_string_ids =
      output_strings.string_dict_proxy_->getOrAddTransientBulk(rand_strings);
  for (int64_t row_idx = 0; row_idx < num_strings; row_idx++) {
    output_id[row_idx] = row_idx;
    output_strings[row_idx] = rand_string_ids[row_idx];
  }
  return num_strings;
}

Here is the call graph for this function:

template<typename T , typename K >

NEVER_INLINE HOST int32_t generate_series__cpu_template	(	TableFunctionManager &	mgr,
		const T	start,
		const T	stop,
		const K	step,
		Column< T > &	series_output
	)

Definition at line 58 of file UtilityTableFunctions.cpp.

References generate_series_parallel(), numStepsBetween(), TableFunctionManager::set_output_row_size(), and SUCCESS.

Referenced by generate_series__cpu_template().

                                                                                  {
  const int64_t PARALLEL_THRESHOLD{10000L};
  const int64_t num_rows = numStepsBetween(start, stop, step) + 1;
  if (num_rows <= 0) {
    mgr.set_output_row_size(0);
    return 0;
  }
  // set_output_row_size ensures that the output buffer size will be
  // in a reasonable range (up to 16 TiB), and if it is not, an
  // OutOfHostMemory exception will be thrown.
  mgr.set_output_row_size(num_rows);

#ifdef HAVE_TBB
  if (num_rows > PARALLEL_THRESHOLD) {
    return generate_series_parallel(start, stop, step, series_output);
  }
#endif

  for (int64_t out_idx = 0; out_idx != num_rows; ++out_idx) {
    series_output[out_idx] = start + (step * out_idx);
  }
  return SUCCESS;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

NEVER_INLINE HOST int32_t generate_series__cpu_template	(	TableFunctionManager &	mgr,
		const T	start,
		const T	stop,
		Column< T > &	series_output
	)

Definition at line 87 of file UtilityTableFunctions.cpp.

References generate_series__cpu_template().

                                                                                  {
  return generate_series__cpu_template(
      mgr, start, stop, static_cast<int64_t>(1), series_output);
}

Here is the call graph for this function: