#include "ExtensionFunctionsBinding.h"
#include <algorithm>
#include "ExternalExecutor.h"

Include dependency graph for ExtensionFunctionsBinding.cpp:

Namespaces
	anonymous_namespace{ExtensionFunctionsBinding.cpp}

Functions
ExtArgumentType	anonymous_namespace{ExtensionFunctionsBinding.cpp}::get_column_arg_elem_type (const ExtArgumentType ext_arg_column_type)

ExtArgumentType	anonymous_namespace{ExtensionFunctionsBinding.cpp}::get_column_list_arg_elem_type (const ExtArgumentType ext_arg_column_list_type)

ExtArgumentType	anonymous_namespace{ExtensionFunctionsBinding.cpp}::get_array_arg_elem_type (const ExtArgumentType ext_arg_array_type)

static int	anonymous_namespace{ExtensionFunctionsBinding.cpp}::match_numeric_argument (const SQLTypeInfo &arg_type_info, const bool is_arg_literal, const ExtArgumentType &sig_ext_arg_type, int32_t &penalty_score)

static int	anonymous_namespace{ExtensionFunctionsBinding.cpp}::match_arguments (const SQLTypeInfo &arg_type, const bool is_arg_literal, int sig_pos, const std::vector< ExtArgumentType > &sig_types, int &penalty_score)

bool	anonymous_namespace{ExtensionFunctionsBinding.cpp}::is_valid_identifier (std::string str)

template<typename T >
std::tuple< T, std::vector < SQLTypeInfo > >	bind_function (std::string name, Analyzer::ExpressionPtrVector func_args, const std::vector< T > &ext_funcs, const std::string processor)

const std::tuple < table_functions::TableFunction, std::vector< SQLTypeInfo > >	bind_table_function (std::string name, Analyzer::ExpressionPtrVector input_args, const std::vector< table_functions::TableFunction > &table_funcs, const bool is_gpu)

ExtensionFunction	bind_function (std::string name, Analyzer::ExpressionPtrVector func_args)

ExtensionFunction	bind_function (std::string name, Analyzer::ExpressionPtrVector func_args, const bool is_gpu)

ExtensionFunction	bind_function (const Analyzer::FunctionOper *function_oper, const bool is_gpu)

const std::tuple < table_functions::TableFunction, std::vector< SQLTypeInfo > >	bind_table_function (std::string name, Analyzer::ExpressionPtrVector input_args, const bool is_gpu)

Function Documentation

template<typename T >

std::tuple<T, std::vector<SQLTypeInfo> > bind_function	(	std::string	name,
		Analyzer::ExpressionPtrVector	func_args,
		const std::vector< T > &	ext_funcs,
		const std::string	processor
	)

Definition at line 548 of file ExtensionFunctionsBinding.cpp.

References CHECK, CHECK_EQ, DEFAULT_ROW_MULTIPLIER_SUFFIX, ext_arg_type_to_type_info(), logger::FATAL, generate_column_list_type(), generate_column_type(), SQLTypeInfo::get_type(), SQLTypeInfo::has_same_itemtype(), is_ext_arg_type_column_list(), anonymous_namespace{ExtensionFunctionsBinding.cpp}::is_valid_identifier(), kINT, kNULLT, kTEXT, LOG, anonymous_namespace{ExtensionFunctionsBinding.cpp}::match_arguments(), setup::name, SQLTypeInfo::set_dimension(), SQLTypeInfo::supportsFlatBuffer(), heavydb.dtypes::T, to_string(), SQLTypeInfo::to_string(), ExtensionFunctionsWhitelist::toString(), and UNREACHABLE.

Referenced by bind_function(), CodeGenerator::codegenFunctionOper(), and RelAlgTranslator::translateFunction().

                                {
   /* worker function
 
      Template type T must implement the following methods:
 
        std::vector<ExtArgumentType> getInputArgs()
    */
   /*
     Return extension function/table function that has the following
     properties
 
     1. each argument type in `arg_types` matches with extension
        function argument types.
 
        For scalar types, the matching means that the types are either
        equal or the argument type is smaller than the corresponding
        the extension function argument type. This ensures that no
        information is lost when casting of argument values is
        required.
 
        For array and geo types, the matching means that the argument
        type matches exactly with a group of extension function
        argument types. See `match_arguments`.
 
     2. has minimal penalty score among all implementations of the
        extension function with given `name`, see `get_penalty_score`
        for the definition of penalty score.
 
     It is assumed that function_oper and extension functions in
     ext_funcs have the same name.
    */
   if (!is_valid_identifier(name)) {
     throw NativeExecutionError(
         "Cannot bind function with invalid UDF/UDTF function name: " + name);
   }
 
   std::vector<SQLTypeInfo> type_infos_input;
   std::vector<bool> args_are_constants;
   for (auto atype : func_args) {
     if constexpr (std::is_same_v<T, table_functions::TableFunction>) {
       if (dynamic_cast<const Analyzer::ColumnVar*>(atype.get())) {
         SQLTypeInfo type_info = atype->get_type_info();
         auto ti = generate_column_type(type_info);
         if (ti.get_subtype() == kNULLT) {
           throw std::runtime_error(std::string(__FILE__) + "#" +
                                    std::to_string(__LINE__) +
                                    ": column support for type info " +
                                    type_info.to_string() + " is not implemented");
         }
         ti.setUsesFlatBuffer(type_info.supportsFlatBuffer());
         type_infos_input.push_back(ti);
         args_are_constants.push_back(type_info.get_type() != kTEXT);
         continue;
       }
     }
     type_infos_input.push_back(atype->get_type_info());
     if (dynamic_cast<const Analyzer::Constant*>(atype.get())) {
       args_are_constants.push_back(true);
     } else {
       args_are_constants.push_back(false);
     }
   }
   CHECK_EQ(type_infos_input.size(), args_are_constants.size());
 
   if (type_infos_input.size() == 0 && ext_funcs.size() > 0) {
     CHECK_EQ(ext_funcs.size(), static_cast<size_t>(1));
     CHECK_EQ(ext_funcs[0].getInputArgs().size(), static_cast<size_t>(0));
     if constexpr (std::is_same_v<T, table_functions::TableFunction>) {
       CHECK(ext_funcs[0].hasNonUserSpecifiedOutputSize());
     }
     std::vector<SQLTypeInfo> empty_type_info_variant(0);
     return {ext_funcs[0], empty_type_info_variant};
   }
 
   int minimal_score = std::numeric_limits<int>::max();
   int index = -1;
   int optimal = -1;
   int optimal_variant = -1;
   std::vector<std::vector<SQLTypeInfo>> type_infos_variants;
 
   // clang-format off
   /*
     Table functions may have arguments such as ColumnList that collect
     neighboring columns with the same data type into a single object. In
     general, the binding of UDTFs with ColumnLists might be ambiguous depending
     on the order of the arguments:
 
       foo(ColumnList<T>, ColumnList<T>) -> Column<T>, T=[int]
       bar(ColumnList<T>, Column<T>, ColumnList<T>) -> Column<T>, T=[int]
 
     Here both declarations above are ambiguous as the first ColumnList can
     consume as many columns as possible, leaving a single column for each one
     one of the remaining types. Or it can consume one argument, leaving the bulk
     to the last ColumnList. Nevertheless, not all ColumnList declarations result
     in an ambiguity signature. The example below shows an example of a function
     that takes two ColumnLists of different types which has an exact match.
 
       baz(ColumnList<P>, ColumnList<T>) -> Column<T>, T=[int], Z=[float]
 
     To match a list of SQL arguments with an extension function, HeavyDB uses a
     greedy algorithm that resolves the issue binding ambiguity as explained
     below. As an example, let us consider a SQL query containing the following
     expression calling a UDTF `bar` defined above:
 
       table(bar(select a, b, c, d, e from tableofints), 1)
 
     The algorithm will generate the following type variant, where the integer
     value in [..] indicate the number of collected columns. This number is later
     stored in the SQLTypeInfo dimension attribute.
 
       bar(ColumnList<T>[3], Column<T>, ColumnList<T>[1])
 
   */
 
   // clang-format on
 
   // Find extension function that gives the best match on the set of
   // argument type variants
   for (const auto& ext_func : ext_funcs) {
     index++;
 
     const auto& ext_func_args = ext_func.getInputArgs();
 
     int penalty_score = 0;
     int pos = 0;
     int original_input_idx = 0;
     type_infos_variants.emplace_back();
 
     for (size_t i = 0; i < type_infos_input.size(); i++) {
       const SQLTypeInfo& ti = type_infos_input[i];
 
       if ((size_t)pos >= ext_func_args.size()) {
         pos = -1;
         break;
       } else if (is_ext_arg_type_column_list(ext_func_args[pos])) {
         SQLTypeInfo ti_col_list = generate_column_list_type(ti);
         int offset = match_arguments(ti_col_list,
                                      args_are_constants[original_input_idx],
                                      pos,
                                      ext_func_args,
                                      penalty_score);
         if (offset < 0) {
           pos = -1;
           break;
         }
 
         // if offset > 0, greedly iterate over the rest of input args
         // to consume columns with the same type as "ti"
         int j = i;
         size_t args_left = ext_func_args.size() - pos - 1;
         while ((type_infos_input.size() - j > args_left) and
                (ti_col_list.has_same_itemtype(type_infos_input[j]))) {
           j++;
         }
         // push_back a ColumnList with dimension equals to the number of columns
         // consumed above
         ti_col_list.set_dimension(j - i);
         type_infos_variants.back().push_back(ti_col_list);
         // Move the "i" pointer to the last argument consumed
         i = j - 1;
         original_input_idx = j;
         pos += offset;
       } else {
         int offset = match_arguments(ti,
                                      args_are_constants[original_input_idx],
                                      pos,
                                      ext_func_args,
                                      penalty_score);
 
         if (offset > 0) {
           type_infos_variants.back().push_back(ti);
           original_input_idx += 1;
           pos += offset;
         } else {
           pos = -1;
           break;
         }
       }
     }
 
     if ((size_t)pos == ext_func_args.size()) {
       CHECK_EQ(args_are_constants.size(), original_input_idx);
       // prefer smaller return types
       penalty_score += ext_arg_type_to_type_info(ext_func.getRet()).get_logical_size();
       if (penalty_score < minimal_score) {
         optimal = index;
         minimal_score = penalty_score;
         optimal_variant = type_infos_variants.size() - 1;
       }
     }
   }
 
   if (optimal == -1) {
     /* no extension function found that argument types would match
        with types in `arg_types` */
     auto sarg_types = ExtensionFunctionsWhitelist::toString(type_infos_input);
     std::string message;
     if (!ext_funcs.size()) {
       message = "Function " + name + "(" + sarg_types + ") not supported.";
       throw ExtensionFunctionBindingError(message);
     } else {
       if constexpr (std::is_same_v<T, table_functions::TableFunction>) {
         message = "Could not bind " + name + "(" + sarg_types + ") to any " + processor +
                   " UDTF implementation.";
       } else if constexpr (std::is_same_v<T, ExtensionFunction>) {
         message = "Could not bind " + name + "(" + sarg_types + ") to any " + processor +
                   " UDF implementation.";
       } else {
         LOG(FATAL) << "bind_function: unknown extension function type "
                    << typeid(T).name();
       }
       message += "\n  Existing extension function implementations:";
       for (const auto& ext_func : ext_funcs) {
         // Do not show functions missing the sizer argument
         if constexpr (std::is_same_v<T, table_functions::TableFunction>) {
           if (ext_func.useDefaultSizer()) {
             continue;
           }
         }
         message += "\n    " + ext_func.toStringSQL();
       }
     }
     throw ExtensionFunctionBindingError(message);
   }
 
   // Functions with "_default_" suffix only exist for calcite
   if constexpr (std::is_same_v<T, table_functions::TableFunction>) {
     if (ext_funcs[optimal].hasUserSpecifiedOutputSizeMultiplier() &&
         ext_funcs[optimal].useDefaultSizer()) {
       std::string name = ext_funcs[optimal].getName();
       name.erase(name.find(DEFAULT_ROW_MULTIPLIER_SUFFIX),
                  sizeof(DEFAULT_ROW_MULTIPLIER_SUFFIX));
       for (size_t i = 0; i < ext_funcs.size(); i++) {
         if (ext_funcs[i].getName() == name) {
           optimal = i;
           std::vector<SQLTypeInfo> type_info = type_infos_variants[optimal_variant];
           size_t sizer = ext_funcs[optimal].getOutputRowSizeParameter();
           type_info.insert(type_info.begin() + sizer - 1, SQLTypeInfo(kINT, true));
           return {ext_funcs[optimal], type_info};
         }
       }
       UNREACHABLE();
     }
   }
 
   return {ext_funcs[optimal], type_infos_variants[optimal_variant]};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ExtensionFunction bind_function	(	std::string	name,
		Analyzer::ExpressionPtrVector	func_args
	)

Definition at line 810 of file ExtensionFunctionsBinding.cpp.

References ExtensionFunctionsWhitelist::get_ext_funcs(), and setup::name.

                                                                        {
   // used in RelAlgTranslator.cpp, first try GPU UDFs, then fall back
   // to CPU UDFs.
   bool is_gpu = true;
   std::string processor = "GPU";
   auto ext_funcs = ExtensionFunctionsWhitelist::get_ext_funcs(name, is_gpu);
   if (!ext_funcs.size()) {
     is_gpu = false;
     processor = "CPU";
     ext_funcs = ExtensionFunctionsWhitelist::get_ext_funcs(name, is_gpu);
   }
   try {
     return std::get<0>(
         bind_function<ExtensionFunction>(name, func_args, ext_funcs, processor));
   } catch (ExtensionFunctionBindingError& e) {
     if (is_gpu) {
       is_gpu = false;
       processor = "GPU|CPU";
       ext_funcs = ExtensionFunctionsWhitelist::get_ext_funcs(name, is_gpu);
       return std::get<0>(
           bind_function<ExtensionFunction>(name, func_args, ext_funcs, processor));
     } else {
       throw;
     }
   }
 }

Here is the call graph for this function:

ExtensionFunction bind_function	(	std::string	name,
		Analyzer::ExpressionPtrVector	func_args,
		const bool	is_gpu
	)

Definition at line 838 of file ExtensionFunctionsBinding.cpp.

References ExtensionFunctionsWhitelist::get_ext_funcs(), and setup::name.

                                                    {
   // used below
   std::vector<ExtensionFunction> ext_funcs =
       ExtensionFunctionsWhitelist::get_ext_funcs(name, is_gpu);
   std::string processor = (is_gpu ? "GPU" : "CPU");
   return std::get<0>(
       bind_function<ExtensionFunction>(name, func_args, ext_funcs, processor));
 }

Here is the call graph for this function:

ExtensionFunction bind_function	(	const Analyzer::FunctionOper *	function_oper,
		const bool	is_gpu
	)

Definition at line 849 of file ExtensionFunctionsBinding.cpp.

References bind_function(), Analyzer::FunctionOper::getArity(), Analyzer::FunctionOper::getName(), Analyzer::FunctionOper::getOwnArg(), and setup::name.

                                                    {
   // used in ExtensionsIR.cpp
   auto name = function_oper->getName();
   Analyzer::ExpressionPtrVector func_args = {};
   for (size_t i = 0; i < function_oper->getArity(); ++i) {
     func_args.push_back(function_oper->getOwnArg(i));
   }
   return bind_function(name, func_args, is_gpu);
 }

Here is the call graph for this function:

const std::tuple<table_functions::TableFunction, std::vector<SQLTypeInfo> > bind_table_function	(	std::string	name,
		Analyzer::ExpressionPtrVector	input_args,
		const std::vector< table_functions::TableFunction > &	table_funcs,
		const bool	is_gpu
	)

Definition at line 801 of file ExtensionFunctionsBinding.cpp.

References setup::name.

Referenced by bind_table_function(), and RelAlgExecutor::createTableFunctionWorkUnit().

                                        {
   std::string processor = (is_gpu ? "GPU" : "CPU");
   return bind_function<table_functions::TableFunction>(
       name, input_args, table_funcs, processor);
 }

Here is the caller graph for this function:

const std::tuple<table_functions::TableFunction, std::vector<SQLTypeInfo> > bind_table_function	(	std::string	name,
		Analyzer::ExpressionPtrVector	input_args,
		const bool	is_gpu
	)

Definition at line 861 of file ExtensionFunctionsBinding.cpp.

References bind_table_function(), and table_functions::TableFunctionsFactory::get_table_funcs().

                                        {
   // used in RelAlgExecutor.cpp
   std::vector<table_functions::TableFunction> table_funcs =
       table_functions::TableFunctionsFactory::get_table_funcs(name, is_gpu);
   return bind_table_function(name, input_args, table_funcs, is_gpu);
 }

Here is the call graph for this function:

Namespaces

Functions

Function Documentation