#include "StringTransform.h"
#include "Logger/Logger.h"
#include <numeric>
#include <random>
#include <regex>
#include <cmath>
#include <boost/filesystem.hpp>
#include <iomanip>

Include dependency graph for StringTransform.cpp:

Functions
void	apply_shim (std::string &result, const boost::regex &reg_expr, const std::function< void(std::string &, const boost::smatch &)> &shim_fn)

std::vector< std::pair< size_t, size_t > >	find_string_literals (const std::string &query)

std::string	hide_sensitive_data_from_query (std::string const &query_str)

std::string	format_num_bytes (const size_t bytes)

template<>
std::string	to_string (char const *&&v)

template<>
std::string	to_string (std::string &&v)

std::pair< std::string_view, const char * >	substring (const std::string &str, size_t substr_length)
	return substring of str with postfix if str.size() > substr_length More...

std::string	generate_random_string (const size_t len)

std::vector< std::string >	split (std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
	split apart a string into a vector of substrings More...

std::string_view	sv_strip (std::string_view str)
	return trimmed string_view More...

std::string	strip (std::string_view str)
	trim any whitespace from the left and right ends of a string More...

std::optional< size_t >	inside_string_literal (const size_t start, const size_t length, std::vector< std::pair< size_t, size_t >> const &literal_positions)

bool	remove_unquoted_newlines_linefeeds_and_tabs_from_sql_string (std::string &str) noexcept
	sanitize an SQL string More...

std::string	get_quoted_string (const std::string &filename, char quote, char escape)
	Quote a string while escaping any existing quotes in the string. More...

std::string	simple_sanitize (const std::string &str)
	simple sanitize string (replace control characters with space) More...

Function Documentation

void apply_shim	(	std::string &	result,
		const boost::regex &	reg_expr,
		const std::function< void(std::string &, const boost::smatch &)> &	shim_fn
	)

Definition at line 31 of file StringTransform.cpp.

References find_string_literals(), and inside_string_literal().

Referenced by pg_shim().

                                                                                   {
   boost::smatch what;
   auto lit_pos = find_string_literals(result);
   auto start_it = result.cbegin();
   auto end_it = result.cend();
   while (true) {
     if (!boost::regex_search(start_it, end_it, what, reg_expr)) {
       break;
     }
     const auto next_start =
         inside_string_literal(what.position(), what.length(), lit_pos);
     if (next_start) {
       start_it = result.cbegin() + *next_start;
     } else {
       shim_fn(result, what);
       lit_pos = find_string_literals(result);
       start_it = result.cbegin();
       end_it = result.cend();
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<std::pair<size_t, size_t> > find_string_literals ( const std::string & query )

Definition at line 57 of file StringTransform.cpp.

References CHECK_GT.

Referenced by apply_shim().

                                                                                 {
   boost::regex literal_string_regex{R"(([^']+)('(?:[^']+|'')*'))", boost::regex::perl};
   boost::smatch what;
   auto it = query.begin();
   auto prev_it = it;
   std::vector<std::pair<size_t, size_t>> positions;
   while (true) {
     try {
       if (!boost::regex_search(it, query.end(), what, literal_string_regex)) {
         break;
       }
     } catch (const std::exception& e) {
       // boost::regex throws an exception about the complexity of matching when
       // the wrong type of quotes are used or they're mismatched. Let the query
       // through unmodified, the parser will throw a much more informative error.
       // This can also throw on very long queries
       std::ostringstream oss;
       oss << "Detecting an error while processing string literal regex search: "
           << e.what();
       throw std::runtime_error(oss.str());
     }
     CHECK_GT(what[1].length(), 0);
     prev_it = it;
     it += what.length();
     positions.emplace_back(prev_it + what[1].length() - query.begin(),
                            it - query.begin());
   }
   return positions;
 }

Here is the caller graph for this function:

std::string format_num_bytes ( const size_t bytes )

Definition at line 102 of file StringTransform.cpp.

References CHECK_GE, CHECK_LE, and to_string().

Referenced by ExecutorResourceMgr_Namespace::ExecutorResourcePool::add_chunk_requests_to_allocated_pool(), ExecutorResourceMgr_Namespace::ExecutorResourcePool::allocate_resources(), ExecutorResourceMgr_Namespace::ExecutorResourcePool::can_currently_satisfy_chunk_request(), ExecutorResourceMgr_Namespace::ExecutorResourcePool::deallocate_resources(), DBHandler::init_executor_resource_mgr(), ExecutorResourceMgr_Namespace::ResourceGrant::print(), ExecutorResourceMgr_Namespace::ExecutorResourcePool::remove_chunk_requests_from_allocated_pool(), and ExecutorResourceMgr_Namespace::ResourceGrant::to_string().

                                                {
   const size_t units_per_k_unit{1024};
   const std::vector<std::string> byte_units = {" bytes", "KB", "MB", "GB", "TB", "PB"};
   const std::vector<size_t> bytes_per_scale_unit = {size_t(1),
                                                     size_t(1) << 10,
                                                     size_t(1) << 20,
                                                     size_t(1) << 30,
                                                     size_t(1) << 40,
                                                     size_t(1) << 50,
                                                     size_t(1) << 60};
   if (bytes < units_per_k_unit) {
     return std::to_string(bytes) + " bytes";
   }
   CHECK_GE(bytes, units_per_k_unit);
   const size_t byte_scale = log(bytes) / log(units_per_k_unit);
   CHECK_GE(byte_scale, size_t(1));
   CHECK_LE(byte_scale, size_t(5));
   const size_t scaled_bytes_left_of_decimal = bytes / bytes_per_scale_unit[byte_scale];
   const size_t scaled_bytes_right_of_decimal = bytes % bytes_per_scale_unit[byte_scale];
   const size_t fractional_digits = static_cast<double>(scaled_bytes_right_of_decimal) /
                                    bytes_per_scale_unit[byte_scale] * 100.;
   return std::to_string(scaled_bytes_left_of_decimal) + "." +
          std::to_string(fractional_digits) + " " + byte_units[byte_scale];
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string generate_random_string ( const size_t len )

Definition at line 150 of file StringTransform.cpp.

Referenced by CachedSessionStore::add(), DBHandler::createInMemoryCalciteSession(), Catalog_Namespace::SysCatalog::syncUserWithRemoteProvider(), and Catalog_Namespace::SysCatalog::updateBlankPasswordsToRandom().

                                                    {
   static char charset[] =
       "0123456789"
       "abcdefghijklmnopqrstuvwxyz"
       "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
   static std::mt19937 prng{std::random_device{}()};
   static std::uniform_int_distribution<size_t> dist(0, strlen(charset) - 1);
 
   std::string str;
   str.reserve(len);
   for (size_t i = 0; i < len; i++) {
     str += charset[dist(prng)];
   }
   return str;
 }

Here is the caller graph for this function:

std::string get_quoted_string	(	const std::string &	filename,
		char	quote,
		char	escape
	)

Quote a string while escaping any existing quotes in the string.

Definition at line 288 of file StringTransform.cpp.

Referenced by TableArchiver::dumpTable(), Catalog_Namespace::Catalog::quoteIfRequired(), TableArchiver::restoreTable(), and anonymous_namespace{TableArchiver.cpp}::simple_file_cat().

                                                                                 {
   std::stringstream ss;
   ss << std::quoted(filename, quote, escape);  // TODO: prevents string_view Jun 2020
   return ss.str();
 }

Here is the caller graph for this function:

std::string hide_sensitive_data_from_query ( std::string const & query_str )

Definition at line 87 of file StringTransform.cpp.

References gpu_enabled::accumulate().

Referenced by query_state::StdLog::log(), and Calcite::processImpl().

                                                                      {
   constexpr std::regex::flag_type flags =
       std::regex::ECMAScript | std::regex::icase | std::regex::optimize;
   static const std::initializer_list<std::pair<std::regex, std::string>> rules{
       {std::regex(
            R"(\b((?:password|s3_access_key|s3_secret_key|s3_session_token|username|credential_string)\s*=\s*)'.+?')",
            flags),
        "$1'XXXXXXXX'"},
       {std::regex(R"((\\set_license\s+)\S+)", flags), "$1XXXXXXXX"}};
   return std::accumulate(
       rules.begin(), rules.end(), query_str, [](auto& str, auto& rule) {
         return std::regex_replace(str, rule.first, rule.second);
       });
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::optional<size_t> inside_string_literal	(	const size_t	start,
		const size_t	length,
		std::vector< std::pair< size_t, size_t >> const &	literal_positions
	)

Definition at line 233 of file StringTransform.cpp.

Referenced by apply_shim().

                                                                  {
   const auto end = start + length;
   for (const auto& literal_position : literal_positions) {
     if (literal_position.first <= start && end <= literal_position.second) {
       return literal_position.second;
     }
   }
   return std::nullopt;
 }

Here is the caller graph for this function:

bool remove_unquoted_newlines_linefeeds_and_tabs_from_sql_string ( std::string & str )

noexcept

sanitize an SQL string

Definition at line 248 of file StringTransform.cpp.

                              {
   char inside_quote = 0;
   bool previous_c_was_backslash = false;
   for (auto& c : str) {
     // if this character is a quote of either type
     if (c == '\'' || c == '\"') {
       // ignore if previous character was a backslash
       if (!previous_c_was_backslash) {
         // start or end of a quoted region
         if (inside_quote == c) {
           // end region
           inside_quote = 0;
         } else if (inside_quote == 0) {
           // start region
           inside_quote = c;
         }
       }
     } else if (inside_quote == 0) {
       // outside quoted region
       if (c == '\n' || c == '\t' || c == '\r') {
         // replace these with space
         c = ' ';
       }
       // otherwise leave alone, including quotes of a different type
     }
     // handle backslashes, except for double backslashes
     if (c == '\\') {
       previous_c_was_backslash = !previous_c_was_backslash;
     } else {
       previous_c_was_backslash = false;
     }
   }
   // if we didn't end a region, there were unclosed or mixed-nested quotes
   // accounting for backslashes should mean that this should only be the
   // case with truly malformed strings which Calcite will barf on anyway
   return (inside_quote == 0);
 }

std::string simple_sanitize ( const std::string & str )

simple sanitize string (replace control characters with space)

Definition at line 296 of file StringTransform.cpp.

                                                 {
   auto sanitized_str{str};
   for (auto& c : sanitized_str) {
     c = (c < 32) ? ' ' : c;
   }
   return sanitized_str;
 }

std::vector<std::string> split	(	std::string_view	str,
		std::string_view	delim,
		std::optional< size_t >	maxsplit
	)

split apart a string into a vector of substrings

Definition at line 171 of file StringTransform.cpp.

References run_benchmark_import::result.

Referenced by foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::add_nonce_values(), run_benchmark::benchmark(), create_table.SyntheticTable::createDataAndImportTable(), AlterTableCommand::execute(), ai.heavy.jdbc.HeavyAIStatement::executeQuery(), anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), anonymous_namespace{TableArchiver.cpp}::find_render_group_columns(), generate_TableFunctionsFactory_init::find_signatures(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_data_sources(), get_qualified_column_hints(), table_functions::TableFunction::getCursorFields(), ColumnDescriptor::getDefaultValueLiteral(), com.mapd.parser.server.ExtensionFunction::getPrettyArgNames(), import_export::RasterImporter::getRawBandNamesForFormat(), QueryPlanDagExtractor::handleTranslatedJoin(), import_export::RasterImporter::initializeFiltering(), AutomaticIRMetadataGuard::makeBaseFilename(), AutomaticIRMetadataGuard::makeQueryEngineFilename(), import_export::parse_add_metadata_columns(), TrackingProcessor::process(), Data_Namespace::ProcMeminfoParser::ProcMeminfoParser(), run_benchmark::read_query_files(), TableArchiver::restoreTable(), QueryRunner::QueryRunner::runMultipleStatements(), Parser::splitObjectHierName(), tf_geo_multi_rasterize__cpu_template(), com.mapd.parser.server.ExtensionFunctionSignatureParser::toSignature(), anonymous_namespace{TableArchiver.cpp}::update_or_drop_column_ids_in_page_headers(), and ddl_utils::anonymous_namespace{DdlUtils.cpp}::validate_literal().

                                                              {
   std::vector<std::string> result;
 
   // Use an explicit delimiter.
   if (!delim.empty()) {
     std::string::size_type i = 0, j = 0;
     while ((i = str.find(delim, i)) != std::string::npos &&
            (!maxsplit || result.size() < maxsplit.value())) {
       result.emplace_back(str, j, i - j);
       i += delim.size();
       j = i;
     }
     result.emplace_back(str, j, std::string::npos);
     return result;
 
     // Treat any number of consecutive whitespace characters as a delimiter.
   } else {
     bool prev_ws = true;
     std::string::size_type i = 0, j = 0;
     for (; i < str.size(); ++i) {
       if (prev_ws) {
         if (!isspace(str[i])) {
           // start of word
           prev_ws = false;
           j = i;
         }
       } else {
         if (isspace(str[i])) {
           // start of space
           result.emplace_back(str, j, i - j);
           prev_ws = true;
           j = i;
           if ((maxsplit && result.size() == maxsplit.value())) {
             // stop early if maxsplit was reached
             result.emplace_back(str, j, std::string::npos);
             return result;
           }
         }
       }
     }
     if (!prev_ws) {
       result.emplace_back(str, j, std::string::npos);
     }
     return result;
   }
 }

Here is the caller graph for this function:

std::string strip ( std::string_view str )

trim any whitespace from the left and right ends of a string

Definition at line 229 of file StringTransform.cpp.

References sv_strip().

Referenced by import_export::anonymous_namespace{ExpressionParser.cpp}::Function_bool::Eval(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_data_sources(), getCurrentStackTrace(), import_export::RasterImporter::initializeFiltering(), TableFunctionsFactory_declbracket.Bracket::parse(), import_export::parse_add_metadata_columns(), Data_Namespace::ProcMeminfoParser::ProcMeminfoParser(), QueryRunner::QueryRunner::runMultipleStatements(), and DBHandler::sql_execute_impl().

                                     {
   return std::string(sv_strip(str));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair<std::string_view, const char*> substring	(	const std::string &	str,
		size_t	substr_length
	)

return substring of str with postfix if str.size() > substr_length

Definition at line 137 of file StringTransform.cpp.

Referenced by anonymous_namespace{RelAlgOptimizer.cpp}::add_new_indices_for(), anonymous_namespace{RelAlgOptimizer.cpp}::cleanup_dead_nodes(), ai.heavy.jdbc.HeavyAIEscapeFunctions::createFunctionMap(), ct_substr__cpu_(), fold_filters(), com.mapd.utility.db_vendors.PostGis_types::get_wkt(), and DBHandler::sql_execute_impl().

                                                                          {
   // return substring with a post_fix
   // assume input str is valid and we perform substring starting from str's initial pos
   // (=0)
   const auto str_size = str.size();
   if (substr_length >= str_size) {
     return {str, ""};
   }
   std::string_view substr(str.c_str(), substr_length);
   return {substr, "..."};
 }

Here is the caller graph for this function:

std::string_view sv_strip ( std::string_view str )

return trimmed string_view

Definition at line 220 of file StringTransform.cpp.

Referenced by import_export::import_thread_delimited(), foreign_storage::CsvFileBufferParser::parseBuffer(), and strip().

                                             {
   std::string::size_type i, j;
   for (i = 0; i < str.size() && std::isspace(str[i]); ++i) {
   }
   for (j = str.size(); j > i && std::isspace(str[j - 1]); --j) {
   }
   return str.substr(i, j - i);
 }

Here is the caller graph for this function:

template<>

std::string to_string ( char const *&& v )

Definition at line 128 of file StringTransform.cpp.

                                      {
   return std::string(v);
 }

template<>

std::string to_string ( std::string && v )

Definition at line 133 of file StringTransform.cpp.

                                    {
   return std::move(v);
 }