foreign_storage::LogFileBufferParser Class Reference

#include <LogFileBufferParser.h>

Inheritance diagram for foreign_storage::LogFileBufferParser:

Collaboration diagram for foreign_storage::LogFileBufferParser:

Public Member Functions
	LogFileBufferParser (const ForeignTable *foreign_table, int32_t db_id)
Public Member Functions inherited from foreign_storage::RegexFileBufferParser
	RegexFileBufferParser (const ForeignTable *foreign_table)
ParseBufferResult	parseBuffer (ParseBufferRequest &request, bool convert_data_blocks, bool columns_are_pre_filtered=false, bool skip_dict_encoding=false) const override
import_export::CopyParams	validateAndGetCopyParams (const ForeignTable *foreign_table) const override
size_t	findRowEndPosition (size_t &alloc_size, std::unique_ptr< char[]> &buffer, size_t &buffer_size, const import_export::CopyParams &copy_params, const size_t buffer_first_row_index, unsigned int &num_rows_in_buffer, FileReader *file_reader) const override
void	validateFiles (const FileReader *file_reader, const ForeignTable *foreign_table) const override

Protected Member Functions
bool	regexMatchColumns (const std::string &row_str, const boost::regex &line_regex, size_t logical_column_count, std::vector< std::string > &parsed_columns_str, std::vector< std::string_view > &parsed_columns_sv, const std::string &file_path) const override
bool	shouldRemoveNonMatches () const override
bool	shouldTruncateStringValues () const override

Private Attributes
const ForeignTable *	foreign_table_
const int32_t	db_id_

Additional Inherited Members
Static Public Member Functions inherited from foreign_storage::RegexFileBufferParser
static void	setMaxBufferResize (size_t max_buffer_resize)
Static Public Member Functions inherited from foreign_storage::TextFileBufferParser
static std::map< int, DataBlockPtr >	convertImportBuffersToDataBlocks (const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const bool skip_dict_encoding=false)
static bool	isCoordinateScalar (const std::string_view datum)
static void	processGeoColumn (std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, size_t &col_idx, const import_export::CopyParams &copy_params, std::list< const ColumnDescriptor * >::iterator &cd_it, std::vector< std::string_view > &row, size_t &import_idx, bool is_null, size_t first_row_index, size_t row_index_plus_one, std::shared_ptr< Catalog_Namespace::Catalog > catalog)
static void	fillRejectedRowWithInvalidData (const std::list< const ColumnDescriptor * > &columns, std::list< const ColumnDescriptor * >::iterator &cd_it, const size_t col_idx, ParseBufferRequest &request)
static bool	isNullDatum (const std::string_view datum, const ColumnDescriptor *column, const std::string &null_indicator)
Static Public Attributes inherited from foreign_storage::RegexFileBufferParser
static const std::string	LINE_REGEX_KEY = "LINE_REGEX"
static const std::string	LINE_START_REGEX_KEY = "LINE_START_REGEX"
static const std::string	HEADER_KEY = "HEADER"
Static Public Attributes inherited from foreign_storage::TextFileBufferParser
static const std::string	BUFFER_SIZE_KEY = "BUFFER_SIZE"

Detailed Description

Definition at line 22 of file LogFileBufferParser.h.

Constructor & Destructor Documentation

foreign_storage::LogFileBufferParser::LogFileBufferParser	(	const ForeignTable *	foreign_table,
		int32_t	db_id
	)

Definition at line 126 of file LogFileBufferParser.cpp.

    : RegexFileBufferParser(foreign_table)
    , foreign_table_(foreign_table)
    , db_id_(db_id) {}

Member Function Documentation

bool foreign_storage::LogFileBufferParser::regexMatchColumns ( const std::string &  row_str, const boost::regex &  line_regex, size_t  logical_column_count, std::vector< std::string > &  parsed_columns_str, std::vector< std::string_view > &  parsed_columns_sv, const std::string &  file_path  ) const

overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 131 of file LogFileBufferParser.cpp.

References foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::add_column_value(), foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::add_nonce_values(), CHECK, CHECK_EQ, CHECK_GT, foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::create_map_from_arrays(), db_id_, foreign_table_, foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::get_node_name(), foreign_storage::RegexFileBufferParser::regexMatchColumns(), TableDescriptor::tableId, and TableDescriptor::tableName.

                                      {
  CHECK(parsed_columns_str.empty());
  CHECK(parsed_columns_sv.empty());
  CHECK(foreign_table_);
  if (foreign_table_->tableName == "request_logs") {
    boost::smatch match;
    bool set_all_nulls{false};
    if (boost::regex_match(row_str, match, line_regex)) {
      auto matched_column_count = match.size() - 1;
      // Last 2 matched columns are associative arrays.
      CHECK_GT(matched_column_count, size_t(2));
      for (size_t i = 1; i < match.size() - 2; i++) {
        add_column_value(parsed_columns_str, parsed_columns_sv, match[i].str());
      }
      // Special handling for associative arrays.
      auto values_map = create_map_from_arrays(match[matched_column_count - 1].str(),
                                               match[matched_column_count].str());
      static const std::array<std::string, 5> keys{
          "query_str", "client", "nonce", "execution_time_ms", "total_time_ms"};
      CHECK_EQ(logical_column_count, matched_column_count + keys.size());
      for (const auto& key : keys) {
        auto it = values_map.find(key);
        if (it == values_map.end()) {
          if (key == "nonce") {
            // Null dashboard id, dashboard name, and chart id.
            add_column_value(parsed_columns_str, parsed_columns_sv, "", 3);
          } else {
            // Add null value for missing entry.
            add_column_value(parsed_columns_str, parsed_columns_sv, "");
          }
        } else {
          if (key == "nonce") {
            add_nonce_values(parsed_columns_str,
                             parsed_columns_sv,
                             it->second,
                             foreign_table_->tableId,
                             db_id_);
          } else {
            add_column_value(parsed_columns_str, parsed_columns_sv, it->second);
          }
        }
      }
      CHECK_EQ(parsed_columns_str.size(), parsed_columns_sv.size());
    } else {
      parsed_columns_str.clear();
      parsed_columns_sv =
          std::vector<std::string_view>(logical_column_count, std::string_view{});
      set_all_nulls = true;
    }
    CHECK_EQ(parsed_columns_sv.size(), logical_column_count) << "In row: " << row_str;
    return set_all_nulls;
  } else {
    // Add fixed value for the server_logs table "node" column.
    if (foreign_table_->tableName == "server_logs") {
      add_column_value(parsed_columns_str, parsed_columns_sv, get_node_name());
    }
    return RegexFileBufferParser::regexMatchColumns(row_str,
                                                    line_regex,
                                                    logical_column_count,
                                                    parsed_columns_str,
                                                    parsed_columns_sv,
                                                    file_path);
  }
}

Here is the call graph for this function:

bool foreign_storage::LogFileBufferParser::shouldRemoveNonMatches ( ) const

overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 202 of file LogFileBufferParser.cpp.

                                                       {
  return true;
}

bool foreign_storage::LogFileBufferParser::shouldTruncateStringValues ( ) const

overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 206 of file LogFileBufferParser.cpp.

                                                           {
  return true;
}

Member Data Documentation

const int32_t foreign_storage::LogFileBufferParser::db_id_

private

Definition at line 40 of file LogFileBufferParser.h.

Referenced by regexMatchColumns().

const ForeignTable* foreign_storage::LogFileBufferParser::foreign_table_

private

Definition at line 39 of file LogFileBufferParser.h.

Referenced by regexMatchColumns().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/LogFileBufferParser.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/LogFileBufferParser.cpp