#include <FileReader.h>

Inheritance diagram for foreign_storage::LocalMultiFileReader:

Collaboration diagram for foreign_storage::LocalMultiFileReader:

Public Member Functions
	LocalMultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const shared::FilePathOptions &options, const std::optional< size_t > &max_file_count)

	LocalMultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const rapidjson::Value &value)

void	checkForMoreRows (size_t file_offset, const shared::FilePathOptions &options, const ForeignServer server_options, const UserMapping user_mapping) override

Public Member Functions inherited from foreign_storage::MultiFileReader
	MultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

	MultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const rapidjson::Value &value)

size_t	getRemainingSize () override

bool	isRemainingSizeKnown () override

size_t	read (void *buffer, size_t max_size) override

size_t	readRegion (void *buffer, size_t offset, size_t size) override

bool	isScanFinished () const override

void	serialize (rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const override

FirstLineByFilePath	getFirstLineForEachFile () const override

bool	isEndOfLastFile () override

std::string	getCurrentFilePath () const override

virtual std::set< std::string >	checkForRolledOffFiles (const shared::FilePathOptions &file_path_options)

Public Member Functions inherited from foreign_storage::FileReader
	FileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

virtual	~FileReader ()=default

Private Member Functions
std::vector< std::string >	getAllFilePaths (const shared::FilePathOptions &file_path_options) const override

void	insertFile (std::string location)

Additional Inherited Members
Protected Attributes inherited from foreign_storage::MultiFileReader
std::vector< std::unique_ptr < FileReader > >	files_

std::vector< std::string >	file_locations_

std::vector< size_t >	cumulative_sizes_

size_t	current_index_

size_t	current_offset_

size_t	starting_offset_

bool	is_end_of_last_file_

Protected Attributes inherited from foreign_storage::FileReader
import_export::CopyParams	copy_params_

std::string	file_path_

Detailed Description

Definition at line 396 of file FileReader.h.

Constructor & Destructor Documentation

foreign_storage::LocalMultiFileReader::LocalMultiFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params,
		const shared::FilePathOptions &	options,
		const std::optional< size_t > &	max_file_count
	)

Definition at line 631 of file FileReader.cpp.

References insertFile(), and shared::local_glob_filter_sort_files().

     : MultiFileReader(file_path, copy_params) {
   auto file_paths = shared::local_glob_filter_sort_files(file_path, options);
   if (max_file_count.has_value() && file_paths.size() > max_file_count.value()) {
     file_paths.erase(file_paths.begin(),
                      file_paths.begin() + (file_paths.size() - max_file_count.value()));
   }
   for (const auto& file_path : file_paths) {
     insertFile(file_path);
   }
 }

Here is the call graph for this function:

foreign_storage::LocalMultiFileReader::LocalMultiFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params,
		const rapidjson::Value &	value
	)

Definition at line 646 of file FileReader.cpp.

References foreign_storage::FileReader::copy_params_, foreign_storage::MultiFileReader::file_locations_, foreign_storage::MultiFileReader::files_, and shared::is_compressed_file_extension().

     : MultiFileReader(file_path, copy_params, value) {
   // Constructs file from files_metadata
   for (size_t index = 0; index < file_locations_.size(); index++) {
     if (shared::is_compressed_file_extension(file_locations_[index])) {
       files_.emplace_back(std::make_unique<CompressedFileReader>(
           file_locations_[index],
           copy_params_,
           value["files_metadata"].GetArray()[index]));
     } else {
       files_.emplace_back(std::make_unique<SingleTextFileReader>(
           file_locations_[index],
           copy_params_,
           value["files_metadata"].GetArray()[index]));
     }
   }
 }

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::LocalMultiFileReader::checkForMoreRows	(	size_t	file_offset,
		const shared::FilePathOptions &	options,
		const ForeignServer *	server_options,
		const UserMapping *	user_mapping
	)

overridevirtual

Rescan the target files Throws an exception if the rescan fails (ie files are not in a valid appended state or not supported)

Parameters

file_offset	- where to resume the scan from (end of the last row) as not all of the bytes may have been consumed by the upstream compoennet
server_options	- only needed for S3 backed files
user_mapping	- only needed for S3 backed files

Reimplemented from foreign_storage::FileReader.

Definition at line 680 of file FileReader.cpp.

                                      {
   // Look for new files
   std::set<std::string> new_locations;
   CHECK(isScanFinished());
   CHECK(file_offset == current_offset_);
   if (boost::filesystem::is_directory(file_path_)) {
     // Find all files in this directory
     auto all_file_paths = getAllFilePaths(file_path_options);
     for (const auto& path : all_file_paths) {
       if (!shared::contains(file_locations_, path)) {
         new_locations.insert(path);
       }
     }
 
     for (const auto& file_path : file_locations_) {
       if (!shared::contains(all_file_paths, file_path)) {
         throw_removed_file_error(file_path);
       }
     }
   }
 
   if (!files_.empty()) {
     // Check if last file has new data
     size_t base = starting_offset_;
     CHECK_GT(current_index_, size_t(0));
     auto last_file_index = current_index_ - 1;
     if (last_file_index > 0) {
       base = cumulative_sizes_[last_file_index - 1];
     }
     files_.back()->checkForMoreRows(current_offset_ - base, file_path_options);
     if (!files_.back()->isScanFinished()) {
       // Go back to the last file, if more rows are found.
       current_index_ = last_file_index;
       is_end_of_last_file_ = false;
       cumulative_sizes_.pop_back();
     }
   }
 
   if (new_locations.size() > 0) {
     for (const auto& location : new_locations) {
       insertFile(location);
     }
   }
 }

Here is the call graph for this function:

std::vector< std::string > foreign_storage::LocalMultiFileReader::getAllFilePaths ( const shared::FilePathOptions & file_path_options ) const

overrideprivatevirtual

Implements foreign_storage::MultiFileReader.

Definition at line 729 of file FileReader.cpp.

References foreign_storage::FileReader::file_path_, and shared::local_glob_filter_sort_files().

Referenced by checkForMoreRows().

                                                           {
   return shared::local_glob_filter_sort_files(file_path_, file_path_options);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::LocalMultiFileReader::insertFile ( std::string location )

private

Definition at line 666 of file FileReader.cpp.

References foreign_storage::FileReader::copy_params_, foreign_storage::MultiFileReader::file_locations_, foreign_storage::MultiFileReader::files_, and shared::is_compressed_file_extension().

Referenced by checkForMoreRows(), and LocalMultiFileReader().

                                                         {
   if (shared::is_compressed_file_extension(location)) {
     files_.emplace_back(std::make_unique<CompressedFileReader>(location, copy_params_));
   } else {
     files_.emplace_back(std::make_unique<SingleTextFileReader>(location, copy_params_));
   }
   if (files_.back()->isScanFinished()) {
     // skip any initially empty files
     files_.pop_back();
   } else {
     file_locations_.push_back(location);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/FileReader.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/FileReader.cpp

Public Member Functions

Private Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation