#include <FileReader.h>

Inheritance diagram for foreign_storage::FileReader:

Collaboration diagram for foreign_storage::FileReader:

Public Member Functions
	FileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

virtual	~FileReader ()=default

virtual size_t	read (void *buffer, size_t max_size)=0

virtual bool	isScanFinished () const =0

virtual size_t	readRegion (void *buffer, size_t offset, size_t size)=0

virtual size_t	getRemainingSize ()=0

virtual bool	isRemainingSizeKnown ()=0

virtual void	checkForMoreRows (size_t file_offset, const shared::FilePathOptions &options, const ForeignServer server_options=nullptr, const UserMapping user_mapping=nullptr)

virtual void	serialize (rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const =0

virtual FirstLineByFilePath	getFirstLineForEachFile () const =0

virtual bool	isEndOfLastFile ()=0

virtual std::string	getCurrentFilePath () const =0

Protected Attributes
import_export::CopyParams	copy_params_

std::string	file_path_

Detailed Description

Definition at line 43 of file FileReader.h.

Constructor & Destructor Documentation

foreign_storage::FileReader::FileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params
	)

inline

Definition at line 45 of file FileReader.h.

46 : copy_params_(copy_params), file_path_(file_path){};

foreign_storage::FileReader::copy_params_

import_export::CopyParams copy_params_

Definition: FileReader.h:128

foreign_storage::FileReader::file_path_

std::string file_path_

Definition: FileReader.h:129

virtual foreign_storage::FileReader::~FileReader ( )

virtualdefault

Member Function Documentation

virtual void foreign_storage::FileReader::checkForMoreRows	(	size_t	file_offset,
		const shared::FilePathOptions &	options,
		const ForeignServer *	server_options = `nullptr`,
		const UserMapping *	user_mapping = `nullptr`
	)

inlinevirtual

Rescan the target files Throws an exception if the rescan fails (ie files are not in a valid appended state or not supported)

Parameters

file_offset	- where to resume the scan from (end of the last row) as not all of the bytes may have been consumed by the upstream compoennet
server_options	- only needed for S3 backed files
user_mapping	- only needed for S3 backed files

Reimplemented in foreign_storage::LocalMultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

Definition at line 93 of file FileReader.h.

                                                                            {
     throw std::runtime_error{"APPEND mode not yet supported for this table."};
   }

virtual std::string foreign_storage::FileReader::getCurrentFilePath ( ) const

pure virtual

Returns the path of the currently processed file.

Implemented in foreign_storage::MultiFileReader, and foreign_storage::SingleFileReader.

Referenced by foreign_storage::dispatch_scan_requests().

Here is the caller graph for this function:

virtual FirstLineByFilePath foreign_storage::FileReader::getFirstLineForEachFile ( ) const

pure virtual

Returns a map containing the first line for each file that will be read.

Implemented in foreign_storage::MultiFileReader, and foreign_storage::SingleFileReader.

Referenced by foreign_storage::RegexFileBufferParser::validateFiles().

Here is the caller graph for this function:

virtual size_t foreign_storage::FileReader::getRemainingSize ( )

pure virtual

Returns: size of the remaining content to be read

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

virtual bool foreign_storage::FileReader::isEndOfLastFile ( )

pure virtual

Returns a boolean indicating whether the reader is at the end of the last file that was read.

Implemented in foreign_storage::MultiFileReader, and foreign_storage::SingleFileReader.

Referenced by foreign_storage::RegexFileBufferParser::findRowEndPosition().

Here is the caller graph for this function:

virtual bool foreign_storage::FileReader::isRemainingSizeKnown ( )

pure virtual

Returns: if remaining size is known

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

virtual bool foreign_storage::FileReader::isScanFinished ( ) const

pure virtual

Returns: true if the entire file has been read

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

Referenced by foreign_storage::dispatch_scan_requests(), import_export::delimited_parser::find_row_end_pos(), foreign_storage::RegexFileBufferParser::findRowEndPosition(), foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::is_file_scan_finished(), and foreign_storage::SingleFileReader::isEndOfLastFile().

Here is the caller graph for this function:

virtual size_t foreign_storage::FileReader::read	(	void *	buffer,
		size_t	max_size
	)

pure virtual

Read up to max_size bytes from archive into buffer starting starting from the end of the last read

Parameters

buffer	- buffer to load into
max_size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

Referenced by foreign_storage::dispatch_scan_requests(), and import_export::delimited_parser::extend_buffer().

Here is the caller graph for this function:

virtual size_t foreign_storage::FileReader::readRegion	(	void *	buffer,
		size_t	offset,
		size_t	size
	)

pure virtual

Read up to max_size bytes from archive, starting at given offset isScanFinished() must return true to use readRegion

Parameters

buffer	- buffer to load into
offset	- starting point into the archive to read
size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

Referenced by foreign_storage::parse_file_regions().

Here is the caller graph for this function:

virtual void foreign_storage::FileReader::serialize	(	rapidjson::Value &	value,
		rapidjson::Document::AllocatorType &	allocator
	)		const

pure virtual

Serialize internal state to given json object This Json will later be used to restore the reader state through a constructor must be called when isScanFinished() is true

Parameters

value	- json object to store needed state to this function can store any needed data or none
allocator	- allocator to use for json contruction

Implemented in foreign_storage::MultiFileReader, foreign_storage::CompressedFileReader, and foreign_storage::SingleTextFileReader.

Member Data Documentation

import_export::CopyParams foreign_storage::FileReader::copy_params_

protected

Definition at line 128 of file FileReader.h.

std::string foreign_storage::FileReader::file_path_

protected

Definition at line 129 of file FileReader.h.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), foreign_storage::CompressedFileReader::checkForMoreRows(), foreign_storage::LocalMultiFileReader::checkForMoreRows(), foreign_storage::LocalMultiFileReader::getAllFilePaths(), foreign_storage::SingleFileReader::getCurrentFilePath(), foreign_storage::SingleTextFileReader::getFirstLine(), foreign_storage::CompressedFileReader::getFirstLine(), foreign_storage::SingleFileReader::getFirstLineForEachFile(), and foreign_storage::SingleTextFileReader::readRegion().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/FileReader.h

Public Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation