#include <S3Archive.h>

Inheritance diagram for S3Archive:

Collaboration diagram for S3Archive:

Public Member Functions
	S3Archive (const std::string &url, const bool plain_text)

	S3Archive (const std::string &url, const std::string &s3_access_key, const std::string &s3_secret_key, const std::string &s3_session_token, const std::string &s3_region, const std::string &s3_endpoint, const bool plain_text, const std::optional< std::string > &regex_path_filter, const std::optional< std::string > &file_sort_order_by, const std::optional< std::string > &file_sort_regex, const std::string &s3_temp_dir_path={})

	~S3Archive () override

void	init_for_read () override

const std::vector< std::string > &	get_objkeys ()

const std::string	land (const std::string &objkey, std::exception_ptr &teptr, const bool for_detection)

void	vacuum (const std::string &objkey)

size_t	get_total_file_size () const

Public Member Functions inherited from Archive
	Archive (const std::string url, const bool plain_text)

virtual	~Archive ()

virtual std::string	archive_error (int err)

virtual bool	read_next_header ()

virtual bool	read_data_block (const void *buff, size_t size, int64_t *offset)

virtual int64_t	get_position_compressed () const

virtual int	open ()

virtual int	close ()

virtual ptrdiff_t	read (const void **buff)

const std::string	url_part (const int i)

std::string	entryName ()

Private Attributes
std::string	s3_access_key

std::string	s3_secret_key

std::string	s3_session_token

std::string	s3_region

std::string	s3_endpoint

std::string	s3_temp_dir

std::string	bucket_name

std::string	prefix_name

std::optional< std::string >	regex_path_filter

std::optional< std::string >	file_sort_order_by

std::optional< std::string >	file_sort_regex

std::vector< std::string >	objkeys

std::map< const std::string, const std::string >	file_paths

size_t	total_file_size {0}

Additional Inherited Members
Static Public Member Functions inherited from Archive
static ptrdiff_t	read (struct archive a, void client_data, const void **buff)

static int	open (struct archive a, void client_data)

static int	close (struct archive a, void client_data)

static void	parse_url (const std::string url, std::map< int, std::string > &url_parts)

Protected Attributes inherited from Archive
std::string	url

std::map< int, std::string >	url_parts

archive *	ar = 0

archive_entry *	entry

bool	plain_text

Detailed Description

Definition at line 41 of file S3Archive.h.

Constructor & Destructor Documentation

S3Archive::S3Archive	(	const std::string &	url,
		const bool	plain_text
	)

inline

Definition at line 43 of file S3Archive.h.

References s3_access_key, s3_endpoint, s3_region, s3_secret_key, and s3_session_token.

                                                          : Archive(url, plain_text) {
     // these envs are on server side so are global settings
     // which make few senses in case of private s3 resources
     char* env;
     if (0 != (env = getenv("AWS_REGION"))) {
       s3_region = env;
     }
     if (0 != (env = getenv("AWS_ACCESS_KEY_ID"))) {
       s3_access_key = env;
     }
     if (0 != (env = getenv("AWS_SECRET_ACCESS_KEY"))) {
       s3_secret_key = env;
     }
     if (0 != (env = getenv("AWS_SESSION_TOKEN"))) {
       s3_session_token = env;
     }
 
     if (0 != (env = getenv("AWS_ENDPOINT"))) {
       s3_endpoint = env;
     }
   }

S3Archive::S3Archive	(	const std::string &	url,
		const std::string &	s3_access_key,
		const std::string &	s3_secret_key,
		const std::string &	s3_session_token,
		const std::string &	s3_region,
		const std::string &	s3_endpoint,
		const bool	plain_text,
		const std::optional< std::string > &	regex_path_filter,
		const std::optional< std::string > &	file_sort_order_by,
		const std::optional< std::string > &	file_sort_regex,
		const std::string &	s3_temp_dir_path = `{}`
	)

inline

Definition at line 65 of file S3Archive.h.

                                                 {})
       : S3Archive(url, plain_text) {
     this->s3_access_key = s3_access_key;
     this->s3_secret_key = s3_secret_key;
     this->s3_session_token = s3_session_token;
     this->s3_region = s3_region;
     this->s3_endpoint = s3_endpoint;
     this->regex_path_filter = regex_path_filter;
     this->file_sort_order_by = file_sort_order_by;
     this->file_sort_regex = file_sort_regex;
 
     if (s3_temp_dir_path.empty()) {
       // this must be local to heavydb not client
       // or posix dir path accessible to heavydb
       auto env_s3_temp_dir = getenv("TMPDIR");
       s3_temp_dir = env_s3_temp_dir ? env_s3_temp_dir : "/tmp";
     } else {
       s3_temp_dir = s3_temp_dir_path;
     }
   }

S3Archive::~S3Archive ( )

inlineoverride

Definition at line 96 of file S3Archive.h.

                         {
 #ifdef HAVE_AWS_S3
     for (auto& thread : threads) {
       if (thread.joinable()) {
         thread.join();
       }
     }
 #endif  // HAVE_AWS_S3
   }

Member Function Documentation

const std::vector<std::string>& S3Archive::get_objkeys ( )

inline

Definition at line 113 of file S3Archive.h.

References objkeys.

                                             {
     return objkeys;
   }

size_t S3Archive::get_total_file_size ( ) const

inline

Definition at line 133 of file S3Archive.h.

References total_file_size.

                                      {
     return total_file_size;
   }

void S3Archive::init_for_read ( )

inlineoverridevirtual

Reimplemented from Archive.

Definition at line 109 of file S3Archive.h.

                                 {
     throw std::runtime_error("AWS S3 support not available");
   }

const std::string S3Archive::land	(	const std::string &	objkey,
		std::exception_ptr &	teptr,
		const bool	for_detection
	)

inline

Definition at line 124 of file S3Archive.h.

                                                    {
     throw std::runtime_error("AWS S3 support not available");
   }

void S3Archive::vacuum ( const std::string & objkey )

inline

Definition at line 129 of file S3Archive.h.

                                        {
     throw std::runtime_error("AWS S3 support not available");
   }

Member Data Documentation

std::string S3Archive::bucket_name

private

Definition at line 153 of file S3Archive.h.

std::map<const std::string, const std::string> S3Archive::file_paths

private

Definition at line 159 of file S3Archive.h.

std::optional<std::string> S3Archive::file_sort_order_by

private

Definition at line 156 of file S3Archive.h.

std::optional<std::string> S3Archive::file_sort_regex

private

Definition at line 157 of file S3Archive.h.

std::vector<std::string> S3Archive::objkeys

private

Definition at line 158 of file S3Archive.h.

Referenced by get_objkeys().

std::string S3Archive::prefix_name

private

Definition at line 154 of file S3Archive.h.

std::optional<std::string> S3Archive::regex_path_filter

private

Definition at line 155 of file S3Archive.h.

std::string S3Archive::s3_access_key

private

Definition at line 146 of file S3Archive.h.

Referenced by heavydb.thrift.ttypes.TCopyParams::read(), S3Archive(), and heavydb.thrift.ttypes.TCopyParams::write().

std::string S3Archive::s3_endpoint

private

Definition at line 150 of file S3Archive.h.

Referenced by heavydb.thrift.ttypes.TCopyParams::read(), S3Archive(), and heavydb.thrift.ttypes.TCopyParams::write().

std::string S3Archive::s3_region

private

Definition at line 149 of file S3Archive.h.

Referenced by heavydb.thrift.ttypes.TCopyParams::read(), S3Archive(), and heavydb.thrift.ttypes.TCopyParams::write().

std::string S3Archive::s3_secret_key

private

Definition at line 147 of file S3Archive.h.

Referenced by heavydb.thrift.ttypes.TCopyParams::read(), S3Archive(), and heavydb.thrift.ttypes.TCopyParams::write().

std::string S3Archive::s3_session_token

private

Definition at line 148 of file S3Archive.h.

Referenced by heavydb.thrift.ttypes.TCopyParams::read(), S3Archive(), and heavydb.thrift.ttypes.TCopyParams::write().

std::string S3Archive::s3_temp_dir

private

Definition at line 151 of file S3Archive.h.

size_t S3Archive::total_file_size {0}

private

Definition at line 160 of file S3Archive.h.

Referenced by get_total_file_size().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/Archive/S3Archive.h
/home/jenkins-slave/workspace/core-os-doxygen/Archive/S3Archive.cpp

Public Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation