OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Functions
RowToColumnLoader.cpp File Reference

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table. More...

#include "ImportExport/RowToColumnLoader.h"
#include "ImportExport/DelimitedParserUtils.h"
#include "Logger/Logger.h"
#include <chrono>
#include <thread>
+ Include dependency graph for RowToColumnLoader.cpp:

Go to the source code of this file.

Functions

SQLTypes get_sql_types (const TColumnType &ct)
 
SQLTypeInfo create_sql_type_info_from_col_type (const TColumnType &ct)
 
SQLTypeInfo create_array_sql_type_info_from_col_type (const TColumnType &ct)
 
void remove_partial_row (size_t failed_column, std::vector< SQLTypeInfo > column_type_info_vector, std::vector< TColumn > &input_col_vec)
 
void populate_TColumn (TStringValue ts, SQLTypeInfo column_type_info, TColumn &input_col, const import_export::CopyParams &copy_params)
 

Detailed Description

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table.

Definition in file RowToColumnLoader.cpp.

Function Documentation

SQLTypeInfo create_array_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 108 of file RowToColumnLoader.cpp.

References get_sql_types(), kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

108  {
109  return SQLTypeInfo(get_sql_types(ct),
110  ct.col_type.precision,
111  ct.col_type.scale,
112  ct.col_type.nullable,
113  kENCODING_NONE,
114  0,
115  SQLTypes::kNULLT);
116 }
get_sql_types
SQLTypes get_sql_types(const TColumnType &ct)
Definition: RowToColumnLoader.cpp:33

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypeInfo create_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 82 of file RowToColumnLoader.cpp.

References get_sql_types(), kARRAY, kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

82  {
83  if (ct.col_type.is_array) {
84  return SQLTypeInfo(SQLTypes::kARRAY,
85  ct.col_type.precision,
86  ct.col_type.scale,
87  ct.col_type.nullable,
88  kENCODING_NONE,
89  0,
90  get_sql_types(ct));
91  } else {
92  // normal column
93  // NOTE(se)
94  // for geo types, the values inserted for the other fields
95  // may not be valid, but only the type field is ever used
96  return SQLTypeInfo(get_sql_types(ct),
97  ct.col_type.precision,
98  ct.col_type.scale,
99  ct.col_type.nullable,
100  kENCODING_NONE,
101  0,
102  SQLTypes::kNULLT);
103  }
104 }
get_sql_types
SQLTypes get_sql_types(const TColumnType &ct)
Definition: RowToColumnLoader.cpp:33

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes get_sql_types ( const TColumnType &  ct)

Definition at line 33 of file RowToColumnLoader.cpp.

References logger::FATAL, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNULLT, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and LOG.

Referenced by create_array_sql_type_info_from_col_type(), and create_sql_type_info_from_col_type().

33  {
34  switch (ct.col_type.type) {
35  case TDatumType::BIGINT:
36  return SQLTypes::kBIGINT;
37  case TDatumType::BOOL:
38  return SQLTypes::kBOOLEAN;
39  case TDatumType::DATE:
40  return SQLTypes::kDATE;
41  case TDatumType::DECIMAL:
42  return SQLTypes::kDECIMAL;
43  case TDatumType::DOUBLE:
44  return SQLTypes::kDOUBLE;
45  case TDatumType::FLOAT:
46  return SQLTypes::kFLOAT;
47  case TDatumType::INT:
48  return SQLTypes::kINT;
49  case TDatumType::STR:
50  // Tdataum is lossy here so need to look at precision to see if it was defined
51  if (ct.col_type.precision == 0) {
52  return SQLTypes::kTEXT;
53  } else {
54  return SQLTypes::kVARCHAR;
55  }
56  case TDatumType::TIME:
57  return SQLTypes::kTIME;
58  case TDatumType::TIMESTAMP:
59  return SQLTypes::kTIMESTAMP;
60  case TDatumType::SMALLINT:
61  return SQLTypes::kSMALLINT;
62  case TDatumType::TINYINT:
63  return SQLTypes::kTINYINT;
64  case TDatumType::POINT:
65  return SQLTypes::kPOINT;
66  case TDatumType::MULTIPOINT:
67  return SQLTypes::kMULTIPOINT;
68  case TDatumType::LINESTRING:
69  return SQLTypes::kLINESTRING;
70  case TDatumType::MULTILINESTRING:
71  return SQLTypes::kMULTILINESTRING;
72  case TDatumType::POLYGON:
73  return SQLTypes::kPOLYGON;
74  case TDatumType::MULTIPOLYGON:
75  return SQLTypes::kMULTIPOLYGON;
76  default:
77  LOG(FATAL) << "Unsupported TColumnType found, should not be possible";
78  return SQLTypes::kNULLT; // satisfy return-type warning
79  }
80 }
kTIME
Definition: sqltypes.h:76
LOG
#define LOG(tag)
Definition: Logger.h:285
kTEXT
Definition: sqltypes.h:79
kDATE
Definition: sqltypes.h:80
kINT
Definition: sqltypes.h:72

+ Here is the caller graph for this function:

void populate_TColumn ( TStringValue  ts,
SQLTypeInfo  column_type_info,
TColumn &  input_col,
const import_export::CopyParams &  copy_params 
)

Definition at line 184 of file RowToColumnLoader.cpp.

References Datum::bigintval, Datum::doubleval, logger::FATAL, Datum::floatval, SQLTypeInfo::get_precision(), SQLTypeInfo::get_type(), Datum::intval, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, LOG, Datum::smallintval, StringToDatum(), and Datum::tinyintval.

Referenced by RowToColumnLoader::convert_string_to_column().

187  {
188  // create datum and push data to column structure from row data
189  switch (column_type_info.get_type()) {
190  case SQLTypes::kARRAY:
191  LOG(FATAL) << "Trying to process ARRAY at item level something is wrong";
192  break;
193  case SQLTypes::kTEXT:
194  case SQLTypes::kCHAR:
195  case SQLTypes::kVARCHAR:
196  case SQLTypes::kPOINT:
197  case SQLTypes::kMULTIPOINT:
198  case SQLTypes::kLINESTRING:
199  case SQLTypes::kMULTILINESTRING:
200  case SQLTypes::kPOLYGON:
201  case SQLTypes::kMULTIPOLYGON:
202  if (ts.is_null) {
203  input_col.nulls.push_back(true);
204  input_col.data.str_col.emplace_back("");
205 
206  } else {
207  input_col.nulls.push_back(false);
208  switch (column_type_info.get_type()) {
209  case SQLTypes::kCHAR:
210  case SQLTypes::kVARCHAR:
211  input_col.data.str_col.push_back(
212  ts.str_val.substr(0, column_type_info.get_precision()));
213  break;
214  case SQLTypes::kTEXT:
215  case SQLTypes::kPOINT:
216  case SQLTypes::kMULTIPOINT:
217  case SQLTypes::kLINESTRING:
218  case SQLTypes::kMULTILINESTRING:
219  case SQLTypes::kPOLYGON:
220  case SQLTypes::kMULTIPOLYGON:
221  input_col.data.str_col.push_back(ts.str_val);
222  break;
223  default:
224  LOG(FATAL) << " trying to process a STRING transport type not handled "
225  << column_type_info.get_type();
226  }
227  }
228  break;
229  case SQLTypes::kINT:
230  case SQLTypes::kBIGINT:
231  case SQLTypes::kSMALLINT:
232  case SQLTypes::kTINYINT:
233  case SQLTypes::kDATE:
234  case SQLTypes::kTIME:
235  case SQLTypes::kTIMESTAMP:
236  case SQLTypes::kNUMERIC:
237  case SQLTypes::kDECIMAL:
238  case SQLTypes::kBOOLEAN:
239  if (ts.is_null) {
240  input_col.nulls.push_back(true);
241  input_col.data.int_col.push_back(0);
242  } else {
243  input_col.nulls.push_back(false);
244  Datum d = StringToDatum(ts.str_val, column_type_info);
245  switch (column_type_info.get_type()) {
246  case SQLTypes::kINT:
247  case SQLTypes::kBOOLEAN:
248  input_col.data.int_col.push_back(d.intval);
249  break;
250  case SQLTypes::kBIGINT:
251  case SQLTypes::kNUMERIC:
252  case SQLTypes::kDECIMAL:
253  input_col.data.int_col.push_back(d.bigintval);
254  break;
255  case SQLTypes::kSMALLINT:
256  input_col.data.int_col.push_back(d.smallintval);
257  break;
258  case SQLTypes::kTINYINT:
259  input_col.data.int_col.push_back(d.tinyintval);
260  break;
261  case SQLTypes::kDATE:
262  case SQLTypes::kTIME:
263  case SQLTypes::kTIMESTAMP:
264  input_col.data.int_col.push_back(d.bigintval);
265  break;
266  default:
267  LOG(FATAL) << " trying to process an INT transport type not handled "
268  << column_type_info.get_type();
269  }
270  }
271  break;
272  case SQLTypes::kFLOAT:
273  case SQLTypes::kDOUBLE:
274  if (ts.is_null) {
275  input_col.nulls.push_back(true);
276  input_col.data.real_col.push_back(0);
277 
278  } else {
279  input_col.nulls.push_back(false);
280  Datum d = StringToDatum(ts.str_val, column_type_info);
281  switch (column_type_info.get_type()) {
282  case SQLTypes::kFLOAT:
283  input_col.data.real_col.push_back(d.floatval);
284  break;
285  case SQLTypes::kDOUBLE:
286  input_col.data.real_col.push_back(d.doubleval);
287  break;
288  default:
289  LOG(FATAL) << " trying to process a REAL transport type not handled "
290  << column_type_info.get_type();
291  }
292  }
293  break;
294  default:
295  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
296  }
297 }
Datum::tinyintval
int8_t tinyintval
Definition: Datum.h:73
kTIME
Definition: sqltypes.h:76
LOG
#define LOG(tag)
Definition: Logger.h:285
SQLTypeInfo::get_type
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
Datum::intval
int32_t intval
Definition: Datum.h:75
Datum::floatval
float floatval
Definition: Datum.h:77
Datum::bigintval
int64_t bigintval
Definition: Datum.h:76
Datum::smallintval
int16_t smallintval
Definition: Datum.h:74
StringToDatum
Datum StringToDatum(const std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:339
SQLTypeInfo::get_precision
int get_precision() const
Definition: sqltypes.h:394
kTEXT
Definition: sqltypes.h:79
kDATE
Definition: sqltypes.h:80
kCHAR
Definition: sqltypes.h:68
kINT
Definition: sqltypes.h:72
Datum
Definition: Datum.h:71
Datum::doubleval
double doubleval
Definition: Datum.h:78

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void remove_partial_row ( size_t  failed_column,
std::vector< SQLTypeInfo >  column_type_info_vector,
std::vector< TColumn > &  input_col_vec 
)

Definition at line 136 of file RowToColumnLoader.cpp.

References logger::FATAL, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and LOG.

Referenced by RowToColumnLoader::convert_string_to_column().

138  {
139  for (size_t idx = 0; idx < failed_column; idx++) {
140  switch (column_type_info_vector[idx].get_type()) {
141  case SQLTypes::kARRAY:
142  input_col_vec[idx].nulls.pop_back();
143  input_col_vec[idx].data.arr_col.pop_back();
144  break;
145  case SQLTypes::kTEXT:
146  case SQLTypes::kCHAR:
147  case SQLTypes::kVARCHAR:
148  input_col_vec[idx].nulls.pop_back();
149  input_col_vec[idx].data.str_col.pop_back();
150  break;
151  case SQLTypes::kTINYINT:
152  case SQLTypes::kINT:
153  case SQLTypes::kBIGINT:
154  case SQLTypes::kSMALLINT:
155  case SQLTypes::kDATE:
156  case SQLTypes::kTIME:
157  case SQLTypes::kTIMESTAMP:
158  case SQLTypes::kNUMERIC:
159  case SQLTypes::kDECIMAL:
160  case SQLTypes::kBOOLEAN:
161  input_col_vec[idx].nulls.pop_back();
162  input_col_vec[idx].data.int_col.pop_back();
163  break;
164  case SQLTypes::kFLOAT:
165  case SQLTypes::kDOUBLE:
166  input_col_vec[idx].nulls.pop_back();
167  input_col_vec[idx].data.real_col.pop_back();
168  break;
169  case SQLTypes::kPOINT:
170  case SQLTypes::kMULTIPOINT:
171  case SQLTypes::kLINESTRING:
172  case SQLTypes::kMULTILINESTRING:
173  case SQLTypes::kPOLYGON:
174  case SQLTypes::kMULTIPOLYGON:
175  input_col_vec[idx].nulls.pop_back();
176  input_col_vec[idx].data.str_col.pop_back();
177  break;
178  default:
179  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
180  }
181  }
182 }
kTIME
Definition: sqltypes.h:76
LOG
#define LOG(tag)
Definition: Logger.h:285
kTEXT
Definition: sqltypes.h:79
kDATE
Definition: sqltypes.h:80
kCHAR
Definition: sqltypes.h:68
kINT
Definition: sqltypes.h:72

+ Here is the caller graph for this function: