com.mapd.utility.SQLImporter Class Reference

Collaboration diagram for com.mapd.utility.SQLImporter:

Static Public Member Functions
static void	main (String[] args)

Protected Attributes
String	session = null
Heavy.Client	client = null

Package Functions
void	doWork (String[] args)
void	executeQuery ()

Package Attributes
Db_vendor_types	vendor_types = null

Static Package Attributes
static final Logger	LOGGER = LoggerFactory.getLogger(SQLImporter.class)

Private Member Functions
void	run_init (Connection conn)
void	help (Options options)
void	checkDBTable (Connection otherdb_conn, ResultSetMetaData md) throws SQLException
void	verifyColumnSignaturesMatch (Connection otherdb_conn, List< TColumnType > dstColumns, ResultSetMetaData srcColumns) throws SQLException
void	createDBTable (Connection otherdb_conn, ResultSetMetaData metaData)
void	createDBConnection ()
List< TColumnType >	getColumnInfo (String tName)
boolean	tableExists (String tName)
void	executeDBCommand (String sql)
String	getColType (int cType, int precision, int scale)
TColumn	setupBinaryColumn (int i, ResultSetMetaData md, int bufferSize) throws SQLException
void	setColValue (ResultSet rs, TColumn col, int columnType, int colNum, int scale, String colTypeName) throws SQLException
void	resetBinaryColumn (int i, ResultSetMetaData md, int bufferSize, TColumn col) throws SQLException

Private Attributes
CommandLine	cmd = null
DateTimeUtils	dateTimeUtils

Detailed Description

Definition at line 321 of file SQLImporter.java.

Member Function Documentation

void com.mapd.utility.SQLImporter.checkDBTable ( Connection  otherdb_conn, ResultSetMetaData  md  ) throws SQLException

inlineprivate

Definition at line 530 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.createDBConnection(), com.mapd.utility.SQLImporter.createDBTable(), com.mapd.utility.SQLImporter.executeDBCommand(), com.mapd.utility.SQLImporter.getColumnInfo(), com.mapd.utility.SQLImporter.tableExists(), and com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch().

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    createDBConnection();
    String tName = cmd.getOptionValue("targetTable");

    if (tableExists(tName)) {
      // check if we want to truncate
      if (cmd.hasOption("truncate")) {
        executeDBCommand("Drop table " + tName);
        createDBTable(otherdb_conn, md);
      } else {
        List<TColumnType> columnInfo = getColumnInfo(tName);
        verifyColumnSignaturesMatch(otherdb_conn, columnInfo, md);
      }
    } else {
      createDBTable(otherdb_conn, md);
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createDBConnection ( )

inlineprivate

Definition at line 722 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.client, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkDBTable().

                                    {
    TTransport transport = null;
    TProtocol protocol = new TBinaryProtocol(transport);
    int port = Integer.valueOf(cmd.getOptionValue("port", "6274"));
    String server = cmd.getOptionValue("server", "localhost");
    try {
      // Uses default certificate stores.
      boolean load_trust_store = cmd.hasOption("https");
      SockTransportProperties skT = null;
      if (cmd.hasOption("https")) {
        skT = SockTransportProperties.getEncryptedClientDefaultTrustStore(
                !cmd.hasOption("insecure"));
        transport = skT.openHttpsClientTransport(server, port);
        transport.open();
        protocol = new TJSONProtocol(transport);
      } else if (cmd.hasOption("http")) {
        skT = SockTransportProperties.getUnencryptedClient();
        transport = skT.openHttpClientTransport(server, port);
        protocol = new TJSONProtocol(transport);
      } else {
        skT = SockTransportProperties.getUnencryptedClient();
        transport = skT.openClientTransport(server, port);
        transport.open();
        protocol = new TBinaryProtocol(transport);
      }

      client = new Heavy.Client(protocol);
      // This if will be useless until PKI signon
      if (cmd.hasOption("user")) {
        session = client.connect(cmd.getOptionValue("user", "admin"),
                cmd.getOptionValue("passwd", "HyperInteractive"),
                cmd.getOptionValue("database", "omnisci"));
      }
      LOGGER.debug("Connected session is " + session);

    } catch (TTransportException ex) {
      LOGGER.error("Connection failed - " + ex.toString());
      exit(1);
    } catch (TDBException ex) {
      LOGGER.error("Connection failed - " + ex.getError_msg());
      exit(2);
    } catch (TException ex) {
      LOGGER.error("Connection failed - " + ex.toString());
      exit(3);
    } catch (Exception ex) {
      LOGGER.error("General exception - " + ex.toString());
      exit(4);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createDBTable ( Connection  otherdb_conn, ResultSetMetaData  metaData  )

inlineprivate

Definition at line 680 of file SQLImporter.java.

References File_Namespace.append(), com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.executeDBCommand(), com.mapd.utility.SQLImporter.getColType(), Integer, and run_benchmark_import.type.

Referenced by com.mapd.utility.SQLImporter.checkDBTable().

                                                                                  {
    StringBuilder sb = new StringBuilder();
    sb.append("Create table ").append(cmd.getOptionValue("targetTable")).append("(");

    // Now iterate the metadata
    try {
      for (int i = 1; i <= metaData.getColumnCount(); i++) {
        if (i > 1) {
          sb.append(",");
        }
        LOGGER.debug("Column name is " + metaData.getColumnName(i));
        LOGGER.debug("Column type is " + metaData.getColumnTypeName(i));
        LOGGER.debug("Column type is " + metaData.getColumnType(i));

        sb.append(metaData.getColumnName(i)).append(" ");
        int col_type = metaData.getColumnType(i);
        if (col_type == java.sql.Types.OTHER) {
          Db_vendor_types.GisType type =
                  vendor_types.find_gis_type(otherdb_conn, metaData, i);
          sb.append(Db_vendor_types.gis_type_to_str(type));
        } else {
          sb.append(getColType(metaData.getColumnType(i),
                  metaData.getPrecision(i),
                  metaData.getScale(i)));
        }
      }
      sb.append(")");

      if (Integer.valueOf(cmd.getOptionValue("fragmentSize", "0")) > 0) {
        sb.append(" with (fragment_size = ");
        sb.append(cmd.getOptionValue("fragmentSize", "0"));
        sb.append(")");
      }

    } catch (SQLException ex) {
      LOGGER.error("Error processing the metadata - " + ex.toString());
      exit(1);
    }

    executeDBCommand(sb.toString());
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.doWork ( String[]  args )

inlinepackage

Definition at line 337 of file SQLImporter.java.

References run_benchmark_import.args, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.executeQuery().

                             {
    // create Options object

    SQLImporter_args s_args = new SQLImporter_args();

    try {
      cmd = s_args.parse(args);
    } catch (ParseException ex) {
      LOGGER.error(ex.getLocalizedMessage());
      s_args.printHelpMessage();
      exit(0);
    }
    executeQuery();
  }

Here is the call graph for this function:

void com.mapd.utility.SQLImporter.executeDBCommand ( String  sql )

inlineprivate

Definition at line 807 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkDBTable(), and com.mapd.utility.SQLImporter.createDBTable().

                                            {
    LOGGER.info("Run Command - " + sql);

    try {
      TQueryResult sqlResult = client.sql_execute(session, sql + ";", true, null, -1, -1);
    } catch (TDBException ex) {
      LOGGER.error("SQL Execute failed - " + ex.getError_msg());
      exit(1);
    } catch (TException ex) {
      LOGGER.error("SQL Execute failed - " + ex.toString());
      exit(1);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.executeQuery ( )

inlinepackage

Definition at line 352 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.checkDBTable(), com.mapd.utility.SQLImporter.cmd, report.conn, com.mapd.utility.db_vendors.Db_vendor_types.isAutoCommitDisabledRequired(), com.mapd.utility.SQLImporter.resetBinaryColumn(), com.mapd.utility.SQLImporter.run_init(), com.mapd.utility.SQLImporter.session, com.mapd.utility.SQLImporter.setColValue(), com.mapd.utility.SQLImporter.setupBinaryColumn(), and com.mapd.utility.SQLImporter.vendor_types.

Referenced by com.mapd.utility.SQLImporter.doWork().

                      {
    Connection conn = null;
    Statement stmt = null;

    long totalTime = 0;

    try {
      // Open a connection
      if (cmd.hasOption("nlj")) {
        LOGGER.info("Connecting to source database.");
      } else {
        LOGGER.info("Connecting to database url :" + cmd.getOptionValue("jdbcConnect"));
      }
      conn = DriverManager.getConnection(cmd.getOptionValue("jdbcConnect"),
              cmd.getOptionValue("sourceUser"),
              cmd.getOptionValue("sourcePasswd"));
      vendor_types = Db_vendor_types.Db_vendor_factory(cmd.getOptionValue("jdbcConnect"));
      long startTime = System.currentTimeMillis();

      // run init file script on targe DB if present
      if (cmd.hasOption("initializeFile")) {
        run_init(conn);
      }

      try {
        if (vendor_types.isAutoCommitDisabledRequired()) {
          conn.setAutoCommit(false);
        }
      } catch (SQLException se) {
        LOGGER.warn(
                "SQLException when attempting to setAutoCommit to false, jdbc driver probably doesnt support it.  Error is "
                + se.toString());
      }

      // Execute a query
      stmt = conn.createStatement();

      int bufferSize = Integer.valueOf(cmd.getOptionValue("bufferSize", "10000"));
      // set the jdbc fetch buffer size to reduce the amount of records being moved to
      // java from postgress
      stmt.setFetchSize(bufferSize);
      long timer;

      ResultSet rs = stmt.executeQuery(cmd.getOptionValue("sqlStmt"));

      // check if table already exists and is compatible in HEAVYAI with the query
      // metadata
      ResultSetMetaData md = rs.getMetaData();
      checkDBTable(conn, md);

      timer = System.currentTimeMillis();

      long resultCount = 0;
      int bufferCount = 0;
      long total = 0;

      List<TColumn> cols = new ArrayList(md.getColumnCount());
      for (int i = 1; i <= md.getColumnCount(); i++) {
        TColumn col = setupBinaryColumn(i, md, bufferSize);
        cols.add(col);
      }

      // read data from old DB
      while (rs.next()) {
        for (int i = 1; i <= md.getColumnCount(); i++) {
          setColValue(rs,
                  cols.get(i - 1),
                  md.getColumnType(i),
                  i,
                  md.getScale(i),
                  md.getColumnTypeName(i));
        }
        resultCount++;
        bufferCount++;
        if (bufferCount == bufferSize) {
          bufferCount = 0;
          // send the buffer to HEAVY.AI
          client.load_table_binary_columnar(
                  session, cmd.getOptionValue("targetTable"), cols, null);
          // recreate columnar store for use
          for (int i = 1; i <= md.getColumnCount(); i++) {
            resetBinaryColumn(i, md, bufferSize, cols.get(i - 1));
          }

          if (resultCount % 100000 == 0) {
            LOGGER.info("Imported " + resultCount + " records");
          }
        }
      }
      if (bufferCount > 0) {
        // send the LAST buffer to HEAVY.AI
        client.load_table_binary_columnar(
                session, cmd.getOptionValue("targetTable"), cols, null);
        bufferCount = 0;
      }

      LOGGER.info("result set count is " + resultCount + " read time is "
              + (System.currentTimeMillis() - timer) + "ms");

      // Clean-up environment
      rs.close();
      stmt.close();
      conn.close();

      totalTime = System.currentTimeMillis() - startTime;
    } catch (SQLException se) {
      LOGGER.error("SQLException - " + se.toString());
      se.printStackTrace();
    } catch (TDBException ex) {
      LOGGER.error("TDBException - " + ex.getError_msg());
      ex.printStackTrace();
    } catch (TException ex) {
      LOGGER.error("TException failed - " + ex.toString());
      ex.printStackTrace();
    } finally {
      // finally block used to close resources
      try {
        if (stmt != null) {
          stmt.close();
        }
      } catch (SQLException se2) {
      } // nothing we can do
      try {
        if (conn != null) {
          conn.close();
        }
      } catch (SQLException se) {
        LOGGER.error("SQlException in close - " + se.toString());
        se.printStackTrace();
      }
      try {
        if (session != null) {
          client.disconnect(session);
        }
      } catch (TDBException ex) {
        LOGGER.error("TDBException - in finalization " + ex.getError_msg());
        ex.printStackTrace();
      } catch (TException ex) {
        LOGGER.error("TException - in finalization" + ex.toString());
        ex.printStackTrace();
      }
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

String com.mapd.utility.SQLImporter.getColType ( int  cType, int  precision, int  scale  )

inlineprivate

Definition at line 821 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createDBTable().

                                                                 {
    // Note - if cType is OTHER a earlier call will have been made
    // to try and work out the db vendors specific type.
    if (precision > 19) {
      precision = 19;
    }
    if (scale > 19) {
      scale = 18;
    }
    switch (cType) {
      case java.sql.Types.TINYINT:
        return ("TINYINT");
      case java.sql.Types.SMALLINT:
        return ("SMALLINT");
      case java.sql.Types.INTEGER:
        return ("INTEGER");
      case java.sql.Types.BIGINT:
        return ("BIGINT");
      case java.sql.Types.FLOAT:
        return ("FLOAT");
      case java.sql.Types.DECIMAL:
        return ("DECIMAL(" + precision + "," + scale + ")");
      case java.sql.Types.DOUBLE:
        return ("DOUBLE");
      case java.sql.Types.REAL:
        return ("REAL");
      case java.sql.Types.NUMERIC:
        return ("NUMERIC(" + precision + "," + scale + ")");
      case java.sql.Types.TIME:
        return ("TIME");
      case java.sql.Types.TIMESTAMP:
        return ("TIMESTAMP");
      case java.sql.Types.DATE:
        return ("DATE");
      case java.sql.Types.BOOLEAN:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
        return ("BOOLEAN");
      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
        return ("TEXT ENCODING DICT");
      default:
        throw new AssertionError("Column type " + cType + " not Supported");
    }
  }

Here is the caller graph for this function:

List<TColumnType> com.mapd.utility.SQLImporter.getColumnInfo ( String  tName )

inlineprivate

Definition at line 772 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkDBTable().

                                                        {
    LOGGER.debug("Getting columns for  " + tName);
    List<TColumnType> row_descriptor = null;
    try {
      TTableDetails table_details = client.get_table_details(session, tName);
      row_descriptor = table_details.row_desc;
    } catch (TDBException ex) {
      LOGGER.error("column check failed - " + ex.getError_msg());
      exit(3);
    } catch (TException ex) {
      LOGGER.error("column check failed - " + ex.toString());
      exit(3);
    }
    return row_descriptor;
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.help ( Options  options )

inlineprivate

Definition at line 523 of file SQLImporter.java.

                                     {
    // automatically generate the help statement
    HelpFormatter formatter = new HelpFormatter();
    formatter.setOptionComparator(null); // get options in the order they are created
    formatter.printHelp("SQLImporter", options);
  }

static void com.mapd.utility.SQLImporter.main ( String[]  args )

inlinestatic

Definition at line 332 of file SQLImporter.java.

References run_benchmark_import.args.

                                         {
    SQLImporter sq = new SQLImporter();
    sq.doWork(args);
  }

void com.mapd.utility.SQLImporter.resetBinaryColumn ( int  i, ResultSetMetaData  md, int  bufferSize, TColumn  col  ) throws SQLException

inlineprivate

Definition at line 1043 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    col.nulls.clear();

    switch (md.getColumnType(i)) {
      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
      case java.sql.Types.TIME:
      case java.sql.Types.TIMESTAMP:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
      case java.sql.Types.DATE:
      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        col.data.int_col.clear();
        break;

      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        col.data.real_col.clear();
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
      case java.sql.Types.OTHER:
        col.data.str_col.clear();
        break;
      default:
        throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.run_init ( Connection  conn )

inlineprivate

Definition at line 496 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, and parse_ast.line.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                                         {
    // attempt to open file
    String line = "";
    try {
      BufferedReader reader =
              new BufferedReader(new FileReader(cmd.getOptionValue("initializeFile")));
      Statement stmt = conn.createStatement();
      while ((line = reader.readLine()) != null) {
        if (line.isEmpty()) {
          continue;
        }
        LOGGER.info("Running : " + line);
        stmt.execute(line);
      }
      stmt.close();
      reader.close();
    } catch (IOException e) {
      LOGGER.error("Exception occurred trying to read initialize file: "
              + cmd.getOptionValue("initFile"));
      exit(1);
    } catch (SQLException e) {
      LOGGER.error(
              "Exception occurred trying to execute initialize file entry : " + line);
      exit(1);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.setColValue ( ResultSet  rs, TColumn  col, int  columnType, int  colNum, int  scale, String  colTypeName  ) throws SQLException

inlineprivate

Definition at line 917 of file SQLImporter.java.

References heavydb.dtypes.Date, Double, and heavydb.dtypes.Time.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                                                  {
    switch (columnType) {
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
        Boolean b = rs.getBoolean(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(b ? 1L : 0L);
        }
        break;

      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        BigDecimal bd = rs.getBigDecimal(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(bd.multiply(new BigDecimal(pow(10L, scale))).longValue());
        }
        break;

      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
        Long l = rs.getLong(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(new Long(0));
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(l);
        }
        break;

      case java.sql.Types.TIME:
        Time t = rs.getTime(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(t.getTime()));
          col.nulls.add(Boolean.FALSE);
        }

        break;
      case java.sql.Types.TIMESTAMP:
        Timestamp ts = rs.getTimestamp(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(ts.getTime()));
          col.nulls.add(Boolean.FALSE);
        }

        break;
      case java.sql.Types.DATE:
        Date d = rs.getDate(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(d.getTime()));
          col.nulls.add(Boolean.FALSE);
        }
        break;
      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        Double db = rs.getDouble(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.real_col.add(new Double(0));

        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.real_col.add(db);
        }
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
        String strVal = rs.getString(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.str_col.add("");

        } else {
          col.data.str_col.add(strVal);
          col.nulls.add(Boolean.FALSE);
        }
        break;
      case java.sql.Types.OTHER:
        Object objVal = rs.getObject(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.str_col.add("");
        } else {
          col.data.str_col.add(vendor_types.get_wkt(rs, colNum, colTypeName));
          col.nulls.add(Boolean.FALSE);
        }
        break;
      default:
        throw new AssertionError("Column type " + columnType + " not Supported");
    }
  }

Here is the caller graph for this function:

TColumn com.mapd.utility.SQLImporter.setupBinaryColumn ( int  i, ResultSetMetaData  md, int  bufferSize  ) throws SQLException

inlineprivate

Definition at line 871 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    TColumn col = new TColumn();

    col.nulls = new ArrayList<Boolean>(bufferSize);

    col.data = new TColumnData();

    switch (md.getColumnType(i)) {
      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
      case java.sql.Types.TIME:
      case java.sql.Types.TIMESTAMP:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
      case java.sql.Types.DATE:
      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        col.data.int_col = new ArrayList<Long>(bufferSize);
        break;

      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        col.data.real_col = new ArrayList<Double>(bufferSize);
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
      case java.sql.Types.OTHER:
        col.data.str_col = new ArrayList<String>(bufferSize);
        break;

      default:
        throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
    }
    return col;
  }

Here is the caller graph for this function:

boolean com.mapd.utility.SQLImporter.tableExists ( String  tName )

inlineprivate

Definition at line 788 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkDBTable().

                                            {
    LOGGER.debug("Check for table " + tName);
    try {
      List<String> recv_get_tables = client.get_tables(session);
      for (String s : recv_get_tables) {
        if (s.equals(tName)) {
          return true;
        }
      }
    } catch (TDBException ex) {
      LOGGER.error("Table check failed - " + ex.getError_msg());
      exit(3);
    } catch (TException ex) {
      LOGGER.error("Table check failed - " + ex.toString());
      exit(3);
    }
    return false;
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch ( Connection  otherdb_conn, List< TColumnType >  dstColumns, ResultSetMetaData  srcColumns  ) throws SQLException

inlineprivate

Definition at line 549 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd.

Referenced by com.mapd.utility.SQLImporter.checkDBTable().

                                                            {
    if (srcColumns.getColumnCount() != dstColumns.size()) {
      LOGGER.error("Table sizes do not match: Destination " + dstColumns.size()
              + " versus Source " + srcColumns.getColumnCount());
      exit(1);
    }
    for (int i = 1; i <= dstColumns.size(); ++i) {
      if (!dstColumns.get(i - 1).getCol_name().equalsIgnoreCase(
                  srcColumns.getColumnName(i))) {
        LOGGER.error(
                "Destination table does not have matching column in same order for column number "
                + i + " destination column name is " + dstColumns.get(i - 1).col_name
                + " versus target column " + srcColumns.getColumnName(i));
        exit(1);
      }
      TDatumType dstType = dstColumns.get(i - 1).getCol_type().getType();
      int dstPrecision = dstColumns.get(i - 1).getCol_type().getPrecision();
      int dstScale = dstColumns.get(i - 1).getCol_type().getScale();
      int srcType = srcColumns.getColumnType(i);
      int srcPrecision = srcColumns.getPrecision(i);
      int srcScale = srcColumns.getScale(i);

      boolean match = false;
      switch (srcType) {
        case java.sql.Types.TINYINT:
          match |= dstType == TDatumType.TINYINT;
          // NOTE: it's okay to import smaller type to a bigger one,
          // so we just fall through and try to match the next type.
          // But the order of case statements is important here!
        case java.sql.Types.SMALLINT:
          match |= dstType == TDatumType.SMALLINT;
        case java.sql.Types.INTEGER:
          match |= dstType == TDatumType.INT;
        case java.sql.Types.BIGINT:
          match |= dstType == TDatumType.BIGINT;
          if (cmd.hasOption("AllowIntegerNarrowing")) {
            match |= dstType == TDatumType.TINYINT || dstType == TDatumType.SMALLINT
                    || dstType == TDatumType.INT;
          }
          break;
        case java.sql.Types.DECIMAL:
        case java.sql.Types.NUMERIC:
          match = dstType == TDatumType.DECIMAL && dstPrecision == srcPrecision
                  && dstScale == srcScale;
          break;
        case java.sql.Types.FLOAT:
        case java.sql.Types.REAL:
          match |= dstType == TDatumType.FLOAT;
          // Fall through and try double
        case java.sql.Types.DOUBLE:
          match |= dstType == TDatumType.DOUBLE;
          if (cmd.hasOption("AllowDoubleToFloat")) {
            match |= dstType == TDatumType.FLOAT;
          }
          break;
        case java.sql.Types.TIME:
          match = dstType == TDatumType.TIME;
          break;
        case java.sql.Types.TIMESTAMP:
          match = dstType == TDatumType.TIMESTAMP;
          break;
        case java.sql.Types.DATE:
          match = dstType == TDatumType.DATE;
          break;
        case java.sql.Types.BOOLEAN:
        case java.sql.Types
                .BIT: // deal with postgres treating boolean as bit... this will bite me
          match = dstType == TDatumType.BOOL;
          break;
        case java.sql.Types.NVARCHAR:
        case java.sql.Types.VARCHAR:
        case java.sql.Types.NCHAR:
        case java.sql.Types.CHAR:
        case java.sql.Types.LONGVARCHAR:
        case java.sql.Types.LONGNVARCHAR:
          match = (dstType == TDatumType.STR || dstType == TDatumType.POINT
                  || dstType == TDatumType.POLYGON || dstType == TDatumType.MULTIPOLYGON
                  || dstType == TDatumType.LINESTRING
                  || dstType == TDatumType.MULTILINESTRING
                  || dstType == TDatumType.MULTIPOINT);
          break;
        case java.sql.Types.OTHER:
          // NOTE: I ignore subtypes (geography vs geopetry vs none) here just because
          // it makes no difference for OmniSciDB at the moment
          Db_vendor_types.GisType gisType =
                  vendor_types.find_gis_type(otherdb_conn, srcColumns, i);
          if (gisType.srid != dstScale) {
            match = false;
            break;
          }
          switch (dstType) {
            case POINT:
              match = gisType.type.equalsIgnoreCase("POINT");
              break;
            case MULTIPOINT:
              match = gisType.type.equalsIgnoreCase("MULTIPOINT");
              break;
            case LINESTRING:
              match = gisType.type.equalsIgnoreCase("LINESTRING");
              break;
            case MULTILINESTRING:
              match = gisType.type.equalsIgnoreCase("MULTILINESTRING");
              break;
            case POLYGON:
              match = gisType.type.equalsIgnoreCase("POLYGON");
              break;
            case MULTIPOLYGON:
              match = gisType.type.equalsIgnoreCase("MULTIPOLYGON");
              break;
            default:
              LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
                      + " not Supported");
              exit(1);
          }
          break;
        default:
          LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
                  + " not Supported");
          exit(1);
      }
      if (!match) {
        LOGGER.error("Source and destination types for column "
                + srcColumns.getColumnName(i)
                + " do not match. Please make sure that type, precision and scale are exactly the same");
        exit(1);
      }
    }
  }

Here is the caller graph for this function:

Member Data Documentation

Heavy.Client com.mapd.utility.SQLImporter.client = null

protected

Definition at line 323 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createDBConnection().

CommandLine com.mapd.utility.SQLImporter.cmd = null

private

Definition at line 324 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.checkDBTable(), com.mapd.utility.SQLImporter.createDBConnection(), com.mapd.utility.SQLImporter.createDBTable(), com.mapd.utility.SQLImporter.doWork(), com.mapd.utility.SQLImporter.executeQuery(), com.mapd.utility.SQLImporter.run_init(), and com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch().

DateTimeUtils com.mapd.utility.SQLImporter.dateTimeUtils

private

Initial value:

= (milliseconds) -> {
    return milliseconds / 1000;
  }

Definition at line 326 of file SQLImporter.java.

final Logger com.mapd.utility.SQLImporter.LOGGER = LoggerFactory.getLogger(SQLImporter.class)

staticpackage

Definition at line 325 of file SQLImporter.java.

String com.mapd.utility.SQLImporter.session = null

protected

Definition at line 322 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createDBConnection(), com.mapd.utility.SQLImporter.executeDBCommand(), com.mapd.utility.SQLImporter.executeQuery(), com.mapd.utility.SQLImporter.getColumnInfo(), heavydb.thrift.Heavy.disconnect_args.read(), heavydb.thrift.Heavy.switch_database_args.read(), heavydb.thrift.Heavy.clone_session_args.read(), heavydb.thrift.Heavy.get_server_status_args.read(), heavydb.thrift.Heavy.get_status_args.read(), heavydb.thrift.Heavy.get_hardware_info_args.read(), heavydb.thrift.Heavy.get_tables_args.read(), heavydb.thrift.Heavy.get_tables_for_database_args.read(), heavydb.thrift.Heavy.get_physical_tables_args.read(), heavydb.thrift.Heavy.get_views_args.read(), heavydb.thrift.Heavy.get_tables_meta_args.read(), heavydb.thrift.Heavy.get_table_details_args.read(), heavydb.thrift.Heavy.get_table_details_for_database_args.read(), heavydb.thrift.Heavy.get_internal_table_details_args.read(), heavydb.thrift.Heavy.get_internal_table_details_for_database_args.read(), heavydb.thrift.Heavy.get_users_args.read(), heavydb.thrift.Heavy.get_databases_args.read(), heavydb.thrift.Heavy.start_heap_profile_args.read(), heavydb.thrift.Heavy.stop_heap_profile_args.read(), heavydb.thrift.Heavy.get_heap_profile_args.read(), heavydb.thrift.Heavy.get_memory_args.read(), heavydb.thrift.Heavy.clear_cpu_memory_args.read(), heavydb.thrift.Heavy.clear_gpu_memory_args.read(), heavydb.thrift.Heavy.set_table_epoch_args.read(), heavydb.thrift.Heavy.set_table_epoch_by_name_args.read(), heavydb.thrift.Heavy.get_table_epoch_args.read(), heavydb.thrift.Heavy.get_table_epoch_by_name_args.read(), heavydb.thrift.Heavy.get_table_epochs_args.read(), heavydb.thrift.Heavy.set_table_epochs_args.read(), heavydb.thrift.Heavy.get_session_info_args.read(), heavydb.thrift.Heavy.get_queries_info_args.read(), heavydb.thrift.Heavy.set_leaf_info_args.read(), heavydb.thrift.Heavy.sql_execute_args.read(), heavydb.thrift.Heavy.sql_execute_df_args.read(), heavydb.thrift.Heavy.sql_execute_gdf_args.read(), heavydb.thrift.Heavy.deallocate_df_args.read(), heavydb.thrift.Heavy.sql_validate_args.read(), heavydb.thrift.Heavy.get_completion_hints_args.read(), heavydb.thrift.Heavy.set_execution_mode_args.read(), heavydb.thrift.Heavy.render_vega_args.read(), heavydb.thrift.Heavy.get_result_row_for_pixel_args.read(), heavydb.thrift.Heavy.create_custom_expression_args.read(), heavydb.thrift.Heavy.get_custom_expressions_args.read(), heavydb.thrift.Heavy.update_custom_expression_args.read(), heavydb.thrift.Heavy.delete_custom_expressions_args.read(), heavydb.thrift.Heavy.get_dashboard_args.read(), heavydb.thrift.Heavy.get_dashboards_args.read(), heavydb.thrift.Heavy.create_dashboard_args.read(), heavydb.thrift.Heavy.replace_dashboard_args.read(), heavydb.thrift.Heavy.delete_dashboard_args.read(), heavydb.thrift.Heavy.share_dashboards_args.read(), heavydb.thrift.Heavy.delete_dashboards_args.read(), heavydb.thrift.Heavy.share_dashboard_args.read(), heavydb.thrift.Heavy.unshare_dashboard_args.read(), heavydb.thrift.Heavy.unshare_dashboards_args.read(), heavydb.thrift.Heavy.get_dashboard_grantees_args.read(), heavydb.thrift.Heavy.get_link_view_args.read(), heavydb.thrift.Heavy.create_link_args.read(), heavydb.thrift.Heavy.load_table_binary_args.read(), heavydb.thrift.Heavy.load_table_binary_columnar_args.read(), heavydb.thrift.Heavy.load_table_binary_arrow_args.read(), heavydb.thrift.Heavy.load_table_args.read(), heavydb.thrift.Heavy.detect_column_types_args.read(), heavydb.thrift.Heavy.create_table_args.read(), heavydb.thrift.Heavy.import_table_args.read(), heavydb.thrift.Heavy.import_geo_table_args.read(), heavydb.thrift.Heavy.import_table_status_args.read(), heavydb.thrift.Heavy.get_first_geo_file_in_archive_args.read(), heavydb.thrift.Heavy.get_all_files_in_archive_args.read(), heavydb.thrift.Heavy.get_layers_in_geo_file_args.read(), heavydb.thrift.Heavy.query_get_outer_fragment_count_args.read(), heavydb.thrift.Heavy.check_table_consistency_args.read(), heavydb.thrift.Heavy.start_render_query_args.read(), heavydb.thrift.Heavy.insert_data_args.read(), heavydb.thrift.Heavy.insert_chunks_args.read(), heavydb.thrift.Heavy.checkpoint_args.read(), heavydb.thrift.Heavy.get_roles_args.read(), heavydb.thrift.Heavy.get_db_objects_for_grantee_args.read(), heavydb.thrift.Heavy.get_db_object_privs_args.read(), heavydb.thrift.Heavy.get_all_roles_for_user_args.read(), heavydb.thrift.Heavy.get_all_effective_roles_for_user_args.read(), heavydb.thrift.Heavy.has_role_args.read(), heavydb.thrift.Heavy.has_object_privilege_args.read(), heavydb.thrift.Heavy.set_license_key_args.read(), heavydb.thrift.Heavy.get_license_claims_args.read(), heavydb.thrift.Heavy.get_device_parameters_args.read(), heavydb.thrift.Heavy.register_runtime_extension_functions_args.read(), heavydb.thrift.Heavy.get_table_function_names_args.read(), heavydb.thrift.Heavy.get_runtime_table_function_names_args.read(), heavydb.thrift.Heavy.get_table_function_details_args.read(), heavydb.thrift.Heavy.get_function_names_args.read(), heavydb.thrift.Heavy.get_runtime_function_names_args.read(), heavydb.thrift.Heavy.get_function_details_args.read(), com.mapd.utility.SQLImporter.tableExists(), heavydb.thrift.Heavy.disconnect_args.write(), heavydb.thrift.Heavy.switch_database_args.write(), heavydb.thrift.Heavy.clone_session_args.write(), heavydb.thrift.Heavy.get_server_status_args.write(), heavydb.thrift.Heavy.get_status_args.write(), heavydb.thrift.Heavy.get_hardware_info_args.write(), heavydb.thrift.Heavy.get_tables_args.write(), heavydb.thrift.Heavy.get_tables_for_database_args.write(), heavydb.thrift.Heavy.get_physical_tables_args.write(), heavydb.thrift.Heavy.get_views_args.write(), heavydb.thrift.Heavy.get_tables_meta_args.write(), heavydb.thrift.Heavy.get_table_details_args.write(), heavydb.thrift.Heavy.get_table_details_for_database_args.write(), heavydb.thrift.Heavy.get_internal_table_details_args.write(), heavydb.thrift.Heavy.get_internal_table_details_for_database_args.write(), heavydb.thrift.Heavy.get_users_args.write(), heavydb.thrift.Heavy.get_databases_args.write(), heavydb.thrift.Heavy.start_heap_profile_args.write(), heavydb.thrift.Heavy.stop_heap_profile_args.write(), heavydb.thrift.Heavy.get_heap_profile_args.write(), heavydb.thrift.Heavy.get_memory_args.write(), heavydb.thrift.Heavy.clear_cpu_memory_args.write(), heavydb.thrift.Heavy.clear_gpu_memory_args.write(), heavydb.thrift.Heavy.set_table_epoch_args.write(), heavydb.thrift.Heavy.set_table_epoch_by_name_args.write(), heavydb.thrift.Heavy.get_table_epoch_args.write(), heavydb.thrift.Heavy.get_table_epoch_by_name_args.write(), heavydb.thrift.Heavy.get_table_epochs_args.write(), heavydb.thrift.Heavy.set_table_epochs_args.write(), heavydb.thrift.Heavy.get_session_info_args.write(), heavydb.thrift.Heavy.get_queries_info_args.write(), heavydb.thrift.Heavy.set_leaf_info_args.write(), heavydb.thrift.Heavy.sql_execute_args.write(), heavydb.thrift.Heavy.sql_execute_df_args.write(), heavydb.thrift.Heavy.sql_execute_gdf_args.write(), heavydb.thrift.Heavy.deallocate_df_args.write(), heavydb.thrift.Heavy.sql_validate_args.write(), heavydb.thrift.Heavy.get_completion_hints_args.write(), heavydb.thrift.Heavy.set_execution_mode_args.write(), heavydb.thrift.Heavy.render_vega_args.write(), heavydb.thrift.Heavy.get_result_row_for_pixel_args.write(), heavydb.thrift.Heavy.create_custom_expression_args.write(), heavydb.thrift.Heavy.get_custom_expressions_args.write(), heavydb.thrift.Heavy.update_custom_expression_args.write(), heavydb.thrift.Heavy.delete_custom_expressions_args.write(), heavydb.thrift.Heavy.get_dashboard_args.write(), heavydb.thrift.Heavy.get_dashboards_args.write(), heavydb.thrift.Heavy.create_dashboard_args.write(), heavydb.thrift.Heavy.replace_dashboard_args.write(), heavydb.thrift.Heavy.delete_dashboard_args.write(), heavydb.thrift.Heavy.share_dashboards_args.write(), heavydb.thrift.Heavy.delete_dashboards_args.write(), heavydb.thrift.Heavy.share_dashboard_args.write(), heavydb.thrift.Heavy.unshare_dashboard_args.write(), heavydb.thrift.Heavy.unshare_dashboards_args.write(), heavydb.thrift.Heavy.get_dashboard_grantees_args.write(), heavydb.thrift.Heavy.get_link_view_args.write(), heavydb.thrift.Heavy.create_link_args.write(), heavydb.thrift.Heavy.load_table_binary_args.write(), heavydb.thrift.Heavy.load_table_binary_columnar_args.write(), heavydb.thrift.Heavy.load_table_binary_arrow_args.write(), heavydb.thrift.Heavy.load_table_args.write(), heavydb.thrift.Heavy.detect_column_types_args.write(), heavydb.thrift.Heavy.create_table_args.write(), heavydb.thrift.Heavy.import_table_args.write(), heavydb.thrift.Heavy.import_geo_table_args.write(), heavydb.thrift.Heavy.import_table_status_args.write(), heavydb.thrift.Heavy.get_first_geo_file_in_archive_args.write(), heavydb.thrift.Heavy.get_all_files_in_archive_args.write(), heavydb.thrift.Heavy.get_layers_in_geo_file_args.write(), heavydb.thrift.Heavy.query_get_outer_fragment_count_args.write(), heavydb.thrift.Heavy.check_table_consistency_args.write(), heavydb.thrift.Heavy.start_render_query_args.write(), heavydb.thrift.Heavy.insert_data_args.write(), heavydb.thrift.Heavy.insert_chunks_args.write(), heavydb.thrift.Heavy.checkpoint_args.write(), heavydb.thrift.Heavy.get_roles_args.write(), heavydb.thrift.Heavy.get_db_objects_for_grantee_args.write(), heavydb.thrift.Heavy.get_db_object_privs_args.write(), heavydb.thrift.Heavy.get_all_roles_for_user_args.write(), heavydb.thrift.Heavy.get_all_effective_roles_for_user_args.write(), heavydb.thrift.Heavy.has_role_args.write(), heavydb.thrift.Heavy.has_object_privilege_args.write(), heavydb.thrift.Heavy.set_license_key_args.write(), heavydb.thrift.Heavy.get_license_claims_args.write(), heavydb.thrift.Heavy.get_device_parameters_args.write(), heavydb.thrift.Heavy.register_runtime_extension_functions_args.write(), heavydb.thrift.Heavy.get_table_function_names_args.write(), heavydb.thrift.Heavy.get_runtime_table_function_names_args.write(), heavydb.thrift.Heavy.get_table_function_details_args.write(), heavydb.thrift.Heavy.get_function_names_args.write(), heavydb.thrift.Heavy.get_runtime_function_names_args.write(), and heavydb.thrift.Heavy.get_function_details_args.write().

Db_vendor_types com.mapd.utility.SQLImporter.vendor_types = null

package

Definition at line 330 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

---

The documentation for this class was generated from the following file:

• /home/jenkins-slave/workspace/core-os-doxygen/java/utility/src/main/java/com/mapd/utility/SQLImporter.java