From 0c5d869af7fd7a13924de889003ad6b8ea1b5dc1 Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Thu, 14 Apr 2022 01:57:44 +0530 Subject: [PATCH 01/36] dbschema support and connection fixes --- .../postgres/CloudSQLPostgreSQLConnector.java | 37 +++++++++++++------ .../widgets/CloudSQLPostgreSQL-batchsink.json | 2 +- .../CloudSQLPostgreSQL-batchsource.json | 4 ++ .../config/AbstractDBSpecificSinkConfig.java | 2 +- .../plugin/db/batch/sink/AbstractDBSink.java | 30 ++++++++------- .../cdap/plugin/mssql/SqlServerConnector.java | 35 ++++++++++++------ mssql-plugin/widgets/SqlServer-batchsink.json | 2 +- .../cdap/plugin/oracle/OracleConnector.java | 27 +++++++++----- .../plugin/postgres/PostgresConnector.java | 28 ++++++++------ .../widgets/Postgres-batchsink.json | 5 +++ 10 files changed, 112 insertions(+), 60 deletions(-) diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index e7cbf121d..4eeeb3b66 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -30,6 +30,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -38,6 +39,7 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -57,6 +59,11 @@ public CloudSQLPostgreSQLConnector(CloudSQLPostgreSQLConnectorConfig config) { this.config = config; } + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + @Override public boolean supportSchema() { return true; @@ -90,22 +97,30 @@ protected String getTableQuery(String database, String schema, String table, int @Override protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, ConnectorSpec.Builder builder) { - Map properties = new HashMap<>(); - setConnectionProperties(properties, request); + Map sourceProperties = new HashMap<>(); + Map sinkProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + setConnectionProperties(sinkProperties, request); builder - .addRelatedPlugin(new PluginSpec(CloudSQLPostgreSQLConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, properties)) - .addRelatedPlugin(new PluginSpec(CloudSQLPostgreSQLConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, properties)); + .addRelatedPlugin(new PluginSpec(CloudSQLPostgreSQLConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, sourceProperties)) + .addRelatedPlugin(new PluginSpec(CloudSQLPostgreSQLConstants.PLUGIN_NAME, + BatchSink.PLUGIN_TYPE, sinkProperties)); + sinkProperties.put(ConnectionConfig.DATABASE, config.getDatabase()); + String schema = path.getSchema(); + if (schema != null) { + sinkProperties.put(CloudSQLPostgreSQLSink.CloudSQLPostgreSQLSinkConfig.DB_SCHEMA_NAME, schema); + } + sourceProperties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.NUM_SPLITS, "1"); String table = path.getTable(); if (table == null) { return; } - - properties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.IMPORT_QUERY, - getTableQuery(path.getDatabase(), path.getSchema(), path.getTable())); - properties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.NUM_SPLITS, "1"); - properties.put(ConnectionConfig.DATABASE, path.getDatabase()); - properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); - properties.put(CloudSQLPostgreSQLSink.CloudSQLPostgreSQLSinkConfig.TABLE_NAME, table); + sourceProperties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sinkProperties.put(CloudSQLPostgreSQLSink.CloudSQLPostgreSQLSinkConfig.TABLE_NAME, table); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + sinkProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); } } diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index f55b6b7f4..8607d9110 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -123,7 +123,7 @@ } }, { - "widget-type": "hidden", + "widget-type": "textbox", "label": "Schema Name", "name": "dbSchemaName" } diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 0b494e790..9e7c5e28b 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -214,6 +214,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "database" + }, { "type": "property", "name": "connectionArguments" diff --git a/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSinkConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSinkConfig.java index fb9804144..df1be75b7 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSinkConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSinkConfig.java @@ -51,7 +51,7 @@ public abstract class AbstractDBSpecificSinkConfig extends PluginConfig implemen @Description("Name of the database schema of table.") @Macro @Nullable - public String dbSchemaName; + private String dbSchemaName; @Override public String getTableName() { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/batch/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/batch/sink/AbstractDBSink.java index bbd2529e6..9181d9a55 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/batch/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/batch/sink/AbstractDBSink.java @@ -116,10 +116,12 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); + String dbSchemaName = dbSinkConfig.getDBSchemaName(); + String tableName = dbSinkConfig.getTableName(); LOG.debug("tableName = {}; schemaName = {}, pluginType = {}; pluginName = {}; connectionString = {};", - dbSinkConfig.getTableName(), - dbSinkConfig.getDBSchemaName(), + tableName, + dbSchemaName, ConnectionConfig.JDBC_PLUGIN_TYPE, dbSinkConfig.getJdbcPluginName(), connectionString); @@ -132,8 +134,8 @@ public void prepareRun(BatchSinkContext context) { try { if (Objects.nonNull(outputSchema)) { FailureCollector collector = context.getFailureCollector(); - validateSchema(collector, driverClass, dbSinkConfig.getTableName(), - outputSchema, dbSinkConfig.getDBSchemaName()); + validateSchema(collector, driverClass, tableName, + outputSchema, dbSchemaName); collector.getOrThrowException(); } else { outputSchema = inferSchema(driverClass); @@ -151,8 +153,8 @@ public void prepareRun(BatchSinkContext context) { configAccessor.setInitQueries(dbSinkConfig.getInitQueries()); configAccessor.getConfiguration().set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass.getName()); configAccessor.getConfiguration().set(DBConfiguration.URL_PROPERTY, connectionString); - String fullyQualifiedTableName = dbSinkConfig.getDBSchemaName() == null ? dbSinkConfig.getEscapedTableName() - : dbSinkConfig.getDBSchemaName() + "." + dbSinkConfig.getEscapedTableName(); + String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() + : dbSchemaName + "." + dbSinkConfig.getEscapedTableName(); configAccessor.getConfiguration().set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, fullyQualifiedTableName); configAccessor.getConfiguration().set(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, dbColumns); if (dbSinkConfig.getUser() != null) { @@ -205,8 +207,9 @@ public void initialize(BatchRuntimeContext context) throws Exception { private Schema inferSchema(Class driverClass) { List inferredFields = new ArrayList<>(); - String fullyQualifiedTableName = dbSinkConfig.getDBSchemaName() == null ? dbSinkConfig.getEscapedTableName() - : dbSinkConfig.getDBSchemaName() + "." + dbSinkConfig.getEscapedTableName(); + String dbSchemaName = dbSinkConfig.getDBSchemaName(); + String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() + : dbSchemaName + "." + dbSinkConfig.getEscapedTableName(); try { DBUtils.ensureJDBCDriverIsAvailable(driverClass, dbSinkConfig.getConnectionString(), dbSinkConfig.getJdbcPluginName()); @@ -255,8 +258,9 @@ public void destroy() { private void setResultSetMetadata() throws Exception { List columnTypes = new ArrayList<>(columns.size()); String connectionString = dbSinkConfig.getConnectionString(); - String fullyQualifiedTableName = dbSinkConfig.getDBSchemaName() == null ? dbSinkConfig.getEscapedTableName() - : dbSinkConfig.getDBSchemaName() + "." + dbSinkConfig.getEscapedTableName(); + String dbSchemaName = dbSinkConfig.getDBSchemaName(); + String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() + : dbSchemaName + "." + dbSinkConfig.getEscapedTableName(); driverCleanup = DBUtils .ensureJDBCDriverIsAvailable(driverClass, connectionString, dbSinkConfig.getJdbcPluginName()); @@ -305,8 +309,8 @@ static List getMatchedColumnTypeList(ResultSetMetaData resultSetMeta private void validateSchema(FailureCollector collector, Class jdbcDriverClass, String tableName, Schema inputSchema, String dbSchemaName) { String connectionString = dbSinkConfig.getConnectionString(); - String fullyQualifiedTableName = dbSinkConfig.getDBSchemaName() == null ? dbSinkConfig.getEscapedTableName() - : dbSinkConfig.getDBSchemaName() + "." + dbSinkConfig.getEscapedTableName(); + String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() + : dbSchemaName + "." + dbSinkConfig.getEscapedTableName(); try { DBUtils.ensureJDBCDriverIsAvailable(jdbcDriverClass, connectionString, dbSinkConfig.getJdbcPluginName()); } catch (IllegalAccessException | InstantiationException | SQLException e) { @@ -386,7 +390,7 @@ public abstract static class DBSinkConfig extends DBConfig implements DatabaseSi @Description("Name of the database schema of table.") @Macro @Nullable - public String dbSchemaName; + private String dbSchemaName; public String getTableName() { return tableName; diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index 527e17e32..9b75c2546 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -31,6 +31,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; import org.apache.hadoop.io.LongWritable; @@ -68,23 +69,33 @@ protected Class getDBRecordType() { protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, ConnectorSpec.Builder builder) { - Map properties = new HashMap<>(); - setConnectionProperties(properties, request); + Map sourceProperties = new HashMap<>(); + Map sinkProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + setConnectionProperties(sinkProperties, request); builder - .addRelatedPlugin(new PluginSpec(SqlServerConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, properties)) - .addRelatedPlugin(new PluginSpec(SqlServerConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, properties)); + .addRelatedPlugin(new PluginSpec(SqlServerConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, sourceProperties)) + .addRelatedPlugin(new PluginSpec(SqlServerConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); + + String database = path.getDatabase(); + if (database != null) { + sinkProperties.put(ConnectionConfig.DATABASE, database); + sourceProperties.put(ConnectionConfig.DATABASE, database); + } + String schema = path.getSchema(); + if (schema != null) { + sinkProperties.put(SqlServerSink.SqlServerSinkConfig.DB_SCHEMA_NAME, schema); + } + sourceProperties.put(SqlServerSource.SqlServerSourceConfig.NUM_SPLITS, "1"); String table = path.getTable(); if (table == null) { return; } - - properties.put(SqlServerSource.SqlServerSourceConfig.IMPORT_QUERY, getTableQuery(path.getDatabase(), - path.getSchema(), - path.getTable())); - properties.put(SqlServerSource.SqlServerSourceConfig.NUM_SPLITS, "1"); - properties.put(SqlServerSource.SqlServerSourceConfig.DATABASE, path.getDatabase()); - properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); - properties.put(SqlServerSink.SqlServerSinkConfig.TABLE_NAME, table); + sourceProperties.put(SqlServerSource.SqlServerSourceConfig.IMPORT_QUERY, + getTableQuery(database, schema, table)); + sinkProperties.put(SqlServerSink.SqlServerSinkConfig.TABLE_NAME, table); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + sinkProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); } @Override diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 98ce28d15..e83ae52d1 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -128,7 +128,7 @@ "name": "tableName" }, { - "widget-type": "hidden", + "widget-type": "textbox", "label": "Schema Name", "name": "dbSchemaName" } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index a404b7244..583ab60f8 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -72,22 +72,29 @@ protected Class getDBRecordType() { protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, ConnectorSpec.Builder builder) { - Map properties = new HashMap<>(); - setConnectionProperties(properties, request); + Map sourceProperties = new HashMap<>(); + Map sinkProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + setConnectionProperties(sinkProperties, request); builder - .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, properties)) - .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, properties)); + .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, sourceProperties)) + .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); + sinkProperties.put(OracleConstants.NAME_DATABASE, config.getDatabase()); + String schema = path.getSchema(); + if (schema != null) { + sinkProperties.put(OracleSink.OracleSinkConfig.DB_SCHEMA_NAME, schema); + } + sourceProperties.put(OracleSource.OracleSourceConfig.NUM_SPLITS, "1"); String table = path.getTable(); if (table == null) { return; } - - properties.put(OracleSource.OracleSourceConfig.IMPORT_QUERY, getTableQuery(path.getDatabase(), path.getSchema(), - path.getTable())); - properties.put(OracleSource.OracleSourceConfig.NUM_SPLITS, "1"); - properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); - properties.put(OracleSink.OracleSinkConfig.TABLE_NAME, table); + sourceProperties.put(OracleSource.OracleSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sinkProperties.put(OracleSink.OracleSinkConfig.TABLE_NAME, table); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + sinkProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); } @Override diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index ca035a304..b6c37c481 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -73,23 +73,29 @@ protected DBConnectorPath getDBConnectorPath(String path) throws IOException { protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, ConnectorSpec.Builder builder) { - Map properties = new HashMap<>(); - setConnectionProperties(properties, request); + Map sourceProperties = new HashMap<>(); + Map sinkProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + setConnectionProperties(sinkProperties, request); builder - .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, properties)) - .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, properties)); + .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, sourceProperties)) + .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); + sinkProperties.put(PostgresConnectorConfig.NAME_DATABASE, config.getDatabase()); + String schema = path.getSchema(); + if (schema != null) { + sinkProperties.put(PostgresSink.PostgresSinkConfig.DB_SCHEMA_NAME, schema); + } + sourceProperties.put(PostgresSource.PostgresSourceConfig.NUM_SPLITS, "1"); String table = path.getTable(); if (table == null) { return; } - - properties.put(PostgresSource.PostgresSourceConfig.IMPORT_QUERY, getTableQuery(path.getDatabase(), - path.getSchema(), path.getTable())); - properties.put(PostgresSource.PostgresSourceConfig.NUM_SPLITS, "1"); - properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); - properties.put(PostgresSink.PostgresSinkConfig.TABLE_NAME, table); - + sourceProperties.put(PostgresSource.PostgresSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sinkProperties.put(PostgresSink.PostgresSinkConfig.TABLE_NAME, table); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + sinkProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); } @Override diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 5d5b86f23..cd2cd6102 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -107,6 +107,11 @@ "widget-type": "textbox", "label": "Table Name", "name": "tableName" + }, + { + "widget-type": "textbox", + "label": "Schema Name", + "name": "dbSchemaName" } ] }, From 479dd2374351ae03236f784f3166336d66a841bc Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Thu, 14 Apr 2022 03:06:02 +0530 Subject: [PATCH 02/36] removed database from connector spec --- .../plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java | 1 - .../widgets/CloudSQLPostgreSQL-batchsink.json | 4 ++++ .../src/main/java/io/cdap/plugin/oracle/OracleConnector.java | 1 - oracle-plugin/widgets/Oracle-batchsink.json | 4 ++++ .../main/java/io/cdap/plugin/postgres/PostgresConnector.java | 1 - postgresql-plugin/widgets/Postgres-batchsink.json | 4 ++++ 6 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index 4eeeb3b66..1e590287d 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -107,7 +107,6 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa .addRelatedPlugin(new PluginSpec(CloudSQLPostgreSQLConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); - sinkProperties.put(ConnectionConfig.DATABASE, config.getDatabase()); String schema = path.getSchema(); if (schema != null) { sinkProperties.put(CloudSQLPostgreSQLSink.CloudSQLPostgreSQLSinkConfig.DB_SCHEMA_NAME, schema); diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 8607d9110..4e6cbba94 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -185,6 +185,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "database" + }, { "type": "property", "name": "connectionArguments" diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 583ab60f8..8a2962282 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -80,7 +80,6 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, sourceProperties)) .addRelatedPlugin(new PluginSpec(OracleConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); - sinkProperties.put(OracleConstants.NAME_DATABASE, config.getDatabase()); String schema = path.getSchema(); if (schema != null) { sinkProperties.put(OracleSink.OracleSinkConfig.DB_SCHEMA_NAME, schema); diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index ee9b93572..395b8ab56 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -216,6 +216,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "database" } ] }, diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index b6c37c481..25b8133cd 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -81,7 +81,6 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, sourceProperties)) .addRelatedPlugin(new PluginSpec(PostgresConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, sinkProperties)); - sinkProperties.put(PostgresConnectorConfig.NAME_DATABASE, config.getDatabase()); String schema = path.getSchema(); if (schema != null) { sinkProperties.put(PostgresSink.PostgresSinkConfig.DB_SCHEMA_NAME, schema); diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index cd2cd6102..2ee173fc6 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -157,6 +157,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "database" + }, { "type": "property", "name": "connectionArguments" From 6b90d74204ff478941d621eedca1e88ea083e68a Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Thu, 14 Apr 2022 19:57:41 +0530 Subject: [PATCH 03/36] added macro check for connection configs in cloudsql plugins --- .../cloudsql/mysql/CloudSQLMySQLAction.java | 11 +-- .../mysql/CloudSQLMySQLConnectorConfig.java | 8 +-- .../mysql/CloudSQLMySQLConstants.java | 5 -- .../cloudsql/mysql/CloudSQLMySQLSink.java | 8 ++- .../cloudsql/mysql/CloudSQLMySQLSource.java | 8 ++- .../cloudsql/mysql/CloudSQLMySQLUtil.java | 60 ---------------- .../postgres/CloudSQLPostgreSQLAction.java | 11 +-- .../CloudSQLPostgreSQLConnectorConfig.java | 8 +-- .../postgres/CloudSQLPostgreSQLConstants.java | 5 -- .../postgres/CloudSQLPostgreSQLSink.java | 8 ++- .../postgres/CloudSQLPostgreSQLSource.java | 10 ++- .../postgres/CloudSQLPostgreSQLUtil.java | 62 ----------------- .../io/cdap/plugin/util/CloudSQLUtil.java | 69 +++++++++++++++++++ 13 files changed, 114 insertions(+), 159 deletions(-) delete mode 100644 cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLUtil.java delete mode 100644 cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLUtil.java create mode 100644 database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java index 4baf52cd4..c8b9c57ec 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java @@ -25,6 +25,7 @@ import io.cdap.cdap.etl.api.action.Action; import io.cdap.plugin.db.batch.action.AbstractDBAction; import io.cdap.plugin.db.batch.action.QueryConfig; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.Map; import javax.annotation.Nullable; @@ -48,7 +49,7 @@ public CloudSQLMySQLAction(CloudSQLMySQLActionConfig cloudsqlMysqlActionConfig) public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - CloudSQLMySQLUtil.checkConnectionName( + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlActionConfig.instanceType, cloudsqlMysqlActionConfig.connectionName); @@ -60,10 +61,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { public static class CloudSQLMySQLActionConfig extends QueryConfig { public CloudSQLMySQLActionConfig() { - this.instanceType = CloudSQLMySQLConstants.PUBLIC_INSTANCE; + this.instanceType = CloudSQLUtil.PUBLIC_INSTANCE; } - @Name(CloudSQLMySQLConstants.CONNECTION_NAME) + @Name(CloudSQLUtil.CONNECTION_NAME) @Description( "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " @@ -82,14 +83,14 @@ public CloudSQLMySQLActionConfig() { @Nullable public Integer connectionTimeout; - @Name(CloudSQLMySQLConstants.INSTANCE_TYPE) + @Name(CloudSQLUtil.INSTANCE_TYPE) @Description("Whether the CloudSQL instance to connect to is private or public.") @Nullable public String instanceType; @Override public String getConnectionString() { - if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, connectionName, diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java index fe43477f7..42b3227ab 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java @@ -20,6 +20,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.Properties; import javax.annotation.Nullable; @@ -32,7 +33,7 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig { private static final String JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS = "connectTimeout"; private static final String JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS = "socketTimeout"; - @Name(CloudSQLMySQLConstants.CONNECTION_NAME) + @Name(CloudSQLUtil.CONNECTION_NAME) @Description( "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " @@ -43,9 +44,8 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig { @Description("Database name to connect to") private String database; - @Name(CloudSQLMySQLConstants.INSTANCE_TYPE) + @Name(CloudSQLUtil.INSTANCE_TYPE) @Description("Whether the CloudSQL instance to connect to is private or public.") - @Nullable private String instanceType; public CloudSQLMySQLConnectorConfig(String user, String password, String jdbcPluginName, String connectionArguments, @@ -73,7 +73,7 @@ public String getConnectionName() { @Override public String getConnectionString() { - if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, connectionName, diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java index d98a74a5a..ae8a34c6a 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java @@ -23,13 +23,8 @@ private CloudSQLMySQLConstants() { } public static final String PLUGIN_NAME = "CloudSQLMySQL"; - public static final String CONNECTION_NAME = "connectionName"; - public static final String CONNECTION_NAME_PATTERN = "[a-z0-9-]+:[a-z0-9-]+:[a-z0-9-]+"; public static final String CONNECTION_TIMEOUT = "connectionTimeout"; - public static final String PUBLIC_INSTANCE = "public"; public static final String PUBLIC_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory"; - public static final String INSTANCE_TYPE = "instanceType"; - public static final String PRIVATE_INSTANCE = "private"; public static final String PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s/%s"; } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 25596bd36..8130b7c76 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -32,6 +32,7 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.batch.config.AbstractDBSpecificSinkConfig; import io.cdap.plugin.db.batch.sink.AbstractDBSink; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.Map; import javax.annotation.Nullable; @@ -54,11 +55,14 @@ public CloudSQLMySQLSink(CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig) { @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - - CloudSQLMySQLUtil.checkConnectionName( + + if (!cloudsqlMysqlSinkConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) && + !cloudsqlMysqlSinkConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) { + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSinkConfig.connection.getInstanceType(), cloudsqlMysqlSinkConfig.connection.getConnectionName()); + } super.configurePipeline(pipelineConfigurer); } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index eb567b1dc..4e384eb45 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -31,6 +31,7 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.batch.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.batch.source.AbstractDBSource; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.Collections; import java.util.HashMap; @@ -57,10 +58,13 @@ public CloudSQLMySQLSource(CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig) public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - CloudSQLMySQLUtil.checkConnectionName( + if (!cloudsqlMysqlSourceConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) && + !cloudsqlMysqlSourceConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) { + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSourceConfig.connection.getInstanceType(), cloudsqlMysqlSourceConfig.connection.getConnectionName()); + } super.configurePipeline(pipelineConfigurer); } @@ -72,7 +76,7 @@ protected SchemaReader getSchemaReader() { @Override protected String createConnectionString() { - if (CloudSQLMySQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase( + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( cloudsqlMysqlSourceConfig.connection.getInstanceType())) { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLUtil.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLUtil.java deleted file mode 100644 index 1febabc74..000000000 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLUtil.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright © 2020 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.cloudsql.mysql; - -import com.google.common.net.InetAddresses; -import io.cdap.cdap.etl.api.FailureCollector; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** Utility class for CloudSQL MySQL. */ -public class CloudSQLMySQLUtil { - - /** - * Utility method to check the Connection Name format of a CloudSQL MySQL instance. - * - * @param failureCollector {@link FailureCollector} for the pipeline - * @param instanceType CloudSQL MySQL instance type - * @param connectionName Connection Name for the CloudSQL MySQL instance - */ - public static void checkConnectionName( - FailureCollector failureCollector, String instanceType, String connectionName) { - - if (CloudSQLMySQLConstants.PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { - Pattern connectionNamePattern = - Pattern.compile(CloudSQLMySQLConstants.CONNECTION_NAME_PATTERN); - Matcher matcher = connectionNamePattern.matcher(connectionName); - - if (!matcher.matches()) { - failureCollector - .addFailure( - "Connection Name must be in the format :: to connect to " - + "a public CloudSQL MySQL instance.", null) - .withConfigProperty(CloudSQLMySQLConstants.CONNECTION_NAME); - } - } else { - if (!InetAddresses.isInetAddress(connectionName)) { - failureCollector - .addFailure( - "Enter the internal IP address of the Compute Engine VM cloudsql proxy is running on, to " - + "connect to a private CloudSQL MySQL instance.", null) - .withConfigProperty(CloudSQLMySQLConstants.CONNECTION_NAME); - } - } - } -} diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java index 6c1c75aa6..0923c5829 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java @@ -25,6 +25,7 @@ import io.cdap.cdap.etl.api.action.Action; import io.cdap.plugin.db.batch.action.AbstractDBAction; import io.cdap.plugin.db.batch.action.QueryConfig; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.Map; import javax.annotation.Nullable; @@ -48,7 +49,7 @@ public CloudSQLPostgreSQLAction(CloudSQLPostgreSQLActionConfig cloudsqlPostgresq public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - CloudSQLPostgreSQLUtil.checkConnectionName( + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlActionConfig.instanceType, cloudsqlPostgresqlActionConfig.connectionName); @@ -60,10 +61,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { public static class CloudSQLPostgreSQLActionConfig extends QueryConfig { public CloudSQLPostgreSQLActionConfig() { - this.instanceType = CloudSQLPostgreSQLConstants.PUBLIC_INSTANCE; + this.instanceType = CloudSQLUtil.PUBLIC_INSTANCE; } - @Name(CloudSQLPostgreSQLConstants.CONNECTION_NAME) + @Name(CloudSQLUtil.CONNECTION_NAME) @Description( "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " @@ -82,14 +83,14 @@ public CloudSQLPostgreSQLActionConfig() { @Nullable public Integer connectionTimeout; - @Name(CloudSQLPostgreSQLConstants.INSTANCE_TYPE) + @Name(CloudSQLUtil.INSTANCE_TYPE) @Description("Whether the CloudSQL instance to connect to is private or public.") @Nullable public String instanceType; @Override public String getConnectionString() { - if (CloudSQLPostgreSQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, connectionName, diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java index e2d9e33ce..d3ec0849b 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java @@ -20,6 +20,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import io.cdap.plugin.util.CloudSQLUtil; import javax.annotation.Nullable; @@ -28,7 +29,7 @@ */ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig { - @Name(CloudSQLPostgreSQLConstants.CONNECTION_NAME) + @Name(CloudSQLUtil.CONNECTION_NAME) @Description( "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " @@ -39,9 +40,8 @@ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig @Description("Database name to connect to") private String database; - @Name(CloudSQLPostgreSQLConstants.INSTANCE_TYPE) + @Name(CloudSQLUtil.INSTANCE_TYPE) @Description("Whether the CloudSQL instance to connect to is private or public.") - @Nullable private String instanceType; public CloudSQLPostgreSQLConnectorConfig(String username, String password, String jdbcPluginName, @@ -70,7 +70,7 @@ public String getConnectionName() { @Override public String getConnectionString() { - if (CloudSQLPostgreSQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, connectionName, diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java index 79f17a018..946171102 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java @@ -23,13 +23,8 @@ private CloudSQLPostgreSQLConstants() { } public static final String PLUGIN_NAME = "CloudSQLPostgreSQL"; - public static final String CONNECTION_NAME = "connectionName"; - public static final String CONNECTION_NAME_PATTERN = "[a-z0-9-]+:[a-z0-9-]+:[a-z0-9-]+"; public static final String CONNECTION_TIMEOUT = "connectionTimeout"; - public static final String PUBLIC_INSTANCE = "public"; public static final String PUBLIC_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.postgres.SocketFactory"; - public static final String INSTANCE_TYPE = "instanceType"; - public static final String PRIVATE_INSTANCE = "private"; public static final String PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql://%s/%s"; } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 64eb81433..d2a406fb1 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -38,6 +38,7 @@ import io.cdap.plugin.postgres.PostgresDBRecord; import io.cdap.plugin.postgres.PostgresFieldsValidator; import io.cdap.plugin.postgres.PostgresSchemaReader; +import io.cdap.plugin.util.CloudSQLUtil; import java.util.ArrayList; import java.util.Collections; @@ -66,11 +67,14 @@ public CloudSQLPostgreSQLSink(CloudSQLPostgreSQLSinkConfig cloudsqlPostgresqlSin @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - - CloudSQLPostgreSQLUtil.checkConnectionName( + + if (!cloudsqlPostgresqlSinkConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) && + !cloudsqlPostgresqlSinkConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) { + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSinkConfig.connection.getInstanceType(), cloudsqlPostgresqlSinkConfig.connection.getConnectionName()); + } super.configurePipeline(pipelineConfigurer); } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 3d8036341..8ce575d00 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -32,6 +32,7 @@ import io.cdap.plugin.db.batch.source.AbstractDBSource; import io.cdap.plugin.postgres.PostgresDBRecord; import io.cdap.plugin.postgres.PostgresSchemaReader; +import io.cdap.plugin.util.CloudSQLUtil; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Collections; @@ -58,11 +59,14 @@ public CloudSQLPostgreSQLSource(CloudSQLPostgreSQLSourceConfig cloudsqlPostgresq @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - - CloudSQLPostgreSQLUtil.checkConnectionName( + + if (!cloudsqlPostgresqlSourceConfig.containsMacro(CloudSQLUtil.INSTANCE_TYPE) && + !cloudsqlPostgresqlSourceConfig.containsMacro(CloudSQLUtil.CONNECTION_NAME)) { + CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSourceConfig.connection.getInstanceType(), cloudsqlPostgresqlSourceConfig.connection.getConnectionName()); + } super.configurePipeline(pipelineConfigurer); } @@ -79,7 +83,7 @@ protected Class getDBRecordType() { @Override protected String createConnectionString() { - if (CloudSQLPostgreSQLConstants.PRIVATE_INSTANCE.equalsIgnoreCase( + if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( cloudsqlPostgresqlSourceConfig.connection.getInstanceType())) { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLUtil.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLUtil.java deleted file mode 100644 index f6720d3eb..000000000 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLUtil.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright © 2020 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.cloudsql.postgres; - -import com.google.common.net.InetAddresses; -import io.cdap.cdap.etl.api.FailureCollector; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Utility class for CloudSQL PostgreSQL. - */ -public class CloudSQLPostgreSQLUtil { - - /** - * Utility method to check the Connection Name format of a CloudSQL PostgreSQL instance. - * - * @param failureCollector {@link FailureCollector} for the pipeline - * @param instanceType CloudSQL PostgreSQL instance type - * @param connectionName Connection Name for the CloudSQL PostgreSQL instance - */ - public static void checkConnectionName( - FailureCollector failureCollector, String instanceType, String connectionName) { - - if (CloudSQLPostgreSQLConstants.PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { - Pattern connectionNamePattern = - Pattern.compile(CloudSQLPostgreSQLConstants.CONNECTION_NAME_PATTERN); - Matcher matcher = connectionNamePattern.matcher(connectionName); - - if (!matcher.matches()) { - failureCollector - .addFailure( - "Connection Name must be in the format :: to connect to " - + "a public CloudSQL PostgreSQL instance.", null) - .withConfigProperty(CloudSQLPostgreSQLConstants.CONNECTION_NAME); - } - } else { - if (!InetAddresses.isInetAddress(connectionName)) { - failureCollector - .addFailure( - "Enter the internal IP address of the Compute Engine VM cloudsql proxy " - + "is running on, to connect to a private CloudSQL PostgreSQL instance.", null) - .withConfigProperty(CloudSQLPostgreSQLConstants.CONNECTION_NAME); - } - } - } -} diff --git a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java new file mode 100644 index 000000000..eefe7fd5f --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.util; + +import com.google.common.base.Strings; +import com.google.common.net.InetAddresses; +import io.cdap.cdap.etl.api.FailureCollector; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +/** + * Utility class for CloudSQL . + */ +public class CloudSQLUtil { + public static final String CONNECTION_NAME = "connectionName"; + public static final String CONNECTION_NAME_PATTERN = "[a-z0-9-]+:[a-z0-9-]+:[a-z0-9-]+"; + public static final String INSTANCE_TYPE = "instanceType"; + public static final String PUBLIC_INSTANCE = "public"; + public static final String PRIVATE_INSTANCE = "private"; + + /** + * Utility method to check the Connection Name format of a CloudSQL instance. + * + * @param failureCollector {@link FailureCollector} for the pipeline + * @param instanceType CloudSQL instance type + * @param connectionName Connection Name for the CloudSQL instance + */ + public static void checkConnectionName( + FailureCollector failureCollector, String instanceType, String connectionName) { + + if (PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { + Pattern connectionNamePattern = + Pattern.compile(CONNECTION_NAME_PATTERN); + Matcher matcher = connectionNamePattern.matcher(connectionName); + + if (!matcher.matches()) { + failureCollector + .addFailure( + "Connection Name must be in the format :: to connect to " + + "a public CloudSQL PostgreSQL instance.", null) + .withConfigProperty(CONNECTION_NAME); + } + } else { + if (!InetAddresses.isInetAddress(connectionName)) { + failureCollector + .addFailure( + "Enter the internal IP address of the Compute Engine VM cloudsql proxy " + + "is running on, to connect to a private CloudSQL PostgreSQL instance.", null) + .withConfigProperty(CONNECTION_NAME); + } + } + } +} From f2b0a7c0801b3ceedd9092a208d64b7fa7ffd03c Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Thu, 21 Apr 2022 19:09:59 +0530 Subject: [PATCH 04/36] added fetch size to connector spec and default fetch size for postgres source --- .../cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java | 2 ++ .../cloudsql/postgres/CloudSQLPostgreSQLConnector.java | 3 ++- .../plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java | 6 ++++++ .../db/batch/config/AbstractDBSpecificSourceConfig.java | 1 + .../main/java/io/cdap/plugin/mssql/SqlServerConnector.java | 2 ++ .../src/main/java/io/cdap/plugin/mysql/MysqlConnector.java | 1 + .../main/java/io/cdap/plugin/oracle/OracleConnector.java | 2 ++ postgresql-plugin/docs/Postgres-batchsource.md | 2 +- .../java/io/cdap/plugin/postgres/PostgresConnector.java | 4 ++++ .../main/java/io/cdap/plugin/postgres/PostgresSource.java | 7 +++++++ 10 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index a2b93a910..9bc1bc7f5 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -99,6 +99,8 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa path.getSchema(), path.getTable())); properties.put(CloudSQLMySQLSource.CloudSQLMySQLSourceConfig.NUM_SPLITS, "1"); + properties.put(CloudSQLMySQLSource.CloudSQLMySQLSourceConfig.FETCH_SIZE, + CloudSQLMySQLSource.CloudSQLMySQLSourceConfig.DEFAULT_FETCH_SIZE); properties.put(ConnectionConfig.DATABASE, path.getDatabase()); properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); properties.put(CloudSQLMySQLSink.CloudSQLMySQLSinkConfig.TABLE_NAME, table); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index 1e590287d..f6b0490b1 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -31,7 +31,6 @@ import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; import io.cdap.plugin.common.db.DBPath; -import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; import io.cdap.plugin.postgres.PostgresDBRecord; @@ -112,6 +111,8 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa sinkProperties.put(CloudSQLPostgreSQLSink.CloudSQLPostgreSQLSinkConfig.DB_SCHEMA_NAME, schema); } sourceProperties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.FETCH_SIZE, + CloudSQLPostgreSQLSource.CloudSQLPostgreSQLSourceConfig.DEFAULT_FETCH_SIZE); String table = path.getTable(); if (table == null) { return; diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 8ce575d00..95322718c 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -116,6 +116,12 @@ protected Map getDBSpecificArguments() { return Collections.emptyMap(); } + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + @Override protected CloudSQLPostgreSQLConnectorConfig getConnection() { return connection; diff --git a/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSourceConfig.java index 0296e728b..8a8df6a00 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/batch/config/AbstractDBSpecificSourceConfig.java @@ -48,6 +48,7 @@ public abstract class AbstractDBSpecificSourceConfig extends PluginConfig implem public static final String SCHEMA = "schema"; public static final String DATABASE = "database"; public static final String FETCH_SIZE = "fetchSize"; + public static final String DEFAULT_FETCH_SIZE = "1000"; @Name(Constants.Reference.REFERENCE_NAME) @Description(Constants.Reference.REFERENCE_NAME_DESCRIPTION) diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index 9b75c2546..875cd5de1 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -87,6 +87,8 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa sinkProperties.put(SqlServerSink.SqlServerSinkConfig.DB_SCHEMA_NAME, schema); } sourceProperties.put(SqlServerSource.SqlServerSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(SqlServerSource.SqlServerSourceConfig.FETCH_SIZE, + SqlServerSource.SqlServerSourceConfig.DEFAULT_FETCH_SIZE); String table = path.getTable(); if (table == null) { return; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index adeb9ace1..72b1ca2b0 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -80,6 +80,7 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa properties.put(MysqlSource.MysqlSourceConfig.IMPORT_QUERY, getTableQuery(path.getDatabase(), path.getSchema(), path.getTable())); properties.put(MysqlSource.MysqlSourceConfig.NUM_SPLITS, "1"); + properties.put(MysqlSource.MysqlSourceConfig.FETCH_SIZE, MysqlSource.MysqlSourceConfig.DEFAULT_FETCH_SIZE); properties.put(MysqlSource.MysqlSourceConfig.DATABASE, path.getDatabase()); properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); properties.put(MysqlSink.MysqlSinkConfig.TABLE_NAME, table); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 8a2962282..8b6ac80c6 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -85,6 +85,8 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa sinkProperties.put(OracleSink.OracleSinkConfig.DB_SCHEMA_NAME, schema); } sourceProperties.put(OracleSource.OracleSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(OracleSource.OracleSourceConfig.FETCH_SIZE, + OracleSource.OracleSourceConfig.DEFAULT_FETCH_SIZE); String table = path.getTable(); if (table == null) { return; diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index cc4f7ab84..340d66963 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -60,7 +60,7 @@ back from the query. However, it must match the schema that comes back from the except it can mark fields as nullable and can contain a subset of the fields. **Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, -with the tradeoff of higher memory usage. +with the tradeoff of higher memory usage. If not specified, the default value is 1000. Example ------ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index 25b8133cd..b5075c0b2 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -86,6 +86,10 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa sinkProperties.put(PostgresSink.PostgresSinkConfig.DB_SCHEMA_NAME, schema); } sourceProperties.put(PostgresSource.PostgresSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(PostgresSource.PostgresSourceConfig.FETCH_SIZE, + PostgresSource.PostgresSourceConfig.DEFAULT_FETCH_SIZE); + sourceProperties.put(PostgresConstants.CONNECTION_TIMEOUT, + PostgresSource.PostgresSourceConfig.DEFAULT_CONNECTION_TIMEOUT_SECONDS); String table = path.getTable(); if (table == null) { return; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 12dd45003..963bba263 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -75,6 +75,7 @@ public static class PostgresSourceConfig extends AbstractDBSpecificSourceConfig public static final String NAME_USE_CONNECTION = "useConnection"; public static final String NAME_CONNECTION = "connection"; + public static final String DEFAULT_CONNECTION_TIMEOUT_SECONDS = "100"; @Name(NAME_USE_CONNECTION) @Nullable @@ -106,6 +107,12 @@ public Map getDBSpecificArguments() { return ImmutableMap.of(PostgresConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + @Override protected AbstractDBSpecificConnectorConfig getConnection() { return connection; From 9bf239613ad56885bf7779d1b6c81e4f17b3250b Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Fri, 22 Apr 2022 02:42:48 +0530 Subject: [PATCH 05/36] markdown update for CloudsqlPostgres plugin --- cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md index 5871e359e..e2242707a 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md @@ -57,7 +57,7 @@ back from the query. However, it must match the schema that comes back from the except it can mark fields as nullable and can contain a subset of the fields. **Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, -with the tradeoff of higher memory usage. +with the tradeoff of higher memory usage. If not specified, the default value is 1000. Data Types Mapping ------------------ From 1b865e24a839fcc58d956976701562bd9f0175ec Mon Sep 17 00:00:00 2001 From: Amit Virmani Date: Fri, 29 Apr 2022 00:10:06 -0700 Subject: [PATCH 06/36] CDAP-18893 - update UI display name for CloudSQL MySQL and PostgreSQL --- cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json | 2 +- .../widgets/CloudSQLPostgreSQL-connector.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index e82d3fd01..1f8551605 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -2,7 +2,7 @@ "metadata": { "spec-version": "1.0" }, - "display-name": "CloudSQLMySQL", + "display-name": "CloudSQL MySQL", "configuration-groups": [ { "label": "Basic", diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index d2ad9341c..de3af0795 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -2,7 +2,7 @@ "metadata": { "spec-version": "1.0" }, - "display-name": "CloudSQLPostgreSQL", + "display-name": "CloudSQL PostgreSQL", "configuration-groups": [ { "label": "Basic", From 43c6904e68e3a783168421258d8466172fdf9456 Mon Sep 17 00:00:00 2001 From: JiongxinYe Date: Thu, 28 Apr 2022 16:23:18 -0700 Subject: [PATCH 07/36] Check schema type for bigint object --- .../src/main/java/io/cdap/plugin/db/DBRecord.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index a80609f04..7592a5fdf 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -152,10 +152,9 @@ protected void setField(ResultSet resultSet, StructuredRecord.Builder recordBuil } else if (o instanceof BigDecimal) { recordBuilder.setDecimal(field.getName(), (BigDecimal) o); } else if (o instanceof BigInteger) { - if (sqlType == Types.BIGINT && (resultSet.getMetaData().isSigned(columnIndex) || - resultSet.getMetaData().getPrecision(columnIndex) < 19)) { - //SQL BIGINT type is 64-bit long thus signed should be able to convert to long without losing precisions - // or UNSIGNED type is within the scope of signed long + Schema schema = field.getSchema(); + schema = schema.isNullable() ? schema.getNonNullable() : schema; + if (schema.getType() == Schema.Type.LONG) { recordBuilder.set(field.getName(), ((BigInteger) o).longValueExact()); } else { // BigInteger won't have any fraction part and scale is 0 From c32fc5fd0038dbb5d6ca3a8c479fc40c2a5edde8 Mon Sep 17 00:00:00 2001 From: Justin Taras Date: Wed, 27 Apr 2022 14:31:37 -0400 Subject: [PATCH 08/36] add isolation level to oracle plugin --- oracle-plugin/docs/Oracle-batchsink.md | 5 ++++ oracle-plugin/docs/Oracle-batchsource.md | 5 ++++ oracle-plugin/docs/Oracle-connector.md | 5 ++++ .../plugin/oracle/OracleConnectorConfig.java | 19 +++++++++++-- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 3 +- oracle-plugin/widgets/Oracle-batchsink.json | 28 +++++++++++++++++++ oracle-plugin/widgets/Oracle-batchsource.json | 28 +++++++++++++++++++ oracle-plugin/widgets/Oracle-connector.json | 26 +++++++++++++++++ 9 files changed, 116 insertions(+), 4 deletions(-) diff --git a/oracle-plugin/docs/Oracle-batchsink.md b/oracle-plugin/docs/Oracle-batchsink.md index 20f59e86c..23f32dfd4 100644 --- a/oracle-plugin/docs/Oracle-batchsink.md +++ b/oracle-plugin/docs/Oracle-batchsink.md @@ -30,6 +30,11 @@ You also can use the macro function ${conn(connection-name)}. **Role** Login role of the user when connecting to the database. For eg, NORMAL, SYSDBA, SYSOPER, etc. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- Note: If the user role selected is SYSDBA or SYSOPER, the plugin will default to TRANSACTION_READ_COMMITTED to prevent ORA-08178 errors + **Connection Type** Whether to use an SID or Service Name when connecting to the database. **SID/Service Name/TNS Connect Descriptor:** Oracle connection point (Database name, Service name, or a TNS Connect Descriptor). When using TNS, place diff --git a/oracle-plugin/docs/Oracle-batchsource.md b/oracle-plugin/docs/Oracle-batchsource.md index cdca18ca0..1dbdf9836 100644 --- a/oracle-plugin/docs/Oracle-batchsource.md +++ b/oracle-plugin/docs/Oracle-batchsource.md @@ -38,6 +38,11 @@ the full TNS Connect Descriptor in the text field. For example: **Role** Login role of the user when connecting to the database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- Note: If the user role selected is SYSDBA or SYSOPER, the plugin will default to TRANSACTION_READ_COMMITTED to prevent ORA-08178 errors + **Import Query:** The SELECT query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. diff --git a/oracle-plugin/docs/Oracle-connector.md b/oracle-plugin/docs/Oracle-connector.md index 839c5afe5..9650a2fc0 100644 --- a/oracle-plugin/docs/Oracle-connector.md +++ b/oracle-plugin/docs/Oracle-connector.md @@ -31,6 +31,11 @@ the full TNS Connect Descriptor in the text field. For example: **Role:** Login role of the user when connecting to the database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- Note: If the user role selected is SYSDBA or SYSOPER, the plugin will default to TRANSACTION_READ_COMMITTED to prevent ORA-08178 errors + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 35200d5b8..6744cc933 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -79,6 +79,12 @@ public String getConnectionString() { @Macro private String database; + @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + private String transactionIsolationLevel; + @Override protected int getDefaultPort() { return 1521; @@ -105,10 +111,17 @@ public Properties getConnectionArgumentsProperties() { prop.put(INTERNAL_LOGON_PROPERTY, getRole()); return prop; } - + public String getTransactionIsolationLevel() { - return ROLE_NORMAL.equals(getRole()) ? null : - TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name(); + //if null default to the highest isolation level possible + if (transactionIsolationLevel == null) { + transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); + } + //To solve the problem of ORA-08178: illegal SERIALIZABLE clause specified for user INTERNAL + //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input + //if role is SYSDBA or SYSOP it will map to read_committed. else serialized + return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index d3b510598..040780a89 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -35,4 +35,5 @@ private OracleConstants() { public static final String ROLE = "role"; public static final String NAME_DATABASE = "database"; public static final String TNS_CONNECTION_TYPE = "TNS"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index e87e43919..4dafdf95b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -78,6 +78,7 @@ public static class OracleSourceConfig extends AbstractDBSpecificSourceConfig { @Nullable @Description("Whether to use an existing connection.") private Boolean useConnection; + @Name(NAME_CONNECTION) @Macro @Nullable @@ -130,7 +131,7 @@ public void validate(FailureCollector collector) { @Override public String getTransactionIsolationLevel() { - return getConnection().getTransactionIsolationLevel(); + return connection.getTransactionIsolationLevel(); } } diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 395b8ab56..ee9d68ec5 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -88,6 +88,18 @@ ] } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "name": "connectionType", "label": "Connection Type", @@ -179,6 +191,18 @@ ], "outputs": [], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -220,6 +244,10 @@ { "type": "property", "name": "database" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index c5d550b1d..30fa2ef8a 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -88,6 +88,18 @@ ] } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "name": "connectionType", "label": "Connection Type", @@ -237,6 +249,18 @@ } ], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -278,6 +302,10 @@ { "type": "property", "name": "database" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 4566d051e..46f006c9c 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -97,6 +97,18 @@ } ] } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, @@ -118,5 +130,19 @@ ] } ], + "filters" : [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + } + ], "outputs": [] } From 24f018e4ab886e682f8d3e6426dd314bfd3588af Mon Sep 17 00:00:00 2001 From: itsankitjain-google Date: Fri, 13 May 2022 13:31:39 +0530 Subject: [PATCH 09/36] refactor db-connector-test --- .../connector/DBSpecificConnectorBaseTest.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/database-commons/src/test/java/io/cdap/plugin/db/connector/DBSpecificConnectorBaseTest.java b/database-commons/src/test/java/io/cdap/plugin/db/connector/DBSpecificConnectorBaseTest.java index 9ff8ab62c..fbb35076a 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/connector/DBSpecificConnectorBaseTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/connector/DBSpecificConnectorBaseTest.java @@ -249,19 +249,13 @@ protected void testGenerateSpec(AbstractDBSpecificConnector connector, String pl } Set relatedPlugins = connectorSpec.getRelatedPlugins(); Assert.assertEquals(2, relatedPlugins.size()); - Iterator relatedPluginsIterator = relatedPlugins.iterator(); - PluginSpec pluginSpec = relatedPluginsIterator.next(); + Iterator iterator = relatedPlugins.iterator(); + PluginSpec pluginSpec = iterator.next(); Assert.assertEquals(pluginName, pluginSpec.getName()); - Assert.assertTrue(pluginSpec.getType().equals(BatchSource.PLUGIN_TYPE) || - pluginSpec.getType().equals(BatchSink.PLUGIN_TYPE)); - PluginSpec pluginSpec1 = relatedPluginsIterator.next(); - Assert.assertEquals(pluginName, pluginSpec1.getName()); - if (pluginSpec.getType().equals(BatchSink.PLUGIN_TYPE)) { - Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec1.getType()); - } else { - Assert.assertEquals(BatchSink.PLUGIN_TYPE, pluginSpec1.getType()); - } - + Assert.assertEquals(BatchSink.PLUGIN_TYPE, pluginSpec.getType()); + pluginSpec = iterator.next(); + Assert.assertEquals(pluginName, pluginSpec.getName()); + Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec.getType()); Map properties = pluginSpec.getProperties(); Assert.assertNull(properties.get(NAME_USE_CONNECTION)); Assert.assertNull(properties.get(NAME_CONNECTION)); From d245b8accbef4cc19a6eabe446ae0a26ef33648f Mon Sep 17 00:00:00 2001 From: Sebastian Echegaray Date: Thu, 2 Jun 2022 08:53:24 -0700 Subject: [PATCH 10/36] Removed SNAPSHOT --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 6 +++--- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index a44d890b1..25318c183 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 222f131c2..5ab78c02b 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index caef64fcb..5db06b27d 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 4d59d24e6..6ceed420d 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.0-SNAPSHOT + 1.8.0 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.0-SNAPSHOT + 1.8.0 com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index d08fa7051..bc71fe4a4 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index daea4460c..0a815a86f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 04bc87924..0aa65a69b 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 4dabf7d44..4d3f0cecb 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 4398eed32..6d4be0428 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 4cf207226..64f28af9b 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 5d99ec1d1..f8f9aa90c 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 792b23cb7..9e47d0e73 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index f6d5a938d..eb52ec092 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 944249699..dd60711ee 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 Oracle plugin diff --git a/pom.xml b/pom.xml index f305ecea5..fc6ce1986 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.0-SNAPSHOT + 1.8.0 pom Database Plugins Collection of database plugins @@ -60,8 +60,8 @@ true UTF-8 - 6.7.0-SNAPSHOT - 2.9.0-SNAPSHOT + 6.7.0 + 2.9.0 13.0.1 2.3.0 2.2.4 diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index bbac9002f..d31d8aa43 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 76a6befc5..17d960773 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 90756d4b4..0d3a9478a 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0-SNAPSHOT + 1.8.0 teradata-plugin From 8138dce780ed6e87e424920acf5fd192e59e9051 Mon Sep 17 00:00:00 2001 From: Sebastian Echegaray Date: Wed, 22 Jun 2022 15:30:58 -0700 Subject: [PATCH 11/36] Bumped version and added snapshot --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 6 +++--- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 25318c183..90e19b375 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 5ab78c02b..22588f7bb 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 5db06b27d..c516c71dd 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 6ceed420d..a6c3305e9 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.0 + 1.8.1-SNAPSHOT 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.0 + 1.8.1-SNAPSHOT com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index bc71fe4a4..2290c20f8 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0a815a86f..8d146d08a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 0aa65a69b..bfb95a286 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 4d3f0cecb..f6a68b190 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 6d4be0428..d76b5b6c0 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 64f28af9b..957d28659 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index f8f9aa90c..c607697a3 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 9e47d0e73..756b9f60d 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index eb52ec092..93d9d8938 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index dd60711ee..917ba9dbc 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index fc6ce1986..c821ac3ab 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.0 + 1.8.1-SNAPSHOT pom Database Plugins Collection of database plugins @@ -60,8 +60,8 @@ true UTF-8 - 6.7.0 - 2.9.0 + 6.7.1-SNAPSHOT + 2.9.1-SNAPSHOT 13.0.1 2.3.0 2.2.4 diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d31d8aa43..e19895e67 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 17d960773..0ff9004eb 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0d3a9478a..981cde091 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.0 + 1.8.1-SNAPSHOT teradata-plugin From d49615b54435748fdc25924dd537f39cb343bf9b Mon Sep 17 00:00:00 2001 From: Sebastian Echegaray Date: Tue, 9 Aug 2022 17:27:27 -0700 Subject: [PATCH 12/36] Removed -Snapshot --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 6 +++--- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 90e19b375..f943c484c 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 22588f7bb..133f5073a 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c516c71dd..f1675a107 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index a6c3305e9..3ef3534fa 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.1-SNAPSHOT + 1.8.1 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.1-SNAPSHOT + 1.8.1 com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 2290c20f8..afdff30c1 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 8d146d08a..9ae5ce7db 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index bfb95a286..3e937fb7d 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index f6a68b190..f65d7e72c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index d76b5b6c0..06e5d4cdf 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 957d28659..c20ea9c77 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index c607697a3..08ed38672 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 756b9f60d..7a646ad62 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 93d9d8938..5ecfab675 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 917ba9dbc..08acaaedb 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 Oracle plugin diff --git a/pom.xml b/pom.xml index c821ac3ab..776da4372 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.1-SNAPSHOT + 1.8.1 pom Database Plugins Collection of database plugins @@ -60,8 +60,8 @@ true UTF-8 - 6.7.1-SNAPSHOT - 2.9.1-SNAPSHOT + 6.7.1 + 2.9.1 13.0.1 2.3.0 2.2.4 diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index e19895e67..c5bcf6fc3 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 0ff9004eb..a8a2873ae 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 981cde091..1dbc20e15 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1-SNAPSHOT + 1.8.1 teradata-plugin From 862f941a7a2f217ac4a92cc02b78aaaed2d331a3 Mon Sep 17 00:00:00 2001 From: Vaibhav Sethi Date: Thu, 22 Sep 2022 03:18:16 +0530 Subject: [PATCH 13/36] Bump version to 1.8.2-SNAPSHOT --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index f943c484c..d2fbc92d0 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 133f5073a..383a6780b 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f1675a107..dcf711a62 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 3ef3534fa..e5fa34444 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.1 + 1.8.2-SNAPSHOT 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.1 + 1.8.2-SNAPSHOT com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index afdff30c1..8e0256708 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 9ae5ce7db..67a0cfbcb 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 3e937fb7d..7ebce0714 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index f65d7e72c..fcc27abbd 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 06e5d4cdf..fb157f95a 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index c20ea9c77..86b5e8991 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 08ed38672..279595b36 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 7a646ad62..a3017e21c 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 5ecfab675..7232946c8 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 08acaaedb..63509a581 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 776da4372..fddc2cfd2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.1 + 1.8.2-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index c5bcf6fc3..30c121402 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index a8a2873ae..9912581d5 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 1dbc20e15..fca1a03f6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.1 + 1.8.2-SNAPSHOT teradata-plugin From 5b556d42ed0e2541c2021e9a0b13f0e2df45ef84 Mon Sep 17 00:00:00 2001 From: katukota Date: Wed, 21 Sep 2022 11:29:34 -0700 Subject: [PATCH 14/36] CDAP-19532 | allow precision 0 for Oracle Number data type in generic database plugin - Fix tests --- .../src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java index 3fbf65247..0f5a3ca4a 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java @@ -154,7 +154,7 @@ public void testGetSchemaHandlesBytes() throws SQLException { Assert.assertEquals(Schema.of(Schema.Type.BYTES), reader.getSchema(metadata, 4)); } - @Test(expected = SQLException.class) + @Test(expected = IllegalArgumentException.class) public void testGetSchemaThrowsExceptionOnNumericWithZeroPrecision() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.NUMERIC); when(metadata.getPrecision(eq(1))).thenReturn(0); From baba5b0107e3fa123e025c91e04b39730d0aef37 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Fri, 23 Sep 2022 00:33:34 +0530 Subject: [PATCH 15/36] fix cdap plugin version --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index fddc2cfd2..9c8ca269a 100644 --- a/pom.xml +++ b/pom.xml @@ -60,8 +60,8 @@ true UTF-8 - 6.7.1 - 2.9.1 + 6.7.2-SNAPSHOT + 2.9.2-SNAPSHOT 13.0.1 2.3.0 2.2.4 From 74e8c36d3925f4014b0d588b775044b3995c8ce5 Mon Sep 17 00:00:00 2001 From: Vaibhav Sethi Date: Thu, 29 Sep 2022 12:13:45 +0530 Subject: [PATCH 16/36] Bump release version to 1.8.2 --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 6 +++--- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index d2fbc92d0..110d57834 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 383a6780b..2b59afa2b 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index dcf711a62..1caac20ea 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index e5fa34444..c712cb912 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.2-SNAPSHOT + 1.8.2 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.2-SNAPSHOT + 1.8.2 com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 8e0256708..d5a1d1662 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 67a0cfbcb..69e53c92b 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 7ebce0714..6372f3db1 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index fcc27abbd..37c6f944a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index fb157f95a..358a246be 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 86b5e8991..c9dd455e7 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 279595b36..340130f56 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index a3017e21c..ab1d862ae 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 7232946c8..4fc8ca114 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 63509a581..739c29b03 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 Oracle plugin diff --git a/pom.xml b/pom.xml index 9c8ca269a..f90b86703 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.2-SNAPSHOT + 1.8.2 pom Database Plugins Collection of database plugins @@ -60,8 +60,8 @@ true UTF-8 - 6.7.2-SNAPSHOT - 2.9.2-SNAPSHOT + 6.7.2 + 2.9.2 13.0.1 2.3.0 2.2.4 diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 30c121402..d9becb101 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 9912581d5..683e8446a 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index fca1a03f6..3af6982b5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2-SNAPSHOT + 1.8.2 teradata-plugin From 65336a9cc9df68d8071940f8565dfca97f353e61 Mon Sep 17 00:00:00 2001 From: ameya-111 Date: Wed, 7 Sep 2022 17:08:20 -0700 Subject: [PATCH 17/36] PLUGIN-1374: Update mySql plugin so that rewriteBatchedStatements is set to true Setting rewriteBatchedStatements to be true by default using MysqlConnectorConfig --- .../main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java | 2 ++ .../java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java index 1c67b3714..36a1eb6c7 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java @@ -28,6 +28,7 @@ public class MysqlConnectorConfig extends AbstractDBSpecificConnectorConfig { private static final String MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s:%s"; private static final String JDBC_PROPERTY_CONNECT_TIMEOUT = "connectTimeout"; private static final String JDBC_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; + private static final String JDBC_REWRITE_BATCHED_STATEMENTS = "rewriteBatchedStatements"; public MysqlConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments) { @@ -56,6 +57,7 @@ public Properties getConnectionArgumentsProperties() { // the unit below is milli-second prop.put(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); prop.put(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + prop.put(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); return prop; } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index 9dfc5b1c8..e5b6ad4b5 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -32,6 +32,7 @@ public void test() throws ClassNotFoundException, IOException { super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + "connectTimeout=20000, socketTimeout=20000}. Error: " + "ConnectException: Connection refused (Connection refused)."); } From 07a18c3f115d95c7571485c130a2ab1583e5c4d0 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Fri, 4 Nov 2022 08:41:29 +0530 Subject: [PATCH 18/36] Bumping the plugin version to 1.8.3-SNAPSHOT (#313) --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 110d57834..e41fa2ab4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 2b59afa2b..d9d34515d 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 1caac20ea..039ddf94b 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index c712cb912..cc6d0d7d0 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.2 + 1.8.3-SNAPSHOT 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.2 + 1.8.3-SNAPSHOT com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index d5a1d1662..f233e972d 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 69e53c92b..863d1671e 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 6372f3db1..f4bc26af4 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 37c6f944a..d0ca26e7f 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 358a246be..fcafe0af9 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index c9dd455e7..d372118be 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 340130f56..37094ec8a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index ab1d862ae..e74f48b3b 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 4fc8ca114..ed44411b8 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 739c29b03..5eceac07a 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index f90b86703..a8e0c1039 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.2 + 1.8.3-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d9becb101..c50ee7046 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 683e8446a..db24bc875 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 3af6982b5..d6e0543f7 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.2 + 1.8.3-SNAPSHOT teradata-plugin From f2d08d36da394412b312a34358c32e6c92f049fd Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:53:21 +0530 Subject: [PATCH 19/36] Using the scale from logical schema instead of the number value. (#314) Using the scale from logical schema instead of the number value. Function rename and doc correction Added Warning logs when detecting precision less Oracle numbers. Added Unit Tests for precision less Number. --- .../plugin/oracle/OracleSourceDBRecord.java | 18 ++-- .../oracle/OracleSourceSchemaReader.java | 12 +++ .../oracle/OracleSourceDBRecordUnitTest.java | 93 +++++++++++++++++++ 3 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 1520ac394..22a7483bb 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -214,19 +214,23 @@ private void handleOracleSpecificType(ResultSet resultSet, StructuredRecord.Buil if (Double.class.getTypeName().equals(resultSet.getMetaData().getColumnClassName(columnIndex))) { recordBuilder.set(field.getName(), resultSet.getDouble(columnIndex)); } else { - // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 - if (precision == 0) { - // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - scale = 0; - } // It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the - // scale of actual value. For example for value '77.12' scale will be '2' even if sql scale is '6' - BigDecimal decimal = resultSet.getBigDecimal(columnIndex, scale); + // scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is + // set to 4 then the number will change to '77.1200'. Also if the value is '22.1274' and the logical schema + // scale is set to 2 then the decimal value used will be '22.13' after rounding. + BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema())); recordBuilder.setDecimal(field.getName(), decimal); } } } + /** + * Get the scale set in Non-nullable schema associated with the schema + * */ + private int getScale(Schema schema) { + return schema.isNullable() ? schema.getNonNullable().getScale() : schema.getScale(); + } + private boolean isLongOrLongRaw(int columnType) { return columnType == OracleSourceSchemaReader.LONG || columnType == OracleSourceSchemaReader.LONG_RAW; } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index c9c9a9e63..adff8d157 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,6 +19,8 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -42,6 +44,11 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { public static final int LONG = -1; public static final int LONG_RAW = -4; + /** + * Logger instance for Oracle Schema reader. + */ + private static final Logger LOG = LoggerFactory.getLogger(OracleSourceSchemaReader.class); + public static final Set ORACLE_TYPES = ImmutableSet.of( INTERVAL_DS, INTERVAL_YM, @@ -89,6 +96,11 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 precision = 38; scale = 0; + LOG.warn(String.format("%s type with undefined precision and scale is detected, " + + "there may be a precision loss while running the pipeline. " + + "Please define an output precision and scale for field '%s' to avoid precision loss.", + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); } return Schema.decimalOf(precision, scale); } diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java new file mode 100644 index 000000000..66e77815c --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Types; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +/** + * Unit Test class for the OracleSourceDBRecord + */ +@RunWith(MockitoJUnitRunner.class) +public class OracleSourceDBRecordUnitTest { + + private static final int DEFAULT_PRECISION = 38; + + @Mock + ResultSet resultSet; + + @Mock + ResultSetMetaData resultSetMetaData; + + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that for Number(0,-127) non nullable type a Number(38,0) is returned by default. + * 2. Ensure that for Number(0,-127) non nullable type a Number(38,4) is returned, + * if schema defined this as Number(38,4). + * 3. Ensure that for Number(0,-127) nullable type a Number(38,0) is returned by default. + * 4. Ensure that for Number(0,-127) nullable type a Number(38,4) is returned, + * if schema defined this as Number(38,4). + * @throws Exception + */ + @Test + public void validatePrecisionLessDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION))); + Schema.Field field4 = Schema.Field.of("ID4", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION, 4))); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2, + field3, + field4 + ); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(2), eq(4))).thenReturn(new BigDecimal("123.4568")); + when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(4), eq(4))).thenReturn(new BigDecimal("123.4568")); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field3, 3, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertEquals(record.getDecimal("ID1").toPlainString(), "123"); + Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.4568"); + Assert.assertEquals(record.getDecimal("ID3").toPlainString(), "123"); + Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.4568"); + } +} From 0e0ed6dcb1dc1cd58f3dd77e47c4e792bacf6ba4 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Fri, 4 Nov 2022 14:01:07 +0530 Subject: [PATCH 20/36] Bumping the plugin version to 1.8.3 (#315) --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index e41fa2ab4..cbc7427e5 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index d9d34515d..3a8f0d3b7 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 039ddf94b..056ee5c2c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index cc6d0d7d0..ad0676741 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.3-SNAPSHOT + 1.8.3 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.3-SNAPSHOT + 1.8.3 com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index f233e972d..c819ecb19 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 863d1671e..029f5366f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index f4bc26af4..edaf030b7 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d0ca26e7f..9fdde2bb5 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index fcafe0af9..e6410feb4 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index d372118be..3db7ab41a 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 37094ec8a..dbaf0efe3 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index e74f48b3b..3121947ad 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index ed44411b8..8aaad7b0a 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 5eceac07a..28c6985d2 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 Oracle plugin diff --git a/pom.xml b/pom.xml index a8e0c1039..e2f2007e8 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.3-SNAPSHOT + 1.8.3 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index c50ee7046..a6141962f 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index db24bc875..986c5c14d 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index d6e0543f7..227b8c4d7 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3-SNAPSHOT + 1.8.3 teradata-plugin From 2dba972833a06dfb977d2dab6208c054b213d583 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Mon, 12 Dec 2022 11:53:21 +0530 Subject: [PATCH 21/36] Converting precision less decimal/numbers into String type. (#326) --- oracle-plugin/docs/Oracle-batchsink.md | 3 +- .../plugin/oracle/OracleSourceDBRecord.java | 17 +++-- .../oracle/OracleSourceSchemaReader.java | 9 +-- .../oracle/OracleSourceDBRecordUnitTest.java | 73 ++++++++++++++++--- 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/oracle-plugin/docs/Oracle-batchsink.md b/oracle-plugin/docs/Oracle-batchsink.md index 23f32dfd4..2538ec8bd 100644 --- a/oracle-plugin/docs/Oracle-batchsink.md +++ b/oracle-plugin/docs/Oracle-batchsink.md @@ -62,7 +62,8 @@ Data Types Mapping | VARCHAR2 | string | | | NVARCHAR2 | string | | | VARCHAR | string | | - | NUMBER | decimal | | + | NUMBER | string | For NUMBER types defined without a precision and scale | + | NUMBER | decimal | For NUMBER types with a defined precision and scale | | FLOAT | double | | | LONG | string | | | DATE | timestamp | | diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 22a7483bb..18bea55ca 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -214,12 +214,17 @@ private void handleOracleSpecificType(ResultSet resultSet, StructuredRecord.Buil if (Double.class.getTypeName().equals(resultSet.getMetaData().getColumnClassName(columnIndex))) { recordBuilder.set(field.getName(), resultSet.getDouble(columnIndex)); } else { - // It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the - // scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is - // set to 4 then the number will change to '77.1200'. Also if the value is '22.1274' and the logical schema - // scale is set to 2 then the decimal value used will be '22.13' after rounding. - BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema())); - recordBuilder.setDecimal(field.getName(), decimal); + if (precision == 0) { + // In case of precision less decimal convert the field to String type + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } else { + // It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the + // scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is + // set to 4 then the number will change to '77.1200'. Also if the value is '22.1274' and the logical schema + // scale is set to 2 then the decimal value used will be '22.13' after rounding. + BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema())); + recordBuilder.setDecimal(field.getName(), decimal); + } } } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index adff8d157..983970d90 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -94,13 +94,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 if (precision == 0) { // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - precision = 38; - scale = 0; - LOG.warn(String.format("%s type with undefined precision and scale is detected, " - + "there may be a precision loss while running the pipeline. " - + "Please define an output precision and scale for field '%s' to avoid precision loss.", + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), metadata.getColumnTypeName(index), metadata.getColumnName(index))); + return Schema.of(Schema.Type.STRING); } return Schema.decimalOf(precision, scale); } diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java index 66e77815c..2c98527d8 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java @@ -48,11 +48,11 @@ public class OracleSourceDBRecordUnitTest { /** * Validate the precision less Numbers handling against following use cases. - * 1. Ensure that for Number(0,-127) non nullable type a Number(38,0) is returned by default. - * 2. Ensure that for Number(0,-127) non nullable type a Number(38,4) is returned, + * 1. Ensure that for Number(0,-127) non nullable type a String is returned by default. + * 2. Ensure that for Number(0,-127) non nullable type a String is returned, * if schema defined this as Number(38,4). - * 3. Ensure that for Number(0,-127) nullable type a Number(38,0) is returned by default. - * 4. Ensure that for Number(0,-127) nullable type a Number(38,4) is returned, + * 3. Ensure that for Number(0,-127) nullable type a String type is returned by default. + * 4. Ensure that for Number(0,-127) nullable type a String is returned, * if schema defined this as Number(38,4). * @throws Exception */ @@ -72,10 +72,10 @@ public void validatePrecisionLessDecimalParsing() throws Exception { ); when(resultSet.getMetaData()).thenReturn(resultSetMetaData); - when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123")); - when(resultSet.getBigDecimal(eq(2), eq(4))).thenReturn(new BigDecimal("123.4568")); - when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123")); - when(resultSet.getBigDecimal(eq(4), eq(4))).thenReturn(new BigDecimal("123.4568")); + when(resultSet.getString(eq(1))).thenReturn("123"); + when(resultSet.getString(eq(2))).thenReturn("123.4568"); + when(resultSet.getString(eq(3))).thenReturn("123"); + when(resultSet.getString(eq(4))).thenReturn("123.4568"); StructuredRecord.Builder builder = StructuredRecord.builder(schema); OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null); @@ -85,9 +85,62 @@ public void validatePrecisionLessDecimalParsing() throws Exception { dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, 0, -127); StructuredRecord record = builder.build(); + Assert.assertTrue(record.get("ID1") instanceof String); + Assert.assertEquals(record.get("ID1"), "123"); + Assert.assertTrue(record.get("ID2") instanceof String); + Assert.assertEquals(record.get("ID2"), "123.4568"); + Assert.assertTrue(record.get("ID3") instanceof String); + Assert.assertEquals(record.get("ID3"), "123"); + Assert.assertTrue(record.get("ID4") instanceof String); + Assert.assertEquals(record.get("ID4"), "123.4568"); + } + + /** + * Validate the default precision Numbers handling against following use cases. + * 1. Ensure that for Number(38, 0) non nullable type a Number(38,0) is returned. + * 2. Ensure that for Number(38, 4) non nullable type a Number(38,6) is returned, + * if schema defined this as Number(38,6). + * 3. Ensure that for Number(38, 0) nullable type a Number(38,0) is returned by default. + * 4. Ensure that for Number(38, 4) nullable type a Number(38,6) is returned, + * if schema defined this as Number(38,6). + * @throws Exception + */ + @Test + public void validateDefaultDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.decimalOf(DEFAULT_PRECISION, 6)); + Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION))); + Schema.Field field4 = Schema.Field.of("ID4", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION, 6))); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2, + field3, + field4 + ); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(2), eq(6))).thenReturn(new BigDecimal("123.456789")); + when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(4), eq(6))).thenReturn(new BigDecimal("123.456789")); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 0); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, DEFAULT_PRECISION, 4); + dbRecord.handleField(resultSet, builder, field3, 3, Types.NUMERIC, DEFAULT_PRECISION, 0); + dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, DEFAULT_PRECISION, 4); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); Assert.assertEquals(record.getDecimal("ID1").toPlainString(), "123"); - Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.4568"); + Assert.assertTrue(record.getDecimal("ID2") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.456789"); + Assert.assertTrue(record.getDecimal("ID3") instanceof BigDecimal); Assert.assertEquals(record.getDecimal("ID3").toPlainString(), "123"); - Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.4568"); + Assert.assertTrue(record.getDecimal("ID4") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.456789"); } } From f4e306c95511e8169ec73ea1fdc4a12d7e62b0ad Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:01:36 +0530 Subject: [PATCH 22/36] Bumped the Version to 1.8.4-SNAPSHOT (#327) --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index cbc7427e5..e1e30616a 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 3a8f0d3b7..8be897037 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 056ee5c2c..0ee4f4e2c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index ad0676741..53d2247ba 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.3 + 1.8.4-SNAPSHOT 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.3 + 1.8.4-SNAPSHOT com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index c819ecb19..57cb091ec 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 029f5366f..9875b6c37 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index edaf030b7..dfe26fc0c 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 9fdde2bb5..f34985b49 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index e6410feb4..5b4629e80 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 3db7ab41a..9302901fd 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index dbaf0efe3..836ae3d73 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 3121947ad..8df30ca50 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 8aaad7b0a..0c104ca27 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 28c6985d2..c6f9e3942 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index e2f2007e8..79bb34220 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.3 + 1.8.4-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index a6141962f..b297b5792 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 986c5c14d..9b620e0b6 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 227b8c4d7..eafc82cdf 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.3 + 1.8.4-SNAPSHOT teradata-plugin From d6995a66eb530a8ba5ec67a597db81d380233d7b Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Mon, 12 Dec 2022 16:13:35 +0530 Subject: [PATCH 23/36] Bumping up plugin version to 1.8.4 (#329) --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 6 +++--- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index e1e30616a..344958d5e 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 8be897037..d3d141b05 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 0ee4f4e2c..f362c4c08 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 53d2247ba..d3cde3c0a 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,12 +20,12 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.4-SNAPSHOT + 1.8.4 4.0.0 @@ -46,7 +46,7 @@ io.cdap.plugin postgresql-plugin - 1.8.4-SNAPSHOT + 1.8.4 com.google.guava diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 57cb091ec..3a25bf483 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 9875b6c37..e2c0c41ee 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index dfe26fc0c..8975a3b42 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index f34985b49..1ac711fda 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 5b4629e80..0b707d753 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 9302901fd..bca3cbf13 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 836ae3d73..5446467a4 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 8df30ca50..a03d0006f 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 0c104ca27..0e473f8ba 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index c6f9e3942..bc34fb417 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 Oracle plugin diff --git a/pom.xml b/pom.xml index 79bb34220..9c5f26de5 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.4-SNAPSHOT + 1.8.4 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index b297b5792..6df46b183 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 9b620e0b6..ab6eb7473 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index eafc82cdf..060de7d6b 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4-SNAPSHOT + 1.8.4 teradata-plugin From 8a8b91784d68594bae69d6f80b7b10e89cbf092a Mon Sep 17 00:00:00 2001 From: sahusanket Date: Wed, 18 Jan 2023 17:49:24 +0530 Subject: [PATCH 24/36] Bumped versions to next SNAPSHOT. --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 344958d5e..f50c497d2 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index d3d141b05..3f5603b6a 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f362c4c08..ccd59ebc1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index d3cde3c0a..29d6e8de5 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 3a25bf483..19f4abc73 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index e2c0c41ee..26c443383 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 8975a3b42..1200f46f1 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 1ac711fda..09c4b5159 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0b707d753..99047e991 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bca3cbf13..f5bfa2694 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 5446467a4..b8af6a7aa 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index a03d0006f..374684937 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 0e473f8ba..b434e2206 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index bc34fb417..4e97b3664 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 9c5f26de5..bd7563fc7 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.4 + 1.8.5-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 6df46b183..057429197 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index ab6eb7473..dbd270e62 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 060de7d6b..dea519f14 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.4 + 1.8.5-SNAPSHOT teradata-plugin From 10fe05d021a00fe43bef4fc4abab32e3ae6999dd Mon Sep 17 00:00:00 2001 From: Sumit Jain Date: Mon, 23 Jan 2023 12:42:37 +0530 Subject: [PATCH 25/36] Remove redundant version identifiers which can go out of sync to fix the build --- cloudsql-postgresql-plugin/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 29d6e8de5..6249b3bbf 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -25,7 +25,6 @@ CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin - 1.8.4 4.0.0 @@ -46,7 +45,7 @@ io.cdap.plugin postgresql-plugin - 1.8.4 + ${project.version} com.google.guava From 6797e579e6ae294305812e3965d8e5b0be17dd6c Mon Sep 17 00:00:00 2001 From: Sumit Jain Date: Wed, 22 Feb 2023 13:35:50 +0530 Subject: [PATCH 26/36] Removed SNAPSHOT from pom files. --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index f50c497d2..ebf49707b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 3f5603b6a..b7db1ed67 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index ccd59ebc1..d96e49b82 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 6249b3bbf..1667b3182 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 19f4abc73..aab5568d3 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 26c443383..073e1bb1e 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 1200f46f1..1c95cf3ce 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 09c4b5159..3af12e52b 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 99047e991..02b295971 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f5bfa2694..2689dacce 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index b8af6a7aa..2ddb3cb8a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 374684937..028ade083 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index b434e2206..815dc48e2 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4e97b3664..ed8ff5a7b 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 Oracle plugin diff --git a/pom.xml b/pom.xml index bd7563fc7..30748a5b7 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.5-SNAPSHOT + 1.8.5 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 057429197..6769fe4c1 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index dbd270e62..ec5128e26 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index dea519f14..5221fe276 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5-SNAPSHOT + 1.8.5 teradata-plugin From f87d6033a664c69199ccf0621420fc000a8c9d27 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:42:12 +0530 Subject: [PATCH 27/36] Backward Compatibility Fix for Oracle Number type without precision and scale. (#365) (#366) --- oracle-plugin/docs/Oracle-batchsink.md | 63 +++++++-------- oracle-plugin/docs/Oracle-batchsource.md | 60 ++++++++------- .../io/cdap/plugin/oracle/OracleSource.java | 16 ++++ .../plugin/oracle/OracleSourceDBRecord.java | 12 ++- .../oracle/OracleSourceDBRecordUnitTest.java | 77 +++++++++++++++---- 5 files changed, 149 insertions(+), 79 deletions(-) diff --git a/oracle-plugin/docs/Oracle-batchsink.md b/oracle-plugin/docs/Oracle-batchsink.md index 2538ec8bd..d52956a86 100644 --- a/oracle-plugin/docs/Oracle-batchsink.md +++ b/oracle-plugin/docs/Oracle-batchsink.md @@ -57,37 +57,38 @@ will be passed to the JDBC driver as connection arguments for JDBC drivers that Data Types Mapping ---------- - | Oracle Data Type | CDAP Schema Data Type | Comment | - | ------------------------------ | --------------------- | ------------------------------------------------------ | - | VARCHAR2 | string | | - | NVARCHAR2 | string | | - | VARCHAR | string | | - | NUMBER | string | For NUMBER types defined without a precision and scale | - | NUMBER | decimal | For NUMBER types with a defined precision and scale | - | FLOAT | double | | - | LONG | string | | - | DATE | timestamp | | - | BINARY_FLOAT | float | | - | BINARY_DOUBLE | double | | - | TIMESTAMP | timestamp | | - | TIMESTAMP WITH TIME ZONE | string | Timestamp string in the following format: | - | | | "2019-07-15 15:57:46.65 GMT" | - | TIMESTAMP WITH LOCAL TIME ZONE | timestamp | | - | INTERVAL YEAR TO MONTH | string | Oracle's 'INTERVAL YEAR TO MONTH' literal in the | - | | | standard format: "year[-month]" | - | INTERVAL DAY TO SECOND | string | Oracle's 'INTERVAL DAY TO SECOND' literal in the | - | | | standard format: | - | | | "[day] [hour][:minutes][:seconds[.milliseconds]" | - | RAW | bytes | | - | LONG RAW | bytes | | - | ROWID | string | | - | UROWID | string | | - | CHAR | string | | - | NCHAR | string | | - | CLOB | string | | - | NCLOB | string | | - | BLOB | bytes | | - | BFILE | | BFILE data type is not supported for the sink | + | Oracle Data Type | CDAP Schema Data Type | Comment | + | ------------------------------ | --------------------- | -----------------------------------------------------------| + | VARCHAR2 | string | | + | NVARCHAR2 | string | | + | VARCHAR | string | | + | NUMBER | string | For NUMBER types defined without a precision and scale. | + | | | Users can manually set output schema to map it to Decimal. | + | NUMBER | decimal | For NUMBER types defined with a precision and scale. | + | FLOAT | double | | + | LONG | string | | + | DATE | timestamp | | + | BINARY_FLOAT | float | | + | BINARY_DOUBLE | double | | + | TIMESTAMP | timestamp | | + | TIMESTAMP WITH TIME ZONE | string | Timestamp string in the following format: | + | | | "2019-07-15 15:57:46.65 GMT" | + | TIMESTAMP WITH LOCAL TIME ZONE | timestamp | | + | INTERVAL YEAR TO MONTH | string | Oracle's 'INTERVAL YEAR TO MONTH' literal in the | + | | | standard format: "year[-month]" | + | INTERVAL DAY TO SECOND | string | Oracle's 'INTERVAL DAY TO SECOND' literal in the | + | | | standard format: | + | | | "[day] [hour][:minutes][:seconds[.milliseconds]" | + | RAW | bytes | | + | LONG RAW | bytes | | + | ROWID | string | | + | UROWID | string | | + | CHAR | string | | + | NCHAR | string | | + | CLOB | string | | + | NCLOB | string | | + | BLOB | bytes | | + | BFILE | | BFILE data type is not supported for the sink | Example diff --git a/oracle-plugin/docs/Oracle-batchsource.md b/oracle-plugin/docs/Oracle-batchsource.md index 1dbdf9836..6d014a3ca 100644 --- a/oracle-plugin/docs/Oracle-batchsource.md +++ b/oracle-plugin/docs/Oracle-batchsource.md @@ -73,35 +73,37 @@ with the tradeoff of higher memory usage. Data Types Mapping ---------- - | Oracle Data Type | CDAP Schema Data Type | Comment | - | ------------------------------ | --------------------- | ------------------------------------------------------ | - | VARCHAR2 | string | | - | NVARCHAR2 | string | | - | VARCHAR | string | | - | NUMBER | decimal | | - | FLOAT | double | | - | LONG | string | | - | DATE | timestamp | | - | BINARY_FLOAT | float | | - | BINARY_DOUBLE | double | | - | TIMESTAMP | timestamp | | - | TIMESTAMP WITH TIME ZONE | string | | - | TIMESTAMP WITH LOCAL TIME ZONE | timestamp | | - | INTERVAL YEAR TO MONTH | string | | - | INTERVAL DAY TO SECOND | string | | - | RAW | bytes | | - | LONG RAW | bytes | | - | ROWID | string | | - | UROWID | string | | - | CHAR | string | | - | NCHAR | string | | - | CLOB | string | | - | NCLOB | string | | - | BLOB | bytes | | - | BFILE | bytes | BFILE is a data type used to store a locator (link) | - | | | to an external file, which is stored outside of the | - | | | database. Only the locator will be read from an | - | | | Oracle table and not the content of the external file. | + | Oracle Data Type | CDAP Schema Data Type | Comment | + | ------------------------------ | --------------------- | -----------------------------------------------------------| + | VARCHAR2 | string | | + | NVARCHAR2 | string | | + | VARCHAR | string | | + | NUMBER | string | For NUMBER types defined without a precision and scale. | + | | | Users can manually set output schema to map it to Decimal. | + | NUMBER | decimal | For NUMBER types defined with a precision and scale. | + | FLOAT | double | | + | LONG | string | | + | DATE | timestamp | | + | BINARY_FLOAT | float | | + | BINARY_DOUBLE | double | | + | TIMESTAMP | timestamp | | + | TIMESTAMP WITH TIME ZONE | string | | + | TIMESTAMP WITH LOCAL TIME ZONE | timestamp | | + | INTERVAL YEAR TO MONTH | string | | + | INTERVAL DAY TO SECOND | string | | + | RAW | bytes | | + | LONG RAW | bytes | | + | ROWID | string | | + | UROWID | string | | + | CHAR | string | | + | NCHAR | string | | + | CLOB | string | | + | NCLOB | string | | + | BLOB | bytes | | + | BFILE | bytes | BFILE is a data type used to store a locator (link) | + | | | to an external file, which is stored outside of the | + | | | database. Only the locator will be read from an | + | | | Oracle table and not the content of the external file. | Example diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 4dafdf95b..d69da192d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.connector.Connector; import io.cdap.plugin.common.ConfigUtil; @@ -133,6 +134,21 @@ public void validate(FailureCollector collector) { public String getTransactionIsolationLevel() { return connection.getTransactionIsolationLevel(); } + + @Override + protected void validateField(FailureCollector collector, + Schema.Field field, Schema actualFieldSchema, Schema expectedFieldSchema) { + // This change is needed to make sure that the pipeline upgrade continues to work post upgrade. + // Since the older handling of the precision less used to convert to the decimal type, + // and the new version would try to convert to the String type. In that case the output schema would + // contain Decimal(38, 0) (or something similar), and the code internally would try to identify + // the schema of the field(without precision and scale) as String. + if (Schema.LogicalType.DECIMAL.equals(expectedFieldSchema.getLogicalType()) + && actualFieldSchema.getType().equals(Schema.Type.STRING)) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } /** diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 18bea55ca..a8f3b163e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -215,8 +215,16 @@ private void handleOracleSpecificType(ResultSet resultSet, StructuredRecord.Buil recordBuilder.set(field.getName(), resultSet.getDouble(columnIndex)); } else { if (precision == 0) { - // In case of precision less decimal convert the field to String type - recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) { + // Handle the field using the schema set in the output schema + BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema())); + recordBuilder.setDecimal(field.getName(), decimal); + } else { + // In case of Number defined without precision and scale convert to String type + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } } else { // It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the // scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java index 2c98527d8..0ae60ad0f 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java @@ -48,20 +48,18 @@ public class OracleSourceDBRecordUnitTest { /** * Validate the precision less Numbers handling against following use cases. - * 1. Ensure that for Number(0,-127) non nullable type a String is returned by default. - * 2. Ensure that for Number(0,-127) non nullable type a String is returned, - * if schema defined this as Number(38,4). - * 3. Ensure that for Number(0,-127) nullable type a String type is returned by default. - * 4. Ensure that for Number(0,-127) nullable type a String is returned, - * if schema defined this as Number(38,4). + * 1. Ensure that for Number(0,-127) non nullable type a String type is returned if output schema is String. + * 2. Ensure that for Number(0,-127) non nullable type a String type is returned if output schema is String. + * 3. Ensure that for Number(0,-127) nullable type a String type is returned if output schema is String. + * 4. Ensure that for Number(0,-127) nullable type a String type is returned if output schema is String. * @throws Exception */ @Test - public void validatePrecisionLessDecimalParsing() throws Exception { - Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION)); - Schema.Field field2 = Schema.Field.of("ID2", Schema.decimalOf(DEFAULT_PRECISION, 4)); - Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION))); - Schema.Field field4 = Schema.Field.of("ID4", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION, 4))); + public void validatePrecisionLessNumberParsingForOutputSchemaAsString() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.of(Schema.Type.STRING)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.of(Schema.Type.STRING)); + Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.of(Schema.Type.STRING))); + Schema.Field field4 = Schema.Field.of("ID4", Schema.nullableOf(Schema.of(Schema.Type.STRING))); Schema schema = Schema.recordOf( "dbRecord", @@ -95,14 +93,59 @@ public void validatePrecisionLessDecimalParsing() throws Exception { Assert.assertEquals(record.get("ID4"), "123.4568"); } + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that for Number(0,-127) non nullable type a Decimal type is returned if output schema is Decimal. + * 2. Ensure that for Number(0,-127) non nullable type a String is returned if output schema is Decimal. + * 3. Ensure that for Number(0,-127) nullable type a String type is returned if output schema is Decimal. + * 4. Ensure that for Number(0,-127) nullable type a String is returned if output schema is Decimal. + * @throws Exception + */ + @Test + public void validatePrecisionLessNumberParsingForOutputSchemaAsDecimal() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION))); + Schema.Field field4 = Schema.Field.of("ID4", Schema.decimalOf(DEFAULT_PRECISION, 4)); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2, + field3, + field4 + ); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(2), eq(4))).thenReturn(new BigDecimal("123.4568")); + when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123")); + when(resultSet.getBigDecimal(eq(4), eq(4))).thenReturn(new BigDecimal("123.4568")); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field3, 3, Types.NUMERIC, 0, -127); + dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID1").toPlainString(), "123"); + Assert.assertTrue(record.getDecimal("ID2") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.4568"); + Assert.assertTrue(record.getDecimal("ID3") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID3").toPlainString(), "123"); + Assert.assertTrue(record.getDecimal("ID4") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.4568"); + } + /** * Validate the default precision Numbers handling against following use cases. - * 1. Ensure that for Number(38, 0) non nullable type a Number(38,0) is returned. - * 2. Ensure that for Number(38, 4) non nullable type a Number(38,6) is returned, - * if schema defined this as Number(38,6). - * 3. Ensure that for Number(38, 0) nullable type a Number(38,0) is returned by default. - * 4. Ensure that for Number(38, 4) nullable type a Number(38,6) is returned, - * if schema defined this as Number(38,6). + * 1. Ensure that for Number(38, 0) non nullable type a Number(38,0) is returned if output schema is Decimal. + * 2. Ensure that for Number(38, 4) non nullable type a Number(38,6) is returned if output schema is Decimal. + * 3. Ensure that for Number(38, 0) nullable type a Number(38,0) is returned if output schema is Decimal. + * 4. Ensure that for Number(38, 4) nullable type a Number(38,6) is returned if output schema is Decimal. * @throws Exception */ @Test From 2d39427885ccdc95eafc2a3126bbc625144e3fa8 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:43:09 +0530 Subject: [PATCH 28/36] Version Bump to 1.8.6-SNAPSHOT (#368) --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ebf49707b..97bee0c46 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index b7db1ed67..660cdc49c 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index d96e49b82..3b53f50a0 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 1667b3182..f4798fcb1 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index aab5568d3..48c308294 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 073e1bb1e..d28ea2f48 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 1c95cf3ce..95442427f 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 3af12e52b..b6af18468 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 02b295971..91d3ae618 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 2689dacce..b33ee453e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 2ddb3cb8a..d2b9f6dcc 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 028ade083..5fb3bba00 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 815dc48e2..6053b8faa 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index ed8ff5a7b..42f1602c9 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 30748a5b7..52060927a 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.5 + 1.8.6-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 6769fe4c1..664627ff9 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index ec5128e26..6390ed12f 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 5221fe276..07639253a 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.5 + 1.8.6-SNAPSHOT teradata-plugin From 3d3f4f12b49f4fc67b9677f433cc83a2d1518c62 Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Tue, 14 Mar 2023 15:14:12 +0530 Subject: [PATCH 29/36] Version bump to 1.8.6 --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 97bee0c46..8c4f2345b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 660cdc49c..394a48de2 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 3b53f50a0..6a0558d6c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index f4798fcb1..000c25796 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 48c308294..661395e6d 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index d28ea2f48..057699929 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 95442427f..45d68157b 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index b6af18468..ef42ecfc4 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 91d3ae618..054becc99 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index b33ee453e..00245fc5c 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index d2b9f6dcc..e7b43999a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 5fb3bba00..664c5c312 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 6053b8faa..9286ab29a 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 42f1602c9..2756f12ca 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 Oracle plugin diff --git a/pom.xml b/pom.xml index 52060927a..9490f7a17 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.6-SNAPSHOT + 1.8.6 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 664627ff9..1f391ae5c 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6390ed12f..7debcdd63 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 07639253a..84b23f508 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6-SNAPSHOT + 1.8.6 teradata-plugin From 2820822a2e8436aff37dcda02f470640381aa31a Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Thu, 24 Aug 2023 20:09:09 +0530 Subject: [PATCH 30/36] Version BumpUp --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 8c4f2345b..a8a0ca310 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 394a48de2..4d0a3741c 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 6a0558d6c..83c82dcf1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 000c25796..d00264147 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 661395e6d..04e7ecaa4 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 057699929..c6a0609a4 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 45d68157b..2e1495106 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index ef42ecfc4..9b47e09b2 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 054becc99..03450ee28 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 00245fc5c..9f6ba5a71 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index e7b43999a..ad11e22e5 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 664c5c312..5cedd287e 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 9286ab29a..7f78db2cd 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 2756f12ca..7190ef4b1 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 9490f7a17..2ffa867e6 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.6 + 1.8.7-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 1f391ae5c..a4fbcdb9a 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7debcdd63..a3e9ed907 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 84b23f508..a8e4682f5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.6 + 1.8.7-SNAPSHOT teradata-plugin From 254a37d52c9c4f4e90face3e3d4331b3412503e7 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Fri, 25 Aug 2023 11:25:21 +0530 Subject: [PATCH 31/36] Postgres BugFix --- .../docs/CloudSQLPostgreSQL-batchsink.md | 1 + .../docs/CloudSQLPostgreSQL-batchsource.md | 1 + postgresql-plugin/docs/Postgres-batchsink.md | 1 + .../docs/Postgres-batchsource.md | 1 + postgresql-plugin/pom.xml | 10 ++- .../plugin/postgres/PostgresDBRecord.java | 54 +++++++++---- .../postgres/PostgresFieldsValidator.java | 8 ++ .../plugin/postgres/PostgresSchemaReader.java | 16 ++++ .../cdap/plugin/postgres/PostgresSource.java | 18 +++++ .../postgres/PostgresDBRecordUnitTest.java | 81 +++++++++++++++++++ 10 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresDBRecordUnitTest.java diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md index d0d60baca..ab57cdf58 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md @@ -147,6 +147,7 @@ Please, refer to PostgreSQL data types documentation to figure out proper format | double precision | double | | | integer | int | | | numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | | real | float | | | smallint | int | | | text | string | | diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md index a55361779..534ea5660 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md @@ -171,6 +171,7 @@ Please, refer to PostgreSQL data types documentation to figure out proper format | double precision | double | | | integer | int | | | numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | | real | float | | | smallint | int | | | smallserial | int | | diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index 9159c2c1d..1bbbf74d8 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -78,6 +78,7 @@ Please, refer to PostgreSQL data types documentation to figure out proper format | double precision | double | | | integer | int | | | numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | | real | float | | | smallint | int | | | text | string | | diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index 340d66963..3e0c0ca9b 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -109,6 +109,7 @@ Please, refer to PostgreSQL data types documentation to figure out proper format | double precision | double | | | integer | int | | | numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | | real | float | | | smallint | int | | | smallserial | int | | diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 1f391ae5c..1938b07c7 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -53,10 +53,6 @@ 42.2.20 test - - org.mockito - mockito-core - io.cdap.plugin database-commons @@ -72,9 +68,15 @@ io.cdap.cdap cdap-data-pipeline2_2.11 + + org.mockito + mockito-core + test + junit junit + test io.cdap.cdap diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java index dde6bdcdd..629eefad5 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java @@ -24,10 +24,13 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.List; /** @@ -49,24 +52,41 @@ public PostgresDBRecord() { @Override protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, int columnIndex, int sqlType, int sqlPrecision, int sqlScale) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); if (isUseSchema(resultSet.getMetaData(), columnIndex)) { setFieldAccordingToSchema(resultSet, recordBuilder, field, columnIndex); - } else { - setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); + return; } + int columnType = metadata.getColumnType(columnIndex); + if (columnType == Types.NUMERIC) { + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + int precision = metadata.getPrecision(columnIndex); + if (precision == 0 && Schema.Type.STRING.equals(nonNullableSchema.getType())) { + // When output schema is set to String for precision less numbers + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + return; + } + BigDecimal orgValue = resultSet.getBigDecimal(columnIndex); + if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType()) && orgValue != null) { + BigDecimal decimalValue = new BigDecimal(orgValue.toPlainString()) + .setScale(nonNullableSchema.getScale(), RoundingMode.HALF_EVEN); + recordBuilder.setDecimal(field.getName(), decimalValue); + return; + } + } + setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); } private static boolean isUseSchema(ResultSetMetaData metadata, int columnIndex) throws SQLException { - switch (metadata.getColumnTypeName(columnIndex)) { - case "bit": - case "timetz": - case "money": - return true; - default: - return PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES.contains(metadata.getColumnType(columnIndex)); - } + String columnTypeName = metadata.getColumnTypeName(columnIndex); + // If the column Type Name is present in the String mapped PostgreSQL types then return true. + return (PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES_NAMES.contains(columnTypeName) + || PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES.contains(metadata.getColumnType(columnIndex))); + } + private Object createPGobject(String type, String value, ClassLoader classLoader) throws SQLException { try { Class pGObjectClass = classLoader.loadClass("org.postgresql.util.PGobject"); @@ -89,11 +109,17 @@ protected void writeToDB(PreparedStatement stmt, Schema.Field field, int fieldIn if (PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES_NAMES.contains(columnType.getTypeName()) || PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES.contains(columnType.getType())) { stmt.setObject(sqlIndex, createPGobject(columnType.getTypeName(), - record.get(field.getName()), - stmt.getClass().getClassLoader())); - } else { - super.writeToDB(stmt, field, fieldIndex); + record.get(field.getName()), + stmt.getClass().getClassLoader())); + return; + } + if (columnType.getType() == Types.NUMERIC && record.get(field.getName()) != null && + field.getSchema().getType() == Schema.Type.STRING) { + stmt.setBigDecimal(sqlIndex, new BigDecimal((String) record.get(field.getName()))); + return; } + + super.writeToDB(stmt, field, fieldIndex); } @Override diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresFieldsValidator.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresFieldsValidator.java index b9fbef295..b2f71d940 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresFieldsValidator.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresFieldsValidator.java @@ -21,6 +21,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.Objects; /** @@ -45,6 +46,13 @@ public boolean isFieldCompatible(Schema.Field field, ResultSetMetaData metadata, return false; } } + // Since Numeric types without precision and scale are getting converted into CDAP String type at the Source + // plugin, hence making the String type compatible with the Numeric type at the Sink as well. + if (fieldType.equals(Schema.Type.STRING)) { + if (Types.NUMERIC == columnType) { + return true; + } + } return super.isFieldCompatible(field, metadata, index); } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSchemaReader.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSchemaReader.java index 8b3aafff6..f24b32cee 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSchemaReader.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSchemaReader.java @@ -20,6 +20,9 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; @@ -30,6 +33,8 @@ */ public class PostgresSchemaReader extends CommonSchemaReader { + private static final Logger LOG = LoggerFactory.getLogger(PostgresSchemaReader.class); + public static final Set STRING_MAPPED_POSTGRES_TYPES = ImmutableSet.of( Types.OTHER, Types.ARRAY, Types.SQLXML ); @@ -46,6 +51,17 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti if (STRING_MAPPED_POSTGRES_TYPES_NAMES.contains(typeName) || STRING_MAPPED_POSTGRES_TYPES.contains(columnType)) { return Schema.of(Schema.Type.STRING); } + // If it is a numeric type without precision then use the Schema of String to avoid any precision loss + if (Types.NUMERIC == columnType) { + int precision = metadata.getPrecision(index); + if (precision == 0) { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index))); + return Schema.of(Schema.Type.STRING); + } + } return super.getSchema(metadata, index); } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 963bba263..5a23b529c 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.connector.Connector; @@ -123,5 +124,22 @@ public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); super.validate(collector); } + + @Override + protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema, + Schema expectedFieldSchema) { + + // This change is needed to make sure that the pipeline upgrade continues to work post upgrade. + // Since the older handling of the precision less used to convert to the decimal type, + // and the new version would try to convert to the String type. In that case the output schema would + // contain Decimal(38, 0) (or something similar), and the code internally would try to identify + // the schema of the field(without precision and scale) as String. + if (Schema.LogicalType.DECIMAL.equals(expectedFieldSchema.getLogicalType()) && + actualFieldSchema.getType().equals(Schema.Type.STRING)) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } } + diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresDBRecordUnitTest.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresDBRecordUnitTest.java new file mode 100644 index 000000000..53a8795b3 --- /dev/null +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresDBRecordUnitTest.java @@ -0,0 +1,81 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.postgres; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Types; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + + +@RunWith(MockitoJUnitRunner.class) + public class PostgresDBRecordUnitTest { + + private static final int DEFAULT_PRECISION = 38; + + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that the numeric type with [p,s] set as [38,4] detect as BigDecimal(38,4) in cdap. + * 2. Ensure that the numeric type without [p,s] detect as String type in cdap. + * @throws Exception + */ + @Test + public void validatePrecisionLessDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.of(Schema.Type.STRING)); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2 + ); + + ResultSetMetaData resultSetMetaData = Mockito.mock(ResultSetMetaData.class); + when(resultSetMetaData.getColumnType(eq(1))).thenReturn(Types.NUMERIC); + when(resultSetMetaData.getPrecision(eq(1))).thenReturn(DEFAULT_PRECISION); + when(resultSetMetaData.getColumnType(eq(2))).thenReturn(Types.NUMERIC); + when(resultSetMetaData.getPrecision(eq(2))).thenReturn(0); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1))).thenReturn(BigDecimal.valueOf(123.4568)); + when(resultSet.getString(eq(2))).thenReturn("123.4568"); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + PostgresDBRecord dbRecord = new PostgresDBRecord(null, null); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 4); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID1"), BigDecimal.valueOf(123.4568)); + Assert.assertTrue(record.get("ID2") instanceof String); + Assert.assertEquals(record.get("ID2"), "123.4568"); + } +} From e169702283783de08bff99fa4f746469e6f90680 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Wed, 30 Aug 2023 20:00:20 +0530 Subject: [PATCH 32/36] Version 1.8.7-SNAPSHOT-> 1.8.7 --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index a8a0ca310..fadf56740 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 4d0a3741c..a0a260a35 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 83c82dcf1..c6e95c62d 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index d00264147..13ee948ca 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 04e7ecaa4..4e028139b 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index c6a0609a4..1353eed5f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 2e1495106..178367c12 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 9b47e09b2..51bc38238 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 03450ee28..44329ad66 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 9f6ba5a71..bdc74cde8 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index ad11e22e5..eb8d8db6d 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 5cedd287e..c349cb234 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 7f78db2cd..a7e783797 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7190ef4b1..36a50e14f 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Oracle plugin diff --git a/pom.xml b/pom.xml index 2ffa867e6..000a07489 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.7-SNAPSHOT + 1.8.7 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 0c775363a..a1e94ed48 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index a3e9ed907..f1aa752d9 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index a8e4682f5..027dd78af 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 teradata-plugin From 232f2c30b7dbaa15c22cb7dabddb24fa1cc56447 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Tue, 12 Sep 2023 11:53:47 +0530 Subject: [PATCH 33/36] version_revert_1.8.7-SNAPSHOT --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index fadf56740..a8a0ca310 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a0a260a35..4d0a3741c 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c6e95c62d..83c82dcf1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 13ee948ca..d00264147 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 4e028139b..04e7ecaa4 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 1353eed5f..c6a0609a4 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 178367c12..2e1495106 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 51bc38238..9b47e09b2 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 44329ad66..03450ee28 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bdc74cde8..9f6ba5a71 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index eb8d8db6d..ad11e22e5 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index c349cb234..5cedd287e 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a7e783797..7f78db2cd 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 36a50e14f..7190ef4b1 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 000a07489..2ffa867e6 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.7 + 1.8.7-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index a1e94ed48..0c775363a 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index f1aa752d9..a3e9ed907 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 027dd78af..a8e4682f5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.7-SNAPSHOT teradata-plugin From 256853ca6d3ad8087ff9f71e817ca11e24d40355 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Tue, 12 Sep 2023 12:10:03 +0530 Subject: [PATCH 34/36] numeric_precision_fix --- database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java | 2 +- .../main/java/io/cdap/plugin/postgres/PostgresDBRecord.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index 7592a5fdf..f4c021ecf 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -218,7 +218,7 @@ public void write(PreparedStatement stmt) throws SQLException { } } - private Schema getNonNullableSchema(Schema.Field field) { + protected Schema getNonNullableSchema(Schema.Field field) { Schema schema = field.getSchema(); if (field.getSchema().isNullable()) { schema = field.getSchema().getNonNullable(); diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java index 629eefad5..48ee35909 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresDBRecord.java @@ -106,6 +106,7 @@ private Object createPGobject(String type, String value, ClassLoader classLoader protected void writeToDB(PreparedStatement stmt, Schema.Field field, int fieldIndex) throws SQLException { int sqlIndex = fieldIndex + 1; ColumnType columnType = columnTypes.get(fieldIndex); + Schema fieldSchema = getNonNullableSchema(field); if (PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES_NAMES.contains(columnType.getTypeName()) || PostgresSchemaReader.STRING_MAPPED_POSTGRES_TYPES.contains(columnType.getType())) { stmt.setObject(sqlIndex, createPGobject(columnType.getTypeName(), @@ -114,7 +115,7 @@ protected void writeToDB(PreparedStatement stmt, Schema.Field field, int fieldIn return; } if (columnType.getType() == Types.NUMERIC && record.get(field.getName()) != null && - field.getSchema().getType() == Schema.Type.STRING) { + fieldSchema.getType() == Schema.Type.STRING) { stmt.setBigDecimal(sqlIndex, new BigDecimal((String) record.get(field.getName()))); return; } From 5f1bda60b25b3fda54aaac7cac1dd5f904ef7c28 Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Wed, 30 Aug 2023 20:00:20 +0530 Subject: [PATCH 35/36] Version 1.8.7-SNAPSHOT-> 1.8.7 --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index a8a0ca310..fadf56740 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 4d0a3741c..a0a260a35 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 83c82dcf1..c6e95c62d 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index d00264147..13ee948ca 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 04e7ecaa4..4e028139b 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index c6a0609a4..1353eed5f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 2e1495106..178367c12 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 9b47e09b2..51bc38238 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 03450ee28..44329ad66 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 9f6ba5a71..bdc74cde8 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index ad11e22e5..eb8d8db6d 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 5cedd287e..c349cb234 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 7f78db2cd..a7e783797 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7190ef4b1..36a50e14f 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 Oracle plugin diff --git a/pom.xml b/pom.xml index 2ffa867e6..000a07489 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.7-SNAPSHOT + 1.8.7 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 0c775363a..a1e94ed48 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index a3e9ed907..f1aa752d9 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index a8e4682f5..027dd78af 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7-SNAPSHOT + 1.8.7 teradata-plugin From b22e06d2cd2835d9a04605edc754eb038d882069 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 20 Sep 2023 17:09:15 +0530 Subject: [PATCH 36/36] Redshift source and connector plugin added. Redshift source and connector plugin added. --- .../docs/Redshift-batchsource.md | 104 ++++++ .../docs/Redshift-connector.md | 27 ++ .../icons/Redshift-batchsource.png | Bin 0 -> 6145 bytes amazon-redshift-plugin/pom.xml | 134 +++++++ .../amazon/redshift/RedshiftConnector.java | 108 ++++++ .../redshift/RedshiftConnectorConfig.java | 87 +++++ .../amazon/redshift/RedshiftConstants.java | 27 ++ .../amazon/redshift/RedshiftDBRecord.java | 130 +++++++ .../amazon/redshift/RedshiftSchemaReader.java | 121 +++++++ .../amazon/redshift/RedshiftSource.java | 119 +++++++ .../redshift/RedshiftPluginTestBase.java | 218 ++++++++++++ .../redshift/RedshiftPluginTestSuite.java | 31 ++ .../redshift/RedshiftSourceTestRun.java | 332 ++++++++++++++++++ .../widgets/Redshift-batchsource.json | 240 +++++++++++++ .../widgets/Redshift-connector.json | 75 ++++ aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- .../java/io/cdap/plugin/util/DBUtils.java | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 3 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 34 files changed, 1773 insertions(+), 19 deletions(-) create mode 100644 amazon-redshift-plugin/docs/Redshift-batchsource.md create mode 100644 amazon-redshift-plugin/docs/Redshift-connector.md create mode 100644 amazon-redshift-plugin/icons/Redshift-batchsource.png create mode 100644 amazon-redshift-plugin/pom.xml create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java create mode 100644 amazon-redshift-plugin/widgets/Redshift-batchsource.json create mode 100644 amazon-redshift-plugin/widgets/Redshift-connector.json diff --git a/amazon-redshift-plugin/docs/Redshift-batchsource.md b/amazon-redshift-plugin/docs/Redshift-batchsource.md new file mode 100644 index 000000000..7bc81cfc0 --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-batchsource.md @@ -0,0 +1,104 @@ +# Amazon RedShift Batch Source + + +Description +----------- +Reads from an Amazon RedShift database using a configurable SQL query. +Outputs one record for each row returned by the query. + + +Use Case +-------- +The source is used whenever you need to read from an Amazon RedShift database. For example, you may want +to create daily snapshots of a database table by using this source and writing to +a TimePartitionedFileSet. + + +Properties +---------- +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host URL of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Import Query:** The SELECT query to use to import data from the specified table. +You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should +contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. +The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. +The '$CONDITIONS' string is not required if numSplits is set to one. + +**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. + +**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. + +**Number of Splits to Generate:** Number of splits to generate. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. + +**Schema:** The schema of records output by the source. This will be used in place of whatever schema comes +back from the query. However, it must match the schema that comes back from the query, +except it can mark fields as nullable and can contain a subset of the fields. + +**Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, +with the tradeoff of higher memory usage. + +Example +------ +Suppose you want to read data from an Amazon Redshift database named "prod" that is running on +"redshift.xyz.eu-central-1.redshift.amazonaws.com", port 5439, as "sa" user with "Test11" password. +Ensure that the driver for Redshift is installed (you can also provide driver name for some specific driver, +otherwise "redshift" will be used), then configure the plugin with:then configure plugin with: + + +``` +Reference Name: "src1" +Driver Name: "redshift" +Host: "redshift.xyz.eu-central-1.redshift.amazonaws.com" +Port: 5439 +Database: "prod" +Import Query: "select id, name, email, phone from users;" +Number of Splits to Generate: 1 +Username: "sa" +Password: "Test11" +``` + +Data Types Mapping +------------------ + +Mapping of Redshift types to CDAP schema: + +| Redshift Data Type | CDAP Schema Data Type | Comment | +|-----------------------------------------------------|-----------------------|----------------------------------------| +| bigint | long | | +| boolean | boolean | | +| character | string | | +| character varying | string | | +| double precision | double | | +| integer | int | | +| numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | +| real | float | | +| smallint | int | | +| smallserial | int | | +| text | string | | +| date | date | | +| time [ (p) ] [ without time zone ] | time | | +| time [ (p) ] with time zone | string | | +| timestamp [ (p) ] [ without time zone ] | timestamp | | +| timestamp [ (p) ] with time zone | timestamp | stored in UTC format in database | +| xml | string | | +| json | string | | +| super | string | | +| geometry | bytes | | +| hllsketch | string | | diff --git a/amazon-redshift-plugin/docs/Redshift-connector.md b/amazon-redshift-plugin/docs/Redshift-connector.md new file mode 100644 index 000000000..bbc18e9a9 --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-connector.md @@ -0,0 +1,27 @@ +# Amazon RedShift Connection + + +Description +----------- +Use this connection to access data in an Amazon RedShift database using JDBC. + +Properties +---------- +**Name:** Name of the connection. Connection names must be unique in a namespace. + +**Description:** Description of the connection. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/amazon-redshift-plugin/icons/Redshift-batchsource.png b/amazon-redshift-plugin/icons/Redshift-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..11c334799daa890a320e5d4aa934d44a71bc988f GIT binary patch literal 6145 zcmZu#1yqzx+a^||Q;_adIwY4aC6*4Qdx2$PS&1bT0YOSB5m-P{K$=Aw=@M2-atS3C zqy(hl$M=1|@B8aHXP%k4=epy%=FEA{GcgAGT4cnG#5g!OWDm8~jc)poKaG&!=A9Vs z$#c`;`Wk7e;#7<2-`bB9h`*& zAl^4<92|uJxtk%x8E(%W0P*zll?woJ{)Ldc8UHbhaI*gefqMWsE%Xf7)u1qEb}3;= zVNp&+Vs>_R1(=hIoRPZbKkhd(Ag3D~?ky)G;_vS->@OhDAdiwoT# zgnR?N;PwGRUcOv^2l-bVb!T5k7}y&QhI+C8iEHlweF_J1a{fv5@8_R7A>RK^_$QiVSudEWxWws zk@zOlzcxydSU~s-5C?~L^r5W!WrW127Uq3WOr7C^6(52)IPjce(E0uiBta9&8DqCFWCwAOt zVy|_7-r0B-*v%1JTRD@l({?(tjYRg193K8W96r3aP&m^#90cvn@-(`yf90YLk9&&-Wm|%K8QXz1(z{; z>fEL99bj9DIec3ulx8C}OpWjHwr+s8)3J#3tZ5n7udB!qJ@wY|ZEGjDk_*vgq{KP& zS4WOq%C}#=*-Ob9vj=J>pnzCmVZw6)7ZV4mmT5~ z3Y|0z;lfl4UFs~uZ$%59szrESZZHl5rfCE?>Hzm?*D92sX+=6C`$LXVL*3GmmkL89 z;n1|zm$UwoDx=7Wl=raXq;FLpgA%-YjV{^WpQ&+XFlfEJHLY<(s>risp&+3!NS!h1 z6+i*{GC2O4znqEDEl%-g8K>Y;?d?@Z@xx>1% z$H1z?VjN4kp+h6NI+GSmZ=^^#%E~8_=JFM9d~li2vk>><5k+EE!4@{J*I(DJ5xMhx zhd;Y(o*xXLcr2bzl-G6OD#Ae-3~Cs@TMATiC$a@7L$h)%-~A@M(_|j$6MpXtUZ+g$ z>$h78Q)O3vOmJ@4R@Chwq>WVdTncqw76~wwN|&&H&6hVJ(@Oqv>zWqN@++JTS{9=R z`b3Ob>l`XXjPLmsGC50cRSEo%UTc`kmHu@|Jhw_Gm`qJv0UnTCe@@*N$Q@ksVEO*> z&Ub%Dp5F-^?`-c?MFx~8s&uK97nRFSJ1aJmA?|}j4wi$B*Yvk!(#{`s-rloSMuiO@ zT+E6AFC)22KU69A?DW8fVPE+hC6x!rd=BK|Q*4 zky(#(W#M3#nm4o7J|FTU8@eJ}6U?I|6%=|z0XH}K5L4-fmCAJT!KiE6bV7dpoWd;l zn$`n3I{Mzc=wa#4EIggz#E5z+uUylABTUkvq8~ zpwZ{Cp(;lKhIb2<)rm%k$gV&9cDKKJCxG|}6sAtOahZD60&}7=lhT#hWjW;fm41QJ z$FNr*gGN}bz;{!i5>SIl72-2Zw=u#E*ZyokTVUO*gTEdJ9d2g7pL?(rEx3QVm_b>l zgnS+7H&ynAs&ao)6~F(mXh)&oi)Z_@oaQFFCSwi0zL;Ml(=`}V9w_4_(Zz#O#AjtK zU}pT!q}Zy@)2PUz-dDXC$upv~`s zi76W(Uq!`&b8|G=V4_*Czj=O>anN-A?@|MHU$0feFOmL1-+IrHt%~{k+jO;Fj|Jbg zr`p`HJ?Uz{!rtd8+nZ+?sWb>H-MrlFB&5J=Q`$(SF8=lNV-PtsO?A#AJvN92``HKB z8jH;^Z4iU(&zV&lpmk=?{nH4oXut`A>OtQ zRs)gi&-Fa7jjzrspwt|n9CHr}+MQf6tuKs3w7Uoo>*dIK59OEw?HrF4>kVnn^kgo4 z!cTZ)I?YnHsi!~RCgxYLH@>%Gh1}Q=x zxgDFMcv+{9=f={SInK9n)=3POT`G#R;_oqEzc*GvHNNaC?bIIO%e9bEd^8lx_Q|k1 z7xl4zqC@8H3dkqUW0#HjYQV?qU>&41pyXLpD(7==s(SUxwH;sc3U3O9RQ^o5DS~=4 za2PNkIfEL7I_5J;zbTGLXEYil#}RHZU*LPC{+S_4=QYlJ2jQN!)#=HYh7WcnRc`>$ z8Y$^m+jE(^gk5_5n=XQK9fT*d>HIhO><7>VV*rL#9rbF>AnPPA@AWq0K4<6UChR>jFpIrzeC zndogOJnp9@@_rs$DtDrvG+zL%Q+XLxtFQ#1!I}Zh&a`(;CI9OF;vNyyC7s z2&ozVdZyS<%mmA*HxN|-GoPS<`)o<~S1`L#QW-v3fH9!Re2G`0xq(F_k8R@SAZpNr zXJVzenD*$7g_>@Y_LsVEl#FcJU$PCjIDJsg-+R5u($c1bUd&ZiSMJJWA^{W#(t6K- z)Ry*$7vWz{ouU}jnnpq7=-*7d?sy|z_C-jG8U!4oj&q|z(q+XG4ol{b-GNU>8R+y= zk9ujAemw7HCnA&B&>cg$#4EdMmyu=}m7}|u0;O1H3V2Ld=plmMIlyw458z?@J3E@G zUl3u24<&`DuSky>4*A&`!=n+a1oUI+!Wh4Dr7p1z74*J;7W!zyg=@c>MUkoW?} zhMI5p9l)!Vw4XrYw=Jy>ow@E*TKkqU(2)woE1kIn*sMJzx4s^?dfKwmbK>@%kXPQ2a!u zMT=N8Oa{^A{uA5!wO0C7*wRtc(|W21jOmdu4{OkwQMUR)LUhWv4T&6Kwxgc`m7Z5W zdyqSoIp~}y8ce3aKC$qXXJ795R$t&IcG?5LFnO+6U3_e_uHsUj*sszH*!dpV)b10t zgteySs%o*`rZ=U^bl4j8Cd`%2ZY-1O?#{yv^UN&e2cR>@64=yD@NZNMClY%bsXgfW zY9TC*{*XOas~$AIhdF%qhMNFb8cwiWhROSGj$(h@+t@9RF6F!itRWm0L(i44s5TLg!7JTIm?~iLJk|xHbofX@l zwJvWlYrPdXS9SHi`-Y3FGw?`ZA!nN&9(HM!401$AbZiQ}w&Ch8>Hi(0LV*|G;D?B3 z&qplwJsH0|jk~evnXt4VD+BZ>p1X7?UM0{Zy&V6@T8tuS*!&sTbS+q2++gIyopb(t zc1+z79YpG`qmH-#-RJ|~+J#nG)yuQn^*3gZqw1P4*=qq69R>&VlT!Y9^0sG<;~!eo zNB*dwPs>ucZ^OAhYC_u79;zC)**+U3Y8`&<#!mnQ0FUF<^Z7BC2*hG{Jl~Eoes!^b zZc7`*N89^&S4yxONs~P8Xhy_&p!w8|Fdqb`K6WW`~r7jQra#r2D4OPB~dqSZ`YOj~QPcGIHtu`o?edGQP zV|52XU5ULqi`f`Q)Rj$}EHA7Fk|I)T5EG=b zErUCoO}Gn>8PRchMH13XOY`ejaiMdpxAdd@&MmD-I0UiQJMp&p zraOna74A_wzGesGRd-LCDO6>=NZI?k2Yyr{FiPIfomyy-s_UHxRt32u#Cz)0i61Cm z4;JPZrJNn->S@0dBj(Z0<9U(BOlidzpR#2BfB{#Q&KfNi8E&SDPU6tGt^ZD!oV>bo z1#fwBuk_Y}Xv%S3aUGAm=ST>OV7plz#%`ZZ=WU*bL&m90d!n<>hl5qC>o1cM1~}=F zRM^X@CysTNOTIc_gUtnp*spzaY^n&$AdpE|d}KFS4GPn?h#vQ2NFr>lf+MxvDpQGT z+b709ELbS_0S@t^?~b9~zMx8?4M)Xf3>qf!T9>X8<4-I;cjP-9E_DK5g{^j82mbWKM6OK?h#4|H4C#JD2Vq?em3J)QM<`Az z?~7&1ivnMFbO@8(7S5BDHPv!V)U&ZxW#kv%H|7}zYBU{#}?_K zdjP6{z#*h-n>+0~1$%I?HMfLZ?V6iSjd+0fb1)A5ftqA^%cm-fd%{*uJ(2#R*e%`u zhEHi%r`QmAMc$Q3bMcIhU)90RX5*WWGbhT0-Qk^OPH95mMklt5z1KP??HwXMdAq4S`V9|KEQV7*YnyqMgq11WAZSv;KrnIEV3e!svBM{JhwqwuaT$n~SV_o1xDOO>OF zM|$N$qTxL`9L+)2dq092FOzE}%g@B^WY+!d1czz{NOT!r`Ud*(0ZdOLYJZtiymV-$ zS+7WscHpWw;HynxTkd3o36F_bC*S49UUtoMUB}y2HnTlA?H(_qbEy+~?B~-^FVhgQ zzjb|j_&M9vjnMkX%%gS!>4Q!E1Es6n^gh`o`Qb+7HF#&~RgU;?hen-K`5@{cyov!5 z%aTpXlbl{@^Y?3m*&B1-N)riZs~xz)UFwTGTBk|&CWPyDx10d*R5IKVzpMUGScx?D zfC%U-(pNirZ?xhov{5Z$*Dg2xwpozkv+K<|jl6dY&%$A8Grf|@t4|9$!wp$O$bxCk zXoH)$4L(wpJWNVrh(5W~toGx;l_AsNL47fO$AcYD?95Rx>NX+}&JgUi*K(c^1MLgK z%D1Ub;fzT}q-h*64kx*ORbt9dWxY&fR9&_uJG zXS@l?h$d;j8U1 zWh7h*i7a)DGv|9;jJeIv#}#3$PhOdL=RTpKP>@d;ytElu<&t^flOKc*w~AC%+TNGH z@MnUHw0sMYV_?_dAZx+(iO(SMW$WZle5|YF=W?w__?`4UJtt_cm0vo2_MwH~)rc?08laQF?douUlsQ((k z{k?KQ%X}{HbpV+1yzRLXvq>H3IGEg(TDfa$2v{&BW|qsOSvBHXFhN2GqQAmv6ia5U zx;qn(bMp*;=#^EMa%#U2OpW{mTbRW~g*p*xx|9LPTei40kVR}$2@ + + + + database-plugins-parent + io.cdap.plugin + 1.8.8-SNAPSHOT + + + Amazon Redshift plugin + amazon-redshift-plugin + 4.0.0 + + + 2.1.0.18 + + + + + redshift + http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release + + + + + + io.cdap.cdap + cdap-etl-api + + + io.cdap.plugin + database-commons + ${project.version} + + + io.cdap.plugin + hydrator-common + + + com.google.guava + guava + + + + + com.amazon.redshift + redshift-jdbc42 + ${redshift-jdbc.version} + test + + + io.cdap.plugin + database-commons + ${project.version} + test-jar + test + + + io.cdap.cdap + hydrator-test + + + io.cdap.cdap + cdap-data-pipeline2_2.11 + + + junit + junit + + + io.cdap.cdap + cdap-api + provided + + + org.jetbrains + annotations + RELEASE + compile + + + + + + io.cdap + cdap-maven-plugin + + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.amazon.redshift.*; + io.cdap.plugin.db.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + + diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java new file mode 100644 index 000000000..f020dba6d --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java @@ -0,0 +1,108 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Category; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Amazon Redshift Database Connector that connects to Amazon Redshift database via JDBC. + */ +@Plugin(type = Connector.PLUGIN_TYPE) +@Name(RedshiftConnector.NAME) +@Description("Connection to access data in Amazon Redshift using JDBC.") +@Category("Database") +public class RedshiftConnector extends AbstractDBSpecificConnector { + public static final String NAME = RedshiftConstants.PLUGIN_NAME; + private final RedshiftConnectorConfig config; + + public RedshiftConnector(RedshiftConnectorConfig config) { + super(config); + this.config = config; + } + + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + + @Override + public boolean supportSchema() { + return true; + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable longWritable, RedshiftDBRecord redshiftDBRecord) { + return redshiftDBRecord.getRecord(); + } + + protected SchemaReader getSchemaReader(String sessionID) { + return new RedshiftSchemaReader(sessionID); + } + + protected String getTableName(String database, String schema, String table) { + return String.format("\"%s\".\"%s\"", schema, table); + } + + @Override + protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, + ConnectorSpec.Builder builder) { + Map sourceProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + builder + .addRelatedPlugin(new PluginSpec(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, sourceProperties)); + + String schema = path.getSchema(); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.FETCH_SIZE, + RedshiftSource.RedshiftSourceConfig.DEFAULT_FETCH_SIZE); + String table = path.getTable(); + if (table == null) { + return; + } + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java new file mode 100644 index 000000000..f05f26d10 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; + +import javax.annotation.Nullable; + +/** + * Configuration for Redshift connector + */ +public class RedshiftConnectorConfig extends AbstractDBConnectorConfig { + + @Name(ConnectionConfig.HOST) + @Description( + "The endpoint of the Amazon Redshift cluster.") + @Macro + private String host; + + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + + @Name(ConnectionConfig.DATABASE) + @Description("Database name to connect to") + @Macro + private String database; + + public RedshiftConnectorConfig(String username, String password, String jdbcPluginName, + String connectionArguments, String host, + String database, @Nullable Integer port) { + this.user = username; + this.password = password; + this.jdbcPluginName = jdbcPluginName; + this.connectionArguments = connectionArguments; + this.host = host; + this.database = database; + this.port = port; + } + + public String getDatabase() { + return database; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port == null ? 5439 : port; + } + + @Override + public String getConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + host, + getPort(), + database); + } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java new file mode 100644 index 000000000..081052fb1 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +/** Amazon Redshift constants. */ +public final class RedshiftConstants { + + private RedshiftConstants() { + } + + public static final String PLUGIN_NAME = "Redshift"; + public static final String REDSHIFT_CONNECTION_STRING_FORMAT = "jdbc:redshift://%s:%s/%s"; +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java new file mode 100644 index 000000000..f6935bc29 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java @@ -0,0 +1,130 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.util.DBUtils; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +/** + * Writable class for Redshift Source + */ +public class RedshiftDBRecord extends DBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public RedshiftDBRecord() { + } + + @Override + protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, + int columnIndex, int sqlType, int sqlPrecision, int sqlScale) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); + String columnTypeName = metadata.getColumnTypeName(columnIndex); + if (isUseSchema(metadata, columnIndex)) { + setFieldAccordingToSchema(resultSet, recordBuilder, field, columnIndex); + return; + } + + // HandleTimestamp + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamp")) { + Timestamp timestamp = resultSet.getTimestamp(columnIndex, DBUtils.PURE_GREGORIAN_CALENDAR); + if (timestamp != null) { + ZonedDateTime zonedDateTime = OffsetDateTime.of(timestamp.toLocalDateTime(), OffsetDateTime.now().getOffset()) + .atZoneSameInstant(ZoneId.of("UTC")); + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + setZonedDateTimeBasedOnOutputSchema(recordBuilder, nonNullableSchema.getLogicalType(), + field.getName(), zonedDateTime); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleTimestampTZ + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamptz")) { + OffsetDateTime timestamp = resultSet.getObject(columnIndex, OffsetDateTime.class); + if (timestamp != null) { + recordBuilder.setTimestamp(field.getName(), timestamp.atZoneSameInstant(ZoneId.of("UTC"))); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleNumeric + int columnType = metadata.getColumnType(columnIndex); + if (columnType == Types.NUMERIC) { + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + int precision = metadata.getPrecision(columnIndex); + if (precision == 0 && Schema.Type.STRING.equals(nonNullableSchema.getType())) { + // When output schema is set to String for precision less numbers + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } else if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) { + BigDecimal orgValue = resultSet.getBigDecimal(columnIndex); + if (orgValue != null) { + BigDecimal decimalValue = new BigDecimal(orgValue.toPlainString()) + .setScale(nonNullableSchema.getScale(), RoundingMode.HALF_EVEN); + recordBuilder.setDecimal(field.getName(), decimalValue); + } + } + return; + } + setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); + } + + private void setZonedDateTimeBasedOnOutputSchema(StructuredRecord.Builder recordBuilder, + Schema.LogicalType logicalType, + String fieldName, + ZonedDateTime zonedDateTime) { + if (Schema.LogicalType.DATETIME.equals(logicalType)) { + recordBuilder.setDateTime(fieldName, zonedDateTime.toLocalDateTime()); + } else if (Schema.LogicalType.TIMESTAMP_MICROS.equals(logicalType)) { + recordBuilder.setTimestamp(fieldName, zonedDateTime); + } + } + + private static boolean isUseSchema(ResultSetMetaData metadata, int columnIndex) throws SQLException { + String columnTypeName = metadata.getColumnTypeName(columnIndex); + // If the column Type Name is present in the String mapped Redshift types then return true. + return RedshiftSchemaReader.STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(columnTypeName) + || RedshiftSchemaReader.STRING_MAPPED_REDSHIFT_TYPES.contains(metadata.getColumnType(columnIndex)); + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java new file mode 100644 index 000000000..5df35940e --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Set; + +/** + * Redshift Schema Reader class + */ +public class RedshiftSchemaReader extends CommonSchemaReader { + + private static final Logger LOG = LoggerFactory.getLogger(RedshiftSchemaReader.class); + + public static final Set STRING_MAPPED_REDSHIFT_TYPES = ImmutableSet.of( + Types.OTHER, Types.ARRAY, Types.SQLXML + ); + + public static final Set STRING_MAPPED_REDSHIFT_TYPES_NAMES = ImmutableSet.of( + "timetz", "money" + ); + + private final String sessionID; + + public RedshiftSchemaReader() { + this(null); + } + + public RedshiftSchemaReader(String sessionID) { + super(); + this.sessionID = sessionID; + } + + @Override + public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLException { + String typeName = metadata.getColumnTypeName(index); + int columnType = metadata.getColumnType(index); + + if (STRING_MAPPED_REDSHIFT_TYPES.contains(columnType) || STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(typeName)) { + return Schema.of(Schema.Type.STRING); + } + if (typeName.equalsIgnoreCase("INT")) { + return Schema.of(Schema.Type.INT); + } + if (typeName.equalsIgnoreCase("BIGINT")) { + return Schema.of(Schema.Type.LONG); + } + + // If it is a numeric type without precision then use the Schema of String to avoid any precision loss + if (Types.NUMERIC == columnType) { + int precision = metadata.getPrecision(index); + if (precision == 0) { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index))); + return Schema.of(Schema.Type.STRING); + } + } + + if (typeName.equalsIgnoreCase("timestamp")) { + return Schema.of(Schema.LogicalType.DATETIME); + } + + return super.getSchema(metadata, index); + } + + @Override + public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { + if (sessionID == null) { + return false; + } + return metadata.getColumnName(index).equals("c_" + sessionID) || + metadata.getColumnName(index).equals("sqn_" + sessionID); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + // Setting up schema as nullable as cdata driver doesn't provide proper information about isNullable. + columnSchema = Schema.nullableOf(columnSchema); + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java new file mode 100644 index 000000000..4948a16cf --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java @@ -0,0 +1,119 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Metadata; +import io.cdap.cdap.api.annotation.MetadataProperty; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.batch.config.AbstractDBSpecificSourceConfig; +import io.cdap.plugin.db.batch.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nullable; + +/** Batch source to read from an Amazon Redshift database. */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(RedshiftConstants.PLUGIN_NAME) +@Description( + "Reads from a Amazon Redshift database table(s) using a configurable SQL query." + + " Outputs one record for each row returned by the query.") +@Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = RedshiftConnector.NAME)}) +public class RedshiftSource + extends AbstractDBSource { + + private final RedshiftSourceConfig redshiftSourceConfig; + + public RedshiftSource(RedshiftSourceConfig redshiftSourceConfig) { + super(redshiftSourceConfig); + this.redshiftSourceConfig = redshiftSourceConfig; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + protected String createConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + redshiftSourceConfig.connection.getHost(), + redshiftSourceConfig.connection.getPort(), + redshiftSourceConfig.connection.getDatabase()); + } + + /** Redshift source config. */ + public static class RedshiftSourceConfig extends AbstractDBSpecificSourceConfig { + + @Name(ConfigUtil.NAME_USE_CONNECTION) + @Nullable + @Description("Whether to use an existing connection.") + private Boolean useConnection; + + @Name(ConfigUtil.NAME_CONNECTION) + @Macro + @Nullable + @Description("The existing connection to use.") + private RedshiftConnectorConfig connection; + + @Override + public Map getDBSpecificArguments() { + return Collections.emptyMap(); + } + + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + + @Override + protected RedshiftConnectorConfig getConnection() { + return connection; + } + + @Override + public void validate(FailureCollector collector) { + ConfigUtil.validateConnection(this, useConnection, connection, collector); + super.validate(collector); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java new file mode 100644 index 000000000..b29503fca --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java @@ -0,0 +1,218 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.collect.Sets; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.batch.DatabasePluginTestBase; +import io.cdap.plugin.db.batch.sink.ETLDBOutputFormat; +import io.cdap.plugin.db.batch.source.DataDrivenETLDBInputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +/** + * Base test class for Redshift plugins. + */ +public abstract class RedshiftPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftPluginTestBase.class); + protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); + protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + protected static final long CURRENT_TS = System.currentTimeMillis(); + + protected static final String JDBC_DRIVER_NAME = "redshift"; + protected static final Map BASE_PROPS = new HashMap<>(); + + protected static String connectionUrl; + protected static int year; + protected static final int PRECISION = 10; + protected static final int SCALE = 6; + private static int startCount; + + @BeforeClass + public static void setupTest() throws Exception { + if (startCount++ > 0) { + return; + } + + getProperties(); + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(CURRENT_TS)); + year = calendar.get(Calendar.YEAR); + + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); + + addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + RedshiftSource.class, DBRecord.class, + ETLDBOutputFormat.class, DataDrivenETLDBInputFormat.class, DBRecord.class); + + // add mysql 3rd party plugin + PluginClass mysqlDriver = new PluginClass(ConnectionConfig.JDBC_PLUGIN_TYPE, JDBC_DRIVER_NAME, + "redshift driver class", Driver.class.getName(), + null, Collections.emptyMap()); + addPluginArtifact(NamespaceId.DEFAULT.artifact("redshift-jdbc-connector", "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + Sets.newHashSet(mysqlDriver), Driver.class); + + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + + connectionUrl = "jdbc:redshift://" + BASE_PROPS.get(ConnectionConfig.HOST) + ":" + + BASE_PROPS.get(ConnectionConfig.PORT) + "/" + BASE_PROPS.get(ConnectionConfig.DATABASE); + Connection conn = createConnection(); + createTestTables(conn); + prepareTestData(conn); + } + + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("redshift.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("redshift.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("redshift.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("redshift.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("redshift.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + + protected static void createTestTables(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"dbActionTest\" (x int, day varchar(10))"); + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"postActionTest\" (x int, day varchar(10))"); + + stmt.execute("CREATE TABLE my_table" + + "(" + + "\"ID\" INT NOT NULL," + + "\"NAME\" VARCHAR(40) NOT NULL," + + "\"SCORE\" REAL," + + "\"GRADUATED\" BOOLEAN," + + "\"NOT_IMPORTED\" VARCHAR(30)," + + "\"SMALLINT_COL\" SMALLINT," + + "\"BIG\" BIGINT," + + "\"NUMERIC_COL\" NUMERIC(" + PRECISION + "," + SCALE + ")," + + "\"DECIMAL_COL\" DECIMAL(" + PRECISION + "," + SCALE + ")," + + "\"DOUBLE_PREC_COL\" DOUBLE PRECISION," + + "\"DATE_COL\" DATE," + + "\"TIME_COL\" TIME," + + "\"TIMESTAMP_COL\" TIMESTAMP(3)," + + "\"TEXT_COL\" TEXT," + + "\"CHAR_COL\" CHAR(100)," + + "\"BYTEA_COL\" BYTEA" + + ")"); + stmt.execute("CREATE TABLE \"MY_DEST_TABLE\" AS " + + "SELECT * FROM my_table"); + stmt.execute("CREATE TABLE your_table AS " + + "SELECT * FROM my_table"); + } + } + + protected static void prepareTestData(Connection conn) throws SQLException { + try ( + Statement stmt = conn.createStatement(); + PreparedStatement pStmt1 = + conn.prepareStatement("INSERT INTO my_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)"); + PreparedStatement pStmt2 = + conn.prepareStatement("INSERT INTO your_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)")) { + + stmt.execute("insert into \"dbActionTest\" values (1, '1970-01-01')"); + stmt.execute("insert into \"postActionTest\" values (1, '1970-01-01')"); + + populateData(pStmt1, pStmt2); + } + } + + private static void populateData(PreparedStatement... stmts) throws SQLException { + // insert the same data into both tables: my_table and your_table + for (PreparedStatement pStmt : stmts) { + for (int i = 1; i <= 5; i++) { + String name = "user" + i; + pStmt.setInt(1, i); + pStmt.setString(2, name); + pStmt.setDouble(3, 123.45 + i); + pStmt.setBoolean(4, (i % 2 == 0)); + pStmt.setString(5, "random" + i); + pStmt.setShort(6, (short) i); + pStmt.setLong(7, (long) i); + pStmt.setBigDecimal(8, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setBigDecimal(9, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setDouble(10, 123.45 + i); + pStmt.setDate(11, new Date(CURRENT_TS)); + pStmt.setTime(12, new Time(CURRENT_TS)); + pStmt.setTimestamp(13, new Timestamp(CURRENT_TS)); + pStmt.setString(14, name); + pStmt.setString(15, "char" + i); + pStmt.setBytes(16, name.getBytes(Charsets.UTF_8)); + pStmt.executeUpdate(); + } + } + } + + public static Connection createConnection() { + try { + Class.forName(Driver.class.getCanonicalName()); + return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER), + BASE_PROPS.get(ConnectionConfig.PASSWORD)); + } catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @AfterClass + public static void tearDownDB() { + try (Connection conn = createConnection(); + Statement stmt = conn.createStatement()) { + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java new file mode 100644 index 000000000..95ad0938b --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.common.test.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * This is a test suite that runs all the tests for Redshift plugins. + */ +@RunWith(TestSuite.class) +@Suite.SuiteClasses({ + RedshiftSourceTestRun.class, +}) +public class RedshiftPluginTestSuite extends RedshiftPluginTestBase { +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java new file mode 100644 index 000000000..fb671e761 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java @@ -0,0 +1,332 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.common.Bytes; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.batch.source.AbstractDBSource; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test for Redshift source plugin. + */ +public class RedshiftSourceTestRun extends RedshiftPluginTestBase { + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBMacroSupport() throws Exception { + String importQuery = "SELECT * FROM my_table WHERE \"DATE_COL\" <= '${logicalStartTime(yyyy-MM-dd,1d)}' " + + "AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(ID),MAX(ID) from my_table"; + String splitBy = "ID"; + + ImmutableMap sourceProps = ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBTestSource").build(); + + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + sourceProps + ); + + ETLPlugin sinkConfig = MockSink.getPlugin("macroOutputTable"); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBMacro"); + runETLOnce(appManager, ImmutableMap.of("logical.start.time", String.valueOf(CURRENT_TS))); + + DataSetManager outputManager = getDataset("macroOutputTable"); + Assert.assertTrue(MockSink.readOutput(outputManager).isEmpty()); + } + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBSource() throws Exception { + String importQuery = "SELECT \"ID\", \"NAME\", \"SCORE\", \"GRADUATED\", \"SMALLINT_COL\", \"BIG\", " + + "\"NUMERIC_COL\", \"CHAR_COL\", \"DECIMAL_COL\", \"BYTEA_COL\", \"DATE_COL\", \"TIME_COL\", \"TIMESTAMP_COL\", " + + "\"TEXT_COL\", \"DOUBLE_PREC_COL\" FROM my_table " + + "WHERE \"ID\" < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from my_table"; + String splitBy = "ID"; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBSourceTest") + .build(), + null + ); + + String outputDatasetName = "output-dbsourcetest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSource"); + runETLOnce(appManager); + + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals("user1", row1.get("TEXT_COL")); + Assert.assertEquals("user2", row2.get("TEXT_COL")); + Assert.assertEquals("char1", ((String) row1.get("CHAR_COL")).trim()); + Assert.assertEquals("char2", ((String) row2.get("CHAR_COL")).trim()); + Assert.assertEquals(124.45f, ((Float) row1.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(125.45f, ((Float) row2.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(false, row1.get("GRADUATED")); + Assert.assertEquals(true, row2.get("GRADUATED")); + Assert.assertNull(row1.get("NOT_IMPORTED")); + Assert.assertNull(row2.get("NOT_IMPORTED")); + + Assert.assertEquals(1, (int) row1.get("SMALLINT_COL")); + Assert.assertEquals(2, (int) row2.get("SMALLINT_COL")); + Assert.assertEquals(1, (long) row1.get("BIG")); + Assert.assertEquals(2, (long) row2.get("BIG")); + + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("125.45", new MathContext(PRECISION)).setScale(SCALE), + row2.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("DECIMAL_COL")); + + Assert.assertEquals(124.45, (double) row1.get("DOUBLE_PREC_COL"), 0.000001); + Assert.assertEquals(125.45, (double) row2.get("DOUBLE_PREC_COL"), 0.000001); + // Verify time columns + java.util.Date date = new java.util.Date(CURRENT_TS); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + LocalDate expectedDate = Date.valueOf(sdf.format(date)).toLocalDate(); + sdf = new SimpleDateFormat("H:mm:ss"); + LocalTime expectedTime = Time.valueOf(sdf.format(date)).toLocalTime(); + ZonedDateTime expectedTs = date.toInstant().atZone(ZoneId.ofOffset("UTC", ZoneOffset.UTC)); + Assert.assertEquals(expectedDate, row1.getDate("DATE_COL")); + Assert.assertEquals(expectedTime, row1.getTime("TIME_COL")); + Assert.assertEquals(expectedTs, row1.getTimestamp("TIMESTAMP_COL", ZoneId.ofOffset("UTC", ZoneOffset.UTC))); + + // verify binary columns + Assert.assertEquals("user1", Bytes.toString(((ByteBuffer) row1.get("BYTEA_COL")).array(), 0, 5)); + Assert.assertEquals("user2", Bytes.toString(((ByteBuffer) row2.get("BYTEA_COL")).array(), 0, 5)); + } + + @Test + public void testDbSourceMultipleTables() throws Exception { + String importQuery = "SELECT \"my_table\".\"ID\", \"your_table\".\"NAME\" FROM \"my_table\", \"your_table\"" + + "WHERE \"my_table\".\"ID\" < 3 and \"my_table\".\"ID\" = \"your_table\".\"ID\" and $CONDITIONS"; + String boundingQuery = "SELECT MIN(MIN(\"my_table\".\"ID\"), MIN(\"your_table\".\"ID\")), " + + "MAX(MAX(\"my_table\".\"ID\"), MAX(\"your_table\".\"ID\"))"; + String splitBy = "\"my_table\".\"ID\""; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBMultipleTest") + .build(), + null + ); + + String outputDatasetName = "output-multitabletest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSourceWithMultipleTables"); + runETLOnce(appManager); + + // records should be written + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals(1, row1.get("ID").intValue()); + Assert.assertEquals(2, row2.get("ID").intValue()); + } + + @Test + public void testUserNamePasswordCombinations() throws Exception { + String importQuery = "SELECT * FROM \"my_table\" WHERE $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from \"my_table\""; + String splitBy = "\"ID\""; + + ETLPlugin sinkConfig = MockSink.getPlugin("outputTable"); + + Map baseSourceProps = ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, BASE_PROPS.get(ConnectionConfig.DATABASE)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "UserPassDBTest") + .build(); + + ApplicationId appId = NamespaceId.DEFAULT.app("dbTest"); + + // null user name, null password. Should succeed. + // as source + ETLPlugin dbConfig = new ETLPlugin(RedshiftConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + baseSourceProps, null); + ETLStage table = new ETLStage("uniqueTableSink", sinkConfig); + ETLStage database = new ETLStage("databaseSource", dbConfig); + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + + // null user name, non-null password. Should fail. + // as source + Map noUser = new HashMap<>(baseSourceProps); + noUser.put(DBConfig.PASSWORD, "password"); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, noUser, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + assertDeploymentFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT, + "Deploying DB Source with null username but non-null password should have failed."); + + // non-null username, non-null, but empty password. Should succeed. + // as source + Map emptyPassword = new HashMap<>(baseSourceProps); + emptyPassword.put(DBConfig.USER, "root"); + emptyPassword.put(DBConfig.PASSWORD, ""); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, emptyPassword, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + } + + @Test + public void testNonExistentDBTable() throws Exception { + // source + String importQuery = "SELECT \"ID\", \"NAME\" FROM \"dummy\" WHERE ID < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") FROM \"dummy\""; + String splitBy = "\"ID\""; + ETLPlugin sinkConfig = MockSink.getPlugin("table"); + ETLPlugin sourceBadNameConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBNonExistentTest") + .build(), + null); + ETLStage sink = new ETLStage("sink", sinkConfig); + ETLStage sourceBadName = new ETLStage("sourceBadName", sourceBadNameConfig); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadName) + .addStage(sink) + .addConnection(sourceBadName.getName(), sink.getName()) + .build(); + ApplicationId appId = NamespaceId.DEFAULT.app("dbSourceNonExistingTest"); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + + // Bad connection + ETLPlugin sourceBadConnConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, "dumDB") + .put(ConnectionConfig.USER, BASE_PROPS.get(ConnectionConfig.USER)) + .put(ConnectionConfig.PASSWORD, BASE_PROPS.get(ConnectionConfig.PASSWORD)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "RedshiftTest") + .build(), + null); + ETLStage sourceBadConn = new ETLStage("sourceBadConn", sourceBadConnConfig); + etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadConn) + .addStage(sink) + .addConnection(sourceBadConn.getName(), sink.getName()) + .build(); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json new file mode 100644 index 000000000..91e860ee9 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -0,0 +1,240 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Connection", + "properties": [ + { + "widget-type": "toggle", + "label": "Use connection", + "name": "useConnection", + "widget-attributes": { + "on": { + "value": "true", + "label": "YES" + }, + "off": { + "value": "false", + "label": "NO" + }, + "default": "false" + } + }, + { + "widget-type": "connection-select", + "label": "Connection", + "name": "connection", + "widget-attributes": { + "connectionType": "Redshift" + } + }, + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "placeholder": "Redshift endpoint host name." + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + }, + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter" : "=", + "delimiter" : ";" + } + } + ] + }, + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage. Typically, the name of the table/view." + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + }, + { + "widget-type": "connection-browser", + "widget-category": "plugin", + "widget-attributes": { + "connectionType": "Redshift", + "label": "Browse Database" + } + } + ] + }, + { + "label": "SQL Query", + "properties": [ + { + "widget-type": "textarea", + "label": "Import Query", + "name": "importQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "get-schema", + "widget-category": "plugin" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "textarea", + "label": "Bounding Query", + "name": "boundingQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "textbox", + "label": "Split Column", + "name": "splitBy" + }, + { + "widget-type": "textbox", + "label": "Number of Splits", + "name": "numSplits", + "widget-attributes": { + "default": "1" + } + }, + { + "widget-type": "number", + "label": "Fetch Size", + "name": "fetchSize", + "widget-attributes": { + "default": "1000", + "minimum": "0" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "showConnectionProperties ", + "condition": { + "expression": "useConnection == false" + }, + "show": [ + { + "type": "property", + "name": "jdbcPluginName" + }, + { + "type": "property", + "name": "instanceType" + }, + { + "type": "property", + "name": "host" + }, + { + "type": "property", + "name": "port" + }, + { + "type": "property", + "name": "user" + }, + { + "type": "property", + "name": "password" + }, + { + "type": "property", + "name": "database" + }, + { + "type": "property", + "name": "connectionArguments" + } + ] + }, + { + "name": "showConnectionId", + "condition": { + "expression": "useConnection == true" + }, + "show": [ + { + "type": "property", + "name": "connection" + } + ] + }, + ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json new file mode 100644 index 000000000..3a2af8e01 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -0,0 +1,75 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "default": "localhost" + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter": "=", + "delimiter": ";" + } + } + ] + } + ], + "outputs": [] +} diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index fadf56740..1414a3c7e 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a0a260a35..3e17627ad 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c6e95c62d..3c16e5849 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 13ee948ca..012657587 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 4e028139b..7c5bb4b42 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Database Commons diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index f6b7e6514..6bd2524a5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -59,7 +59,7 @@ public final class DBUtils { private static final Logger LOG = LoggerFactory.getLogger(DBUtils.class); - private static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); + public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); // Java by default uses October 15, 1582 as a Gregorian cut over date. // Any timestamp created with time less than this cut over date is treated as Julian date. diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 1353eed5f..177cad153 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 178367c12..86733c1d5 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 51bc38238..f79bbe383 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 44329ad66..d32c78897 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bdc74cde8..ac676851d 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index eb8d8db6d..4c44bfd62 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index c349cb234..ae652d887 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a7e783797..7a7fadc9a 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 36a50e14f..69dd7f18b 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 000a07489..a5dc8f46d 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.8.7 + 1.8.8-SNAPSHOT pom Database Plugins Collection of database plugins @@ -44,6 +44,7 @@ cloudsql-postgresql-plugin teradata-plugin generic-db-argument-setter + amazon-redshift-plugin diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index a1e94ed48..83c15caac 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index f1aa752d9..3cfbbeabf 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 027dd78af..bbbb497f0 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.8.7 + 1.8.8-SNAPSHOT teradata-plugin