diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json index bef887b01b6ff..08c18e2d62e41 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", "name": "Oracle (Alpha)", "dockerRepository": "airbyte/destination-oracle", - "dockerImageTag": "0.1.5", + "dockerImageTag": "0.1.6", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e6df8539207e5..7bb143ad6db79 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -68,7 +68,7 @@ - destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72 name: Oracle (Alpha) dockerRepository: airbyte/destination-oracle - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle - destinationDefinitionId: 9f760101-60ae-462f-9ee6-b7a9dafd454d name: Kafka diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 8d2c2c9428bbd..315029cfba782 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index 35b96111cdd81..84e1c9a0b1885 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -21,7 +21,6 @@ dependencies { implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:oracle-xe:1.15.2' integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java index 628a0161fe059..445475a38194f 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java @@ -59,7 +59,7 @@ public String convertStreamName(String input) { if (!result.isEmpty() && result.charAt(0) == '_') { result = result.substring(1); } - return maxStringLength(result, 30); + return maxStringLength(result, 128); } } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index d7ee62483569c..24f52ee1d0045 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -24,8 +24,10 @@ package io.airbyte.integrations.destination.oracle; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.protocol.models.AirbyteRecordMessage; import java.sql.PreparedStatement; @@ -150,8 +152,9 @@ private static void insertRawRecordsInSingleQuery(String tableName, int i = 1; for (final AirbyteRecordMessage message : records) { // 1-indexed + final JsonNode formattedData = StandardNameTransformer.formatJsonPath(message.getData()); statement.setString(i, uuidSupplier.get().toString()); - statement.setString(i + 1, Jsons.serialize(message.getData())); + statement.setString(i + 1, Jsons.serialize(formattedData)); statement.setTimestamp(i + 2, Timestamp.from(Instant.ofEpochMilli(message.getEmittedAt()))); i += 3; } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java index d523ece96d0d8..b3febcef5a3ec 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java @@ -25,37 +25,34 @@ package io.airbyte.integrations.destination.oracle; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.nio.file.Path; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import org.jooq.JSONFormat; import org.jooq.JSONFormat.RecordFormat; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.OracleContainer; public class OracleIntegrationTest extends DestinationAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(OracleIntegrationTest.class); private static final JSONFormat JSON_FORMAT = new JSONFormat().recordFormat(RecordFormat.OBJECT); - private static OracleContainer db; + private static JsonNode baseConfig; private ExtendedNameTransformer namingResolver = new OracleNameTransformer(); - private JsonNode config; + private static JsonNode config; - @BeforeAll - protected static void init() { - db = new OracleContainer("epiclabs/docker-oracle-xe-11g"); - db.start(); + public JsonNode getStaticConfig() { + return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); } @Override @@ -63,34 +60,11 @@ protected String getImageName() { return "airbyte/destination-oracle:dev"; } - private JsonNode getConfig(OracleContainer db) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("schema", "testSchema") - .put("sid", db.getSid()) - .build()); - } - @Override protected JsonNode getConfig() { return config; } - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("username", db.getUsername()) - .put("password", "wrong password") - .put("schema", "public") - .put("port", db.getFirstMappedPort()) - .put("sid", db.getSid()) - .build()); - } - @Override protected List retrieveRecords(TestDestinationEnv env, String streamName, String namespace, JsonNode streamSchema) throws Exception { return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) @@ -116,6 +90,13 @@ protected List retrieveNormalizedRecords(TestDestinationEnv env, Strin return retrieveRecordsFromTable(tableName, namespace); } + @Override + protected JsonNode getFailCheckConfig() { + final JsonNode invalidConfig = Jsons.clone(config); + ((ObjectNode) invalidConfig).put("password", "wrong password"); + return invalidConfig; + } + @Override protected List resolveIdentifier(String identifier) { final List result = new ArrayList<>(); @@ -130,11 +111,10 @@ protected List resolveIdentifier(String identifier) { } private List retrieveRecordsFromTable(String tableName, String schemaName) throws SQLException { - List result = Databases.createOracleDatabase(db.getUsername(), db.getPassword(), db.getJdbcUrl()) - .query(ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT)) - .stream() - .collect(Collectors.toList())); + List result = getDatabase().query(ctx -> ctx + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT)) + .stream() + .collect(Collectors.toList())); return result .stream() .map(r -> r.formatJSON(JSON_FORMAT)) @@ -142,16 +122,16 @@ private List retrieveRecordsFromTable(String tableName, String schemaN .collect(Collectors.toList()); } - private static Database getDatabase(JsonNode config) { + private static Database getDatabase() { // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the // constructor. at least explicitly handle it, even if the impl doesn't change. return Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), + baseConfig.get("username").asText(), + baseConfig.get("password").asText(), String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), + baseConfig.get("host").asText(), + baseConfig.get("port").asText(), + baseConfig.get("sid").asText()), "oracle.jdbc.driver.OracleDriver", null); } @@ -172,21 +152,18 @@ private List getAllTables(Database db) { @Override protected void setup(TestDestinationEnv testEnv) throws SQLException { - config = getConfig(db); - - final Database database = getDatabase(config); - database.query(ctx -> { - ctx.execute("alter database default tablespace users"); - return null; - }); + // config = getConfig(db); + baseConfig = getStaticConfig(); + config = Jsons.clone(baseConfig); + final Database database = getDatabase(); allTables = getAllTables(database); } @Override protected void tearDown(TestDestinationEnv testEnv) { - config = getConfig(db); + config = getStaticConfig(); - final Database database = getDatabase(config); + final Database database = getDatabase(); var tables = getAllTables(database); tables.removeAll(allTables); try { @@ -201,10 +178,4 @@ protected void tearDown(TestDestinationEnv testEnv) { } } - @AfterAll - static void cleanUp() { - db.stop(); - db.close(); - } - } diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 30c1166521bde..1e9e0a37da9fd 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -62,6 +62,7 @@ You should now have all the requirements needed to configure Oracle as a destina ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2021-08-23 | [#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization | | 0.1.5 | 2021-08-10 | [#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role | | 0.1.4 | 2021-07-30 | [#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.1.3 | 2021-07-21 | [#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer |