🐙 octavia-cli: use model serialization (#12133)

airbyte-api/src/main/openapi/config.yaml

@@ -3141,6 +3141,7 @@ components:
         sourceCatalogId:
           type: string
           format: uuid
+          nullable: true
     WebBackendConnectionCreate:
       type: object
       required:
@@ -3221,6 +3222,7 @@ components:
         sourceCatalogId:
           type: string
           format: uuid
+          nullable: true
     WebBackendConnectionUpdate:
       type: object
       required:
@@ -3307,6 +3309,7 @@ components:
         sourceCatalogId:
           type: string
           format: uuid
+          nullable: true
     ConnectionSearch:
       type: object
       properties:
@@ -3597,6 +3600,7 @@ components:
         sourceDefinedCursor:
           description: If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.
           type: boolean
+          nullable: true
         defaultCursorField:
           description: Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.
           type: array
@@ -3611,6 +3615,7 @@ components:
               type: string
         namespace:
           type: string
+          nullable: true
           description: Optional Source-defined namespace. Airbyte streams from the same sources should have the same namespace. Currently only used by JDBC destinations to determine what schema to write to.
     StreamJsonSchema:
       type: object

airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java

@@ -455,6 +455,7 @@ public void testDiscoverSourceSchema() throws ApiException {
         .name(STREAM_NAME)
         .namespace("public")
         .jsonSchema(jsonSchema)
+        .sourceDefinedCursor(null)
         .defaultCursorField(Collections.emptyList())
         .sourceDefinedPrimaryKey(Collections.emptyList())
         .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL));

octavia-cli/README.md

@@ -362,8 +362,9 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment
 
 ## Changelog
 
-| Version | Date       | Description       | PR                                                       |
-|---------|------------|-------------------|----------------------------------------------------------|
-| 0.36.2  | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896)|
-| 0.35.68 | 2022-04-12 | Add telemetry     | [#11896](https://github.com/airbytehq/airbyte/issues/11896)|
-| 0.35.61 | 2022-04-07 | Alpha release     | [EPIC](https://github.com/airbytehq/airbyte/issues/10704)|
+| Version | Date       | Description                         | PR                                                       |
+|---------|------------|-------------------------------------|----------------------------------------------------------|
+| 0.36.11 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133)|
+| 0.36.2  | 2022-04-15 | Improve telemetry                   | [#12072](https://github.com/airbytehq/airbyte/issues/11896)|
+| 0.35.68 | 2022-04-12 | Add telemetry                       | [#11896](https://github.com/airbytehq/airbyte/issues/11896)|
+| 0.35.61 | 2022-04-07 | Alpha release                       | [EPIC](https://github.com/airbytehq/airbyte/issues/10704)|

octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml

@@ -1,38 +1,35 @@
-# Configuration for connection poke_to_g
+# Configuration for connection poke_to_pg
 definition_type: connection
-resource_name: poke_to_g
+resource_name: poke_to_pg
 source_id: TO_UPDATE_FROM_TEST
 destination_id: TO_UPDATE_FROM_TEST
 
 # EDIT THE CONFIGURATION BELOW!
 configuration:
-  sourceId: TO_UPDATE_FROM_TEST # REQUIRED | string
-  destinationId: TO_UPDATE_FROM_TEST # REQUIRED | string
   status: active # REQUIRED | string | Allowed values: active, inactive, deprecated
-  name: poke_to_g # OPTIONAL | string | Optional name of the connection
-  namespaceDefinition: source # OPTIONAL | string | Allowed values: source, destination, customformat
-  namespaceFormat: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
+  namespace_definition: source # OPTIONAL | string | Allowed values: source, destination, customformat
+  namespace_format: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
   prefix: "" # REQUIRED | Prefix that will be prepended to the name of each stream when it is written to the destination
-  resourceRequirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations)
+  resource_requirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations)
     cpu_limit: "" # OPTIONAL
     cpu_request: "" # OPTIONAL
     memory_limit: "" # OPTIONAL
     memory_request: "" # OPTIONAL
   schedule: # OPTIONAL | object
-    timeUnit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
+    time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
     units: 1 # REQUIRED | integer
-  syncCatalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
+  sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
     streams:
       - config:
-          aliasName: pokemon
-          cursorField: []
-          destinationSyncMode: append
-          primaryKey: []
+          alias_name: pokemon
+          cursor_field: []
+          destination_sync_mode: append
+          primary_key: []
           selected: true
-          syncMode: full_refresh
+          sync_mode: full_refresh
         stream:
-          defaultCursorField: []
-          jsonSchema:
+          default_cursor_field: []
+          json_schema:
             $schema: http://json-schema.org/draft-07/schema#
             properties:
               abilities:
@@ -358,7 +355,7 @@ configuration:
             type: object
           name: pokemon
           namespace: null
-          sourceDefinedCursor: null
-          sourceDefinedPrimaryKey: []
-          supportedSyncModes:
+          source_defined_cursor: null
+          source_defined_primary_key: []
+          supported_sync_modes:
             - full_refresh

octavia-cli/integration_tests/conftest.py

@@ -112,15 +112,13 @@ def updated_connection_configuration_and_path(octavia_test_project_directory, so
         local_configuration = yaml.safe_load(dumb_local_configuration_file)
     local_configuration["source_id"] = source.resource_id
     local_configuration["destination_id"] = destination.resource_id
-    local_configuration["configuration"]["sourceId"] = source.resource_id
-    local_configuration["configuration"]["destinationId"] = destination.resource_id
     with open(edited_path, "w") as updated_configuration_file:
         yaml.dump(local_configuration, updated_configuration_file)
     return local_configuration, edited_path
 
 
 @pytest.fixture(scope="session")
-def connection(api_client, workspace_id, octavia_test_project_directory, source, destination):
+def connection(api_client, workspace_id, octavia_test_project_directory, source, destination, connection_state_path):
     configuration, configuration_path = updated_connection_configuration_and_path(octavia_test_project_directory, source, destination)
     connection = Connection(api_client, workspace_id, configuration, configuration_path)
     yield connection

octavia-cli/integration_tests/test_apply/test_resources.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+
 import pytest
 
 pytestmark = pytest.mark.integration
@@ -10,23 +11,25 @@
 def test_source_lifecycle(source):
     assert not source.was_created
     source.create()
-    source.state = source._get_state_from_file()
+    source.state = source._get_state_from_file(source.configuration_path)
     assert source.was_created
     assert not source.get_diff_with_remote_resource()
-    source.local_configuration["configuration"]["pokemon_name"] = "snorlax"
+    source.raw_configuration["configuration"]["pokemon_name"] = "snorlax"
+    source.configuration = source._deserialize_raw_configuration()
     assert 'changed from "ditto" to "snorlax"' in source.get_diff_with_remote_resource()
     source.update()
     assert not source.get_diff_with_remote_resource()
-    assert source.catalog["streams"][0]["config"]["aliasName"] == "pokemon"
+    assert source.catalog["streams"][0]["config"]["alias_name"] == "pokemon"
 
 
 def test_destination_lifecycle(destination):
     assert not destination.was_created
     destination.create()
-    destination.state = destination._get_state_from_file()
+    destination.state = destination._get_state_from_file(destination.configuration_path)
     assert destination.was_created
     assert not destination.get_diff_with_remote_resource()
-    destination.local_configuration["configuration"]["host"] = "foo"
+    destination.raw_configuration["configuration"]["host"] = "foo"
+    destination.configuration = destination._deserialize_raw_configuration()
     assert 'changed from "localhost" to "foo"' in destination.get_diff_with_remote_resource()
     destination.update()
     assert not destination.get_diff_with_remote_resource()
@@ -37,10 +40,11 @@ def test_connection_lifecycle(source, destination, connection):
     assert destination.was_created
     assert not connection.was_created
     connection.create()
-    connection.state = connection._get_state_from_file()
+    connection.state = connection._get_state_from_file(connection.configuration_path)
     assert connection.was_created
     assert not connection.get_diff_with_remote_resource()
-    connection.local_configuration["configuration"]["status"] = "inactive"
+    connection.raw_configuration["configuration"]["status"] = "inactive"
+    connection.configuration = connection._deserialize_raw_configuration()
     assert 'changed from "active" to "inactive"' in connection.get_diff_with_remote_resource()
     connection.update()
     assert not connection.get_diff_with_remote_resource()

octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected.yaml

@@ -6,22 +6,19 @@ destination_id: my_destination_id
 
 # EDIT THE CONFIGURATION BELOW!
 configuration:
-  sourceId: my_source_id # REQUIRED | string
-  destinationId: my_destination_id # REQUIRED | string
   status: active # REQUIRED | string | Allowed values: active, inactive, deprecated
-  name: my_new_connection # OPTIONAL | string | Optional name of the connection
-  namespaceDefinition: source # OPTIONAL | string | Allowed values: source, destination, customformat
-  namespaceFormat: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
+  namespace_definition: source # OPTIONAL | string | Allowed values: source, destination, customformat
+  namespace_format: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
   prefix: "" # REQUIRED | Prefix that will be prepended to the name of each stream when it is written to the destination
-  resourceRequirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations)
+  resource_requirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations)
     cpu_limit: "" # OPTIONAL
     cpu_request: "" # OPTIONAL
     memory_limit: "" # OPTIONAL
     memory_request: "" # OPTIONAL
   schedule: # OPTIONAL | object
-    timeUnit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
+    time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
     units: 1 # REQUIRED | integer
-  syncCatalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
+  sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
     streams:
       - config:
           aliasName: aliasMock