forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
loosen definition of object specs in SecretsHelper (airbytehq#6654)
* address missing secretshelper case * fix test * migration changes * fix build * format * switch secrets migration to platform-only build
- Loading branch information
Showing
13 changed files
with
392 additions
and
205 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
...t/java/io/airbyte/config/persistence/split_secrets/test_cases/PostgresSshKeyTestCase.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.config.persistence.split_secrets.test_cases; | ||
|
|
||
| import io.airbyte.config.persistence.split_secrets.SecretCoordinate; | ||
| import io.airbyte.config.persistence.split_secrets.SecretPersistence; | ||
| import io.airbyte.config.persistence.split_secrets.SecretsHelpersTest; | ||
| import io.airbyte.config.persistence.split_secrets.SecretsTestCase; | ||
| import java.util.Map; | ||
| import java.util.function.Consumer; | ||
|
|
||
| public class PostgresSshKeyTestCase implements SecretsTestCase { | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "postgres_ssh_key"; | ||
| } | ||
|
|
||
| @Override | ||
| public Map<SecretCoordinate, String> getFirstSecretMap() { | ||
| return Map.of( | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(0), 1), "hunter1", | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(1), 1), "hunter2"); | ||
| } | ||
|
|
||
| @Override | ||
| public Map<SecretCoordinate, String> getSecondSecretMap() { | ||
| return Map.of( | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(0), 2), "hunter3", | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(1), 2), "hunter4"); | ||
| } | ||
|
|
||
| @Override | ||
| public Consumer<SecretPersistence> getPersistenceUpdater() { | ||
| return secretPersistence -> { | ||
| secretPersistence.write( | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(0), 1), | ||
| "hunter1"); | ||
| secretPersistence.write( | ||
| new SecretCoordinate("airbyte_workspace_" + SecretsHelpersTest.WORKSPACE_ID + "_secret_" + SecretsHelpersTest.UUIDS.get(1), 1), | ||
| "hunter2"); | ||
| }; | ||
| } | ||
|
|
||
| } |
18 changes: 18 additions & 0 deletions
18
airbyte-config/persistence/src/test/resources/postgres_ssh_key/full_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| { | ||
| "ssl": false, | ||
| "host": "host", | ||
| "port": 5432, | ||
| "database": "db", | ||
| "password": "hunter1", | ||
| "username": "user", | ||
| "tunnel_method": { | ||
| "ssh_key": "hunter2", | ||
| "tunnel_host": "host", | ||
| "tunnel_port": 22, | ||
| "tunnel_user": "user", | ||
| "tunnel_method": "SSH_KEY_AUTH" | ||
| }, | ||
| "replication_method": { | ||
| "method": "Standard" | ||
| } | ||
| } |
22 changes: 22 additions & 0 deletions
22
airbyte-config/persistence/src/test/resources/postgres_ssh_key/partial_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| { | ||
| "ssl": false, | ||
| "host": "host", | ||
| "port": 5432, | ||
| "database": "db", | ||
| "password": { | ||
| "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22_v1" | ||
| }, | ||
| "username": "user", | ||
| "tunnel_method": { | ||
| "ssh_key": { | ||
| "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_2c2ef2b3-259a-4e73-96d1-f56dacee2e5e_v1" | ||
| }, | ||
| "tunnel_host": "host", | ||
| "tunnel_port": 22, | ||
| "tunnel_user": "user", | ||
| "tunnel_method": "SSH_KEY_AUTH" | ||
| }, | ||
| "replication_method": { | ||
| "method": "Standard" | ||
| } | ||
| } |
220 changes: 220 additions & 0 deletions
220
airbyte-config/persistence/src/test/resources/postgres_ssh_key/spec.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,220 @@ | ||
| { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "title": "Postgres Source Spec", | ||
| "type": "object", | ||
| "required": ["host", "port", "database", "username"], | ||
| "additionalProperties": false, | ||
| "properties": { | ||
| "host": { | ||
| "title": "Host", | ||
| "description": "Hostname of the database.", | ||
| "type": "string", | ||
| "order": 0 | ||
| }, | ||
| "port": { | ||
| "title": "Port", | ||
| "description": "Port of the database.", | ||
| "type": "integer", | ||
| "minimum": 0, | ||
| "maximum": 65536, | ||
| "default": 5432, | ||
| "examples": ["5432"], | ||
| "order": 1 | ||
| }, | ||
| "database": { | ||
| "title": "DB Name", | ||
| "description": "Name of the database.", | ||
| "type": "string", | ||
| "order": 2 | ||
| }, | ||
| "username": { | ||
| "title": "User", | ||
| "description": "Username to use to access the database.", | ||
| "type": "string", | ||
| "order": 3 | ||
| }, | ||
| "password": { | ||
| "title": "Password", | ||
| "description": "Password associated with the username.", | ||
| "type": "string", | ||
| "airbyte_secret": true, | ||
| "order": 4 | ||
| }, | ||
| "ssl": { | ||
| "title": "Connect using SSL", | ||
| "description": "Encrypt client/server communications for increased security.", | ||
| "type": "boolean", | ||
| "default": false, | ||
| "order": 5 | ||
| }, | ||
| "replication_method": { | ||
| "type": "object", | ||
| "title": "Replication Method", | ||
| "description": "Replication method to use for extracting data from the database.", | ||
| "order": 6, | ||
| "oneOf": [ | ||
| { | ||
| "title": "Standard", | ||
| "additionalProperties": false, | ||
| "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", | ||
| "required": ["method"], | ||
| "properties": { | ||
| "method": { | ||
| "type": "string", | ||
| "const": "Standard", | ||
| "enum": ["Standard"], | ||
| "default": "Standard", | ||
| "order": 0 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "title": "Logical Replication (CDC)", | ||
| "additionalProperties": false, | ||
| "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the <a href=\"https://docs.airbyte.io/integrations/sources/postgres\">Postgres Source</a> docs for more information.", | ||
| "required": ["method", "replication_slot", "publication"], | ||
| "properties": { | ||
| "method": { | ||
| "type": "string", | ||
| "const": "CDC", | ||
| "enum": ["CDC"], | ||
| "default": "CDC", | ||
| "order": 0 | ||
| }, | ||
| "plugin": { | ||
| "type": "string", | ||
| "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read <a href=\"https://docs.airbyte.io/integrations/sources/postgres\">Postgres Source</a> docs.", | ||
| "enum": ["pgoutput", "wal2json"], | ||
| "default": "pgoutput", | ||
| "order": 1 | ||
| }, | ||
| "replication_slot": { | ||
| "type": "string", | ||
| "description": "A plug-in logical replication slot.", | ||
| "order": 2 | ||
| }, | ||
| "publication": { | ||
| "type": "string", | ||
| "description": "A Postgres publication used for consuming changes.", | ||
| "order": 3 | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| }, | ||
| "tunnel_method": { | ||
| "type": "object", | ||
| "title": "SSH Tunnel Method", | ||
| "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", | ||
| "oneOf": [ | ||
| { | ||
| "title": "No Tunnel", | ||
| "required": ["tunnel_method"], | ||
| "properties": { | ||
| "tunnel_method": { | ||
| "description": "No ssh tunnel needed to connect to database", | ||
| "type": "string", | ||
| "const": "NO_TUNNEL", | ||
| "order": 0 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "title": "SSH Key Authentication", | ||
| "required": [ | ||
| "tunnel_method", | ||
| "tunnel_host", | ||
| "tunnel_port", | ||
| "tunnel_user", | ||
| "ssh_key" | ||
| ], | ||
| "properties": { | ||
| "tunnel_method": { | ||
| "description": "Connect through a jump server tunnel host using username and ssh key", | ||
| "type": "string", | ||
| "const": "SSH_KEY_AUTH", | ||
| "order": 0 | ||
| }, | ||
| "tunnel_host": { | ||
| "title": "SSH Tunnel Jump Server Host", | ||
| "description": "Hostname of the jump server host that allows inbound ssh tunnel.", | ||
| "type": "string", | ||
| "order": 1 | ||
| }, | ||
| "tunnel_port": { | ||
| "title": "SSH Connection Port", | ||
| "description": "Port on the proxy/jump server that accepts inbound ssh connections.", | ||
| "type": "integer", | ||
| "minimum": 0, | ||
| "maximum": 65536, | ||
| "default": 22, | ||
| "examples": ["22"], | ||
| "order": 2 | ||
| }, | ||
| "tunnel_user": { | ||
| "title": "SSH Login Username", | ||
| "description": "OS-level username for logging into the jump server host.", | ||
| "type": "string", | ||
| "order": 3 | ||
| }, | ||
| "ssh_key": { | ||
| "title": "SSH Private Key", | ||
| "description": "OS-level user account ssh key credentials for logging into the jump server host.", | ||
| "type": "string", | ||
| "airbyte_secret": true, | ||
| "multiline": true, | ||
| "order": 4 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "title": "Password Authentication", | ||
| "required": [ | ||
| "tunnel_method", | ||
| "tunnel_host", | ||
| "tunnel_port", | ||
| "tunnel_user", | ||
| "tunnel_user_password" | ||
| ], | ||
| "properties": { | ||
| "tunnel_method": { | ||
| "description": "Connect through a jump server tunnel host using username and password authentication", | ||
| "type": "string", | ||
| "const": "SSH_PASSWORD_AUTH", | ||
| "order": 0 | ||
| }, | ||
| "tunnel_host": { | ||
| "title": "SSH Tunnel Jump Server Host", | ||
| "description": "Hostname of the jump server host that allows inbound ssh tunnel.", | ||
| "type": "string", | ||
| "order": 1 | ||
| }, | ||
| "tunnel_port": { | ||
| "title": "SSH Connection Port", | ||
| "description": "Port on the proxy/jump server that accepts inbound ssh connections.", | ||
| "type": "integer", | ||
| "minimum": 0, | ||
| "maximum": 65536, | ||
| "default": 22, | ||
| "examples": ["22"], | ||
| "order": 2 | ||
| }, | ||
| "tunnel_user": { | ||
| "title": "SSH Login Username", | ||
| "description": "OS-level username for logging into the jump server host", | ||
| "type": "string", | ||
| "order": 3 | ||
| }, | ||
| "tunnel_user_password": { | ||
| "title": "Password", | ||
| "description": "OS-level password for logging into the jump server host", | ||
| "type": "string", | ||
| "airbyte_secret": true, | ||
| "order": 4 | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| } |
18 changes: 18 additions & 0 deletions
18
airbyte-config/persistence/src/test/resources/postgres_ssh_key/update_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| { | ||
| "ssl": false, | ||
| "host": "host", | ||
| "port": 5432, | ||
| "database": "db", | ||
| "password": "hunter3", | ||
| "username": "user", | ||
| "tunnel_method": { | ||
| "ssh_key": "hunter4", | ||
| "tunnel_host": "host", | ||
| "tunnel_port": 22, | ||
| "tunnel_user": "user", | ||
| "tunnel_method": "SSH_KEY_AUTH" | ||
| }, | ||
| "replication_method": { | ||
| "method": "Standard" | ||
| } | ||
| } |
22 changes: 22 additions & 0 deletions
22
airbyte-config/persistence/src/test/resources/postgres_ssh_key/updated_partial_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| { | ||
| "ssl": false, | ||
| "host": "host", | ||
| "port": 5432, | ||
| "database": "db", | ||
| "password": { | ||
| "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22_v2" | ||
| }, | ||
| "username": "user", | ||
| "tunnel_method": { | ||
| "ssh_key": { | ||
| "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_2c2ef2b3-259a-4e73-96d1-f56dacee2e5e_v2" | ||
| }, | ||
| "tunnel_host": "host", | ||
| "tunnel_port": 22, | ||
| "tunnel_user": "user", | ||
| "tunnel_method": "SSH_KEY_AUTH" | ||
| }, | ||
| "replication_method": { | ||
| "method": "Standard" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.