From fa040da0843da3ea6d602ae7cf275a0780434b5b Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Mon, 15 Nov 2021 14:36:50 +0100 Subject: [PATCH] Implement protocol change for OAuth outputs (#7917) * Change OAuth API * Change protocol for new OAuthConfigSpecification * Refactor OAuth classes and tests * Remove webbackend source/destination creation * Change from webback to normal API * Implement new protocol change with OAuth specs Co-authored-by: Sherif A. Nada * format * format Co-authored-by: Sherif A. Nada --- airbyte-api/src/main/openapi/config.yaml | 42 --- .../airbyte_cdk/models/airbyte_protocol.py | 2 +- .../models/airbyte_protocol.py | 2 +- airbyte-oauth/build.gradle | 1 + .../java/io/airbyte/oauth/BaseOAuth2Flow.java | 46 ++- .../java/io/airbyte/oauth/BaseOAuthFlow.java | 94 +++-- .../io/airbyte/oauth/MoreOAuthParameters.java | 77 ++++ .../oauth/OAuthFlowImplementation.java | 20 ++ .../oauth/OAuthImplementationFactory.java | 1 + .../oauth/flows/IntercomOAuthFlow.java | 10 +- .../oauth/flows/QuickbooksOAuthFlow.java | 19 +- .../oauth/flows/SalesforceOAuthFlow.java | 2 +- .../flows/SnapchatMarketingOAuthFlow.java | 4 +- .../oauth/flows/SurveymonkeyOAuthFlow.java | 2 +- .../airbyte/oauth/flows/TrelloOAuthFlow.java | 35 +- .../flows/facebook/FacebookOAuthFlow.java | 4 +- .../facebook/FacebookPagesOAuthFlow.java | 7 + .../flows/facebook/InstagramOAuthFlow.java | 7 + .../google/GoogleSearchConsoleOAuthFlow.java | 3 +- .../oauth/MoreOAuthParametersTest.java | 150 ++++++++ .../oauth/flows/AsanaOAuthFlowTest.java | 75 +--- .../oauth/flows/BaseOAuthFlowTest.java | 331 ++++++++++++++++++ .../oauth/flows/GithubOAuthFlowTest.java | 86 ++--- .../oauth/flows/HubspotOAuthFlowTest.java | 74 +--- .../oauth/flows/IntercomOAuthFlowTest.java | 87 ++--- .../oauth/flows/QuickbooksOAuthFlowTest.java | 21 ++ .../oauth/flows/SalesforceOAuthFlowTest.java | 75 +--- .../oauth/flows/SlackOAuthFlowTest.java | 74 +--- .../flows/SnapchatMarketingOAuthFlowTest.java | 75 +--- .../flows/SurveymonkeyOAuthFlowTest.java | 117 ++----- .../oauth/flows/TrelloOAuthFlowTest.java | 14 +- .../FacebookMarketingOAuthFlowTest.java | 52 +++ .../flows/facebook/FacebookOAuthFlowTest.java | 95 ----- .../facebook/FacebookPagesOAuthFlowTest.java | 52 +++ .../facebook/InstagramOAuthFlowTest.java | 52 +++ .../flows/google/GoogleAdsOAuthFlowTest.java | 97 +---- .../google/GoogleAnalyticsOAuthFlowTest.java | 187 +--------- .../GoogleSearchConsoleOAuthFlowTest.java | 100 +----- .../google/GoogleSheetsOAuthFlowTest.java | 99 +----- .../job_factory/OAuthConfigSupplier.java | 75 ++-- .../job_factory/OAuthConfigSupplierTest.java | 104 +----- .../airbyte/server/apis/ConfigurationApi.java | 18 +- .../converters/OauthModelConverter.java | 28 +- .../airbyte/server/handlers/OAuthHandler.java | 62 +++- .../server/handlers/SchedulerHandler.java | 15 +- .../WebBackendDestinationsHandler.java | 38 -- .../handlers/WebBackendSourcesHandler.java | 35 -- .../server/handlers/OAuthHandlerTest.java | 11 +- .../WebBackendSourcesHandlerTest.java | 98 ------ .../src/core/resources/Destination.tsx | 7 +- airbyte-webapp/src/core/resources/Source.tsx | 7 +- .../api/generated-api-html/index.html | 132 +------ 52 files changed, 1235 insertions(+), 1686 deletions(-) create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/QuickbooksOAuthFlowTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookMarketingOAuthFlowTest.java delete mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlowTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlowTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlowTest.java delete mode 100644 airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendDestinationsHandler.java delete mode 100644 airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendSourcesHandler.java delete mode 100644 airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourcesHandlerTest.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 78ff0c3d7476c..ca392f7a164bb 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1494,48 +1494,6 @@ paths: $ref: "#/components/schemas/WebBackendConnectionReadList" "422": $ref: "#/components/responses/InvalidInputResponse" - /v1/web_backend/sources/create: - post: - tags: - - web_backend - summary: Create a source - operationId: webBackendCreateSource - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SourceCreate" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SourceRead" - "422": - $ref: "#/components/responses/InvalidInputResponse" - /v1/web_backend/destinations/create: - post: - tags: - - web_backend - summary: Create a destination - operationId: webBackendCreateDestination - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/DestinationCreate" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/DestinationRead" - "422": - $ref: "#/components/responses/InvalidInputResponse" /v1/jobs/list: post: tags: diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 0de561d5478db..ed89e1d7b4416 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -114,8 +114,8 @@ class AuthSpecification(BaseModel): class AuthFlowType(Enum): - oauth1_0 = "oauth1.0" oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class OAuthConfigSpecification(BaseModel): diff --git a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py index 0de561d5478db..ed89e1d7b4416 100644 --- a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py +++ b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py @@ -114,8 +114,8 @@ class AuthSpecification(BaseModel): class AuthFlowType(Enum): - oauth1_0 = "oauth1.0" oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class OAuthConfigSpecification(BaseModel): diff --git a/airbyte-oauth/build.gradle b/airbyte-oauth/build.gradle index a5b74ae07a953..3311cc182977d 100644 --- a/airbyte-oauth/build.gradle +++ b/airbyte-oauth/build.gradle @@ -7,5 +7,6 @@ dependencies { implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') implementation project(':airbyte-json-validation') + implementation project(':airbyte-protocol:models') testImplementation project(':airbyte-oauth') } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuth2Flow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuth2Flow.java index a67d7e1bd42f8..e132a8eea87d9 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuth2Flow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuth2Flow.java @@ -11,6 +11,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.protocol.models.OAuthConfigSpecification; import java.io.IOException; import java.lang.reflect.Type; import java.net.URI; @@ -106,6 +107,7 @@ protected String getState() { } @Override + @Deprecated public Map completeSourceOAuth(final UUID workspaceId, final UUID sourceDefinitionId, final Map queryParams, @@ -124,6 +126,7 @@ public Map completeSourceOAuth(final UUID workspaceId, } @Override + @Deprecated public Map completeDestinationOAuth(final UUID workspaceId, final UUID destinationDefinitionId, final Map queryParams, @@ -141,6 +144,46 @@ public Map completeDestinationOAuth(final UUID workspaceId, getDefaultOAuthOutputPath()); } + @Override + public Map completeSourceOAuth(final UUID workspaceId, + final UUID sourceDefinitionId, + final Map queryParams, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final OAuthConfigSpecification oAuthConfigSpecification) + throws IOException, ConfigNotFoundException { + final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, sourceDefinitionId); + return formatOAuthOutput( + oAuthParamConfig, + completeOAuthFlow( + getClientIdUnsafe(oAuthParamConfig), + getClientSecretUnsafe(oAuthParamConfig), + extractCodeParameter(queryParams), + redirectUrl, + oAuthParamConfig), + oAuthConfigSpecification); + } + + @Override + public Map completeDestinationOAuth(final UUID workspaceId, + final UUID destinationDefinitionId, + final Map queryParams, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final OAuthConfigSpecification oAuthConfigSpecification) + throws IOException, ConfigNotFoundException { + final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); + return formatOAuthOutput( + oAuthParamConfig, + completeOAuthFlow( + getClientIdUnsafe(oAuthParamConfig), + getClientSecretUnsafe(oAuthParamConfig), + extractCodeParameter(queryParams), + redirectUrl, + oAuthParamConfig), + oAuthConfigSpecification); + } + protected Map completeOAuthFlow(final String clientId, final String clientSecret, final String authCode, @@ -212,7 +255,8 @@ protected Map extractOAuthOutput(final JsonNode data, final Stri } @Override - protected List getDefaultOAuthOutputPath() { + @Deprecated + public List getDefaultOAuthOutputPath() { return List.of("credentials"); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java index 3d6d5e5794267..f0f2aa17a21cc 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java @@ -5,22 +5,26 @@ package io.airbyte.oauth; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationOAuthParameter; import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.protocol.models.OAuthConfigSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; /** - * Abstract Class implementing common base methods for managing: - oAuth config (instance-wide) - * parameters - oAuth specifications + * Abstract Class implementing common base methods for managing oAuth config (instance-wide) and + * oAuth specifications */ public abstract class BaseOAuthFlow implements OAuthFlowImplementation { @@ -35,7 +39,9 @@ protected JsonNode getSourceOAuthParamConfig(final UUID workspaceId, final UUID final Optional param = MoreOAuthParameters.getSourceOAuthParameter( configRepository.listSourceOAuthParam().stream(), workspaceId, sourceDefinitionId); if (param.isPresent()) { - return param.get().getConfiguration(); + // TODO: if we write a flyway migration to flatten persisted configs in db, we don't need to flatten + // here see https://github.com/airbytehq/airbyte/issues/7624 + return MoreOAuthParameters.flattenOAuthConfig(param.get().getConfiguration()); } else { throw new ConfigNotFoundException(ConfigSchema.SOURCE_OAUTH_PARAM, "Undefined OAuth Parameter."); } @@ -50,7 +56,9 @@ protected JsonNode getDestinationOAuthParamConfig(final UUID workspaceId, final final Optional param = MoreOAuthParameters.getDestinationOAuthParameter( configRepository.listDestinationOAuthParam().stream(), workspaceId, destinationDefinitionId); if (param.isPresent()) { - return param.get().getConfiguration(); + // TODO: if we write a flyway migration to flatten persisted configs in db, we don't need to flatten + // here see https://github.com/airbytehq/airbyte/issues/7624 + return MoreOAuthParameters.flattenOAuthConfig(param.get().getConfiguration()); } else { throw new ConfigNotFoundException(ConfigSchema.DESTINATION_OAUTH_PARAM, "Undefined OAuth Parameter."); } @@ -66,17 +74,7 @@ protected JsonNode getDestinationOAuthParamConfig(final UUID workspaceId, final * @return The configured Client ID used for this oauth flow */ protected String getClientIdUnsafe(final JsonNode oauthConfig) { - final List path = new ArrayList<>(getDefaultOAuthOutputPath()); - path.add("client_id"); - JsonNode result = oauthConfig; - for (final String node : path) { - if (result.get(node) != null) { - result = result.get(node); - } else { - throw new IllegalArgumentException(String.format("Undefined parameter '%s' necessary for the OAuth Flow.", String.join(".", path))); - } - } - return result.asText(); + return getConfigValueUnsafe(oauthConfig, "client_id"); } /** @@ -86,40 +84,70 @@ protected String getClientIdUnsafe(final JsonNode oauthConfig) { * @return The configured client secret for this OAuthFlow */ protected String getClientSecretUnsafe(final JsonNode oauthConfig) { - final List path = new ArrayList<>(getDefaultOAuthOutputPath()); - path.add("client_secret"); - JsonNode result = oauthConfig; - for (final String node : path) { - if (result.get(node) != null) { - result = result.get(node); - } else { - throw new IllegalArgumentException(String.format("Undefined parameter '%s' necessary for the OAuth Flow.", String.join(".", path))); - } + return getConfigValueUnsafe(oauthConfig, "client_secret"); + } + + private static String getConfigValueUnsafe(final JsonNode oauthConfig, final String fieldName) { + if (oauthConfig.get(fieldName) != null) { + return oauthConfig.get(fieldName).asText(); + } else { + throw new IllegalArgumentException(String.format("Undefined parameter '%s' necessary for the OAuth Flow.", fieldName)); } - return result.asText(); } /** * completeOAuth calls should output a flat map of fields produced by the oauth flow to be forwarded - * back to the connector config. This function is in charge of formatting such flat map of fields - * into nested Map accordingly to follow the expected @param outputPath. - * + * back to the connector config. This @deprecated function is used when the connector's oauth + * specifications are unknown. So it ends up using hard-coded output path in the OAuth Flow + * implementation instead of relying on the connector's specification to determine where the outputs + * should be stored. */ + @Deprecated protected Map formatOAuthOutput(final JsonNode oAuthParamConfig, final Map oauthOutput, final List outputPath) { - Map result = oauthOutput; + Map result = new HashMap<>(oauthOutput); + // inject masked params outputs + for (final String key : Jsons.keys(oAuthParamConfig)) { + result.put(key, MoreOAuthParameters.SECRET_MASK); + } for (final String node : outputPath) { result = Map.of(node, result); } - // TODO chris to implement injection of oAuthParamConfig in outputs return result; } + /** + * completeOAuth calls should output a flat map of fields produced by the oauth flow to be forwarded + * back to the connector config. This function follows the connector's oauth specifications of which + * outputs are expected and filters them accordingly. + */ + protected Map formatOAuthOutput(final JsonNode oAuthParamConfig, + final Map completeOAuthFlow, + final OAuthConfigSpecification oAuthConfigSpecification) { + final Builder outputs = ImmutableMap.builder(); + // inject masked params outputs + for (final String key : Jsons.keys(oAuthParamConfig)) { + if (oAuthConfigSpecification.getCompleteOauthServerOutputSpecification().has(key)) { + outputs.put(key, MoreOAuthParameters.SECRET_MASK); + } + } + // collect oauth result outputs + for (final String key : completeOAuthFlow.keySet()) { + if (oAuthConfigSpecification.getCompleteOauthOutputSpecification().has(key)) { + outputs.put(key, completeOAuthFlow.get(key)); + } + } + return outputs.build(); + } + /** * This function should be redefined in each OAuthFlow implementation to isolate such "hardcoded" - * values. + * values. It is being @deprecated because the output path should not be "hard-coded" in the OAuth + * flow implementation classes anymore but will be specified as part of the OAuth Specification + * object */ - protected abstract List getDefaultOAuthOutputPath(); + @Deprecated + public abstract List getDefaultOAuthOutputPath(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/MoreOAuthParameters.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/MoreOAuthParameters.java index 1c8ed097152ce..60ed411008825 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/MoreOAuthParameters.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/MoreOAuthParameters.java @@ -4,15 +4,28 @@ package io.airbyte.oauth; +import static com.fasterxml.jackson.databind.node.JsonNodeType.OBJECT; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Strings; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationOAuthParameter; import io.airbyte.config.SourceOAuthParameter; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MoreOAuthParameters { + private static final Logger LOGGER = LoggerFactory.getLogger(Jsons.class); + public static final String SECRET_MASK = "******"; + public static Optional getSourceOAuthParameter( final Stream stream, final UUID workspaceId, @@ -37,4 +50,68 @@ public static Optional getDestinationOAuthParameter( .thenComparing(DestinationOAuthParameter::getOauthParameterId)); } + public static JsonNode flattenOAuthConfig(final JsonNode config) { + if (config.getNodeType() == OBJECT) { + return flattenOAuthConfig((ObjectNode) Jsons.emptyObject(), (ObjectNode) config); + } else { + throw new IllegalStateException("Config is not an Object config, unable to flatten"); + } + } + + private static ObjectNode flattenOAuthConfig(final ObjectNode flatConfig, final ObjectNode configToFlatten) { + final List keysToFlatten = new ArrayList<>(); + for (final String key : Jsons.keys(configToFlatten)) { + if (configToFlatten.get(key).getNodeType() == OBJECT) { + keysToFlatten.add(key); + } else if (!flatConfig.has(key)) { + flatConfig.set(key, configToFlatten.get(key)); + } else { + throw new IllegalStateException(String.format("OAuth Config's key '%s' already exists", key)); + } + } + keysToFlatten.forEach(key -> flattenOAuthConfig(flatConfig, (ObjectNode) configToFlatten.get(key))); + return flatConfig; + } + + public static JsonNode mergeJsons(final ObjectNode mainConfig, final ObjectNode fromConfig) { + return mergeJsons(mainConfig, fromConfig, null); + } + + public static JsonNode mergeJsons(final ObjectNode mainConfig, final ObjectNode fromConfig, final JsonNode maskedValue) { + for (final String key : Jsons.keys(fromConfig)) { + if (fromConfig.get(key).getNodeType() == OBJECT) { + // nested objects are merged rather than overwrite the contents of the equivalent object in config + if (mainConfig.get(key) == null) { + mergeJsons(mainConfig.putObject(key), (ObjectNode) fromConfig.get(key), maskedValue); + } else if (mainConfig.get(key).getNodeType() == OBJECT) { + mergeJsons((ObjectNode) mainConfig.get(key), (ObjectNode) fromConfig.get(key), maskedValue); + } else { + throw new IllegalStateException("Can't merge an object node into a non-object node!"); + } + } else { + if (maskedValue != null && !maskedValue.isNull()) { + LOGGER.debug(String.format("Masking instance wide parameter %s in config", key)); + mainConfig.set(key, maskedValue); + } else { + if (!mainConfig.has(key) || isSecretMask(mainConfig.get(key).asText())) { + LOGGER.debug(String.format("injecting instance wide parameter %s into config", key)); + mainConfig.set(key, fromConfig.get(key)); + } + } + } + } + return mainConfig; + } + + public static JsonNode getSecretMask() { + // TODO secrets should be masked with the correct type + // https://github.com/airbytehq/airbyte/issues/5990 + // In the short-term this is not world-ending as all secret fields are currently strings + return Jsons.jsonNode(SECRET_MASK); + } + + private static boolean isSecretMask(final String input) { + return Strings.isNullOrEmpty(input.replaceAll("\\*", "")); + } + } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthFlowImplementation.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthFlowImplementation.java index e43bf3da698cd..ec03f0109f283 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthFlowImplementation.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthFlowImplementation.java @@ -4,7 +4,9 @@ package io.airbyte.oauth; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.protocol.models.OAuthConfigSpecification; import java.io.IOException; import java.util.Map; import java.util.UUID; @@ -15,10 +17,28 @@ public interface OAuthFlowImplementation { String getDestinationConsentUrl(UUID workspaceId, UUID destinationDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException; + @Deprecated Map completeSourceOAuth(UUID workspaceId, UUID sourceDefinitionId, Map queryParams, String redirectUrl) throws IOException, ConfigNotFoundException; + Map completeSourceOAuth(UUID workspaceId, + UUID sourceDefinitionId, + Map queryParams, + String redirectUrl, + JsonNode inputOAuthConfiguration, + OAuthConfigSpecification oauthConfigSpecification) + throws IOException, ConfigNotFoundException; + + @Deprecated Map completeDestinationOAuth(UUID workspaceId, UUID destinationDefinitionId, Map queryParams, String redirectUrl) throws IOException, ConfigNotFoundException; + Map completeDestinationOAuth(UUID workspaceId, + UUID destinationDefinitionId, + Map queryParams, + String redirectUrl, + JsonNode inputOAuthConfiguration, + OAuthConfigSpecification oAuthConfigSpecification) + throws IOException, ConfigNotFoundException; + } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 83a044b2ed661..c25d821a9daff 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -34,6 +34,7 @@ public class OAuthImplementationFactory { public OAuthImplementationFactory(final ConfigRepository configRepository, final HttpClient httpClient) { OAUTH_FLOW_MAPPING = ImmutableMap.builder() + // These are listed in alphabetical order below to facilitate manual look-up: .put("airbyte/source-asana", new AsanaOAuthFlow(configRepository, httpClient)) .put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository, httpClient)) .put("airbyte/source-facebook-pages", new FacebookPagesOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java index 25d0daa66316b..3ec6cd48bb1b6 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/IntercomOAuthFlow.java @@ -23,17 +23,17 @@ public class IntercomOAuthFlow extends BaseOAuth2Flow { private static final String AUTHORIZE_URL = "https://app.intercom.com/a/oauth/connect"; private static final String ACCESS_TOKEN_URL = "https://api.intercom.io/auth/eagle/token"; - public IntercomOAuthFlow(ConfigRepository configRepository, HttpClient httpClient) { + public IntercomOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { super(configRepository, httpClient); } @VisibleForTesting - public IntercomOAuthFlow(ConfigRepository configRepository, final HttpClient httpClient, Supplier stateSupplier) { + public IntercomOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { super(configRepository, httpClient, stateSupplier); } @Override - protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException { + protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl) throws IOException { try { return new URIBuilder(AUTHORIZE_URL) .addParameter("client_id", clientId) @@ -41,7 +41,7 @@ protected String formatConsentUrl(UUID definitionId, String clientId, String red .addParameter("response_type", "code") .addParameter("state", getState()) .build().toString(); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { throw new IOException("Failed to format Consent URL for OAuth flow", e); } } @@ -60,7 +60,7 @@ protected Map extractOAuthOutput(final JsonNode data, final Stri } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/QuickbooksOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/QuickbooksOAuthFlow.java index 5c5f410828c07..5918b8892b71f 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/QuickbooksOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/QuickbooksOAuthFlow.java @@ -4,6 +4,7 @@ package io.airbyte.oauth.flows; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.BaseOAuth2Flow; @@ -13,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Supplier; import org.apache.http.client.utils.URIBuilder; public class QuickbooksOAuthFlow extends BaseOAuth2Flow { @@ -20,16 +22,21 @@ public class QuickbooksOAuthFlow extends BaseOAuth2Flow { final String CONSENT_URL = "https://appcenter.intuit.com/app/connect/oauth2"; final String TOKEN_URL = "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer"; - public QuickbooksOAuthFlow(ConfigRepository configRepository, HttpClient httpClient) { + public QuickbooksOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { super(configRepository, httpClient); } + @VisibleForTesting + public QuickbooksOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + public String getScopes() { return "com.intuit.quickbooks.accounting"; } @Override - protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException { + protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl) throws IOException { try { return (new URIBuilder(CONSENT_URL) @@ -45,7 +52,11 @@ protected String formatConsentUrl(UUID definitionId, String clientId, String red } } - protected Map getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl) { + @Override + protected Map getAccessTokenQueryParameters(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl) { return ImmutableMap.builder() // required .put("redirect_uri", redirectUrl) @@ -69,7 +80,7 @@ protected String getAccessTokenUrl() { * values. */ @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of("credentials"); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java index 512311c9ea4cb..e83773c223a1e 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java @@ -68,7 +68,7 @@ protected Map getAccessTokenQueryParameters(final String clientI } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlow.java index c9472f1182006..1579ac8a12b4d 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlow.java @@ -21,8 +21,6 @@ * Following docs from https://marketingapi.snapchat.com/docs/#authentication */ public class SnapchatMarketingOAuthFlow extends BaseOAuth2Flow { - // Clickable link for IDE - // https://help.salesforce.com/s/articleView?language=en_US&id=sf.remoteaccess_oauth_web_server_flow.htm private static final String AUTHORIZE_URL = "https://accounts.snapchat.com/login/oauth2/authorize"; private static final String ACCESS_TOKEN_URL = "https://accounts.snapchat.com/login/oauth2/access_token"; @@ -69,7 +67,7 @@ protected Map getAccessTokenQueryParameters(final String clientI } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlow.java index cda8da4b24dc9..63c10d703a17c 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlow.java @@ -73,7 +73,7 @@ protected Map extractOAuthOutput(final JsonNode data, final Stri } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java index 39443ad327462..a7ae99c61ea3e 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java @@ -16,6 +16,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.protocol.models.OAuthConfigSpecification; import java.io.IOException; import java.net.http.HttpClient; import java.util.List; @@ -84,26 +85,52 @@ private String getConsentUrl(final JsonNode oAuthParamConfig, final String redir } @Override + @Deprecated public Map completeSourceOAuth(final UUID workspaceId, final UUID sourceDefinitionId, final Map queryParams, final String redirectUrl) throws IOException, ConfigNotFoundException { final JsonNode oAuthParamConfig = getSourceOAuthParamConfig(workspaceId, sourceDefinitionId); - return formatOAuthOutput(oAuthParamConfig, completeOAuth(oAuthParamConfig, queryParams), getDefaultOAuthOutputPath()); + return formatOAuthOutput(oAuthParamConfig, internalCompleteOAuth(oAuthParamConfig, queryParams), getDefaultOAuthOutputPath()); } @Override + @Deprecated public Map completeDestinationOAuth(final UUID workspaceId, final UUID destinationDefinitionId, final Map queryParams, final String redirectUrl) throws IOException, ConfigNotFoundException { final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); - return formatOAuthOutput(oAuthParamConfig, completeOAuth(oAuthParamConfig, queryParams), getDefaultOAuthOutputPath()); + return formatOAuthOutput(oAuthParamConfig, internalCompleteOAuth(oAuthParamConfig, queryParams), getDefaultOAuthOutputPath()); } - private Map completeOAuth(final JsonNode oAuthParamConfig, final Map queryParams) + @Override + public Map completeSourceOAuth(final UUID workspaceId, + final UUID sourceDefinitionId, + final Map queryParams, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final OAuthConfigSpecification oAuthConfigSpecification) + throws IOException, ConfigNotFoundException { + final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, sourceDefinitionId); + return formatOAuthOutput(oAuthParamConfig, internalCompleteOAuth(oAuthParamConfig, queryParams), oAuthConfigSpecification); + } + + @Override + public Map completeDestinationOAuth(final UUID workspaceId, + final UUID destinationDefinitionId, + final Map queryParams, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final OAuthConfigSpecification oAuthConfigSpecification) + throws IOException, ConfigNotFoundException { + final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); + return formatOAuthOutput(oAuthParamConfig, internalCompleteOAuth(oAuthParamConfig, queryParams), oAuthConfigSpecification); + } + + private Map internalCompleteOAuth(final JsonNode oAuthParamConfig, final Map queryParams) throws IOException { final String clientKey = getClientIdUnsafe(oAuthParamConfig); if (!queryParams.containsKey("oauth_verifier") || !queryParams.containsKey("oauth_token")) { @@ -124,7 +151,7 @@ private Map completeOAuth(final JsonNode oAuthParamConfig, final } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java index 5499916f04ca7..5b787800e9469 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java @@ -5,7 +5,6 @@ package io.airbyte.oauth.flows.facebook; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import io.airbyte.commons.json.Jsons; import io.airbyte.config.persistence.ConfigRepository; @@ -35,7 +34,6 @@ public FacebookOAuthFlow(final ConfigRepository configRepository, final HttpClie super(configRepository, httpClient); } - @VisibleForTesting FacebookOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { super(configRepository, httpClient, stateSupplier); } @@ -122,7 +120,7 @@ protected String getLongLivedAccessToken(final String clientId, final String cli } @Override - protected List getDefaultOAuthOutputPath() { + public List getDefaultOAuthOutputPath() { return List.of(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlow.java index 466cdaa169fd5..082894b65387d 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlow.java @@ -4,8 +4,10 @@ package io.airbyte.oauth.flows.facebook; +import com.google.common.annotations.VisibleForTesting; import io.airbyte.config.persistence.ConfigRepository; import java.net.http.HttpClient; +import java.util.function.Supplier; public class FacebookPagesOAuthFlow extends FacebookOAuthFlow { @@ -15,6 +17,11 @@ public FacebookPagesOAuthFlow(final ConfigRepository configRepository, final Htt super(configRepository, httpClient); } + @VisibleForTesting + FacebookPagesOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + @Override protected String getScopes() { return SCOPES; diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlow.java index d3a31b4a706da..c330b42980a6f 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlow.java @@ -4,8 +4,10 @@ package io.airbyte.oauth.flows.facebook; +import com.google.common.annotations.VisibleForTesting; import io.airbyte.config.persistence.ConfigRepository; import java.net.http.HttpClient; +import java.util.function.Supplier; // Instagram Graph API require Facebook API User token public class InstagramOAuthFlow extends FacebookMarketingOAuthFlow { @@ -16,6 +18,11 @@ public InstagramOAuthFlow(final ConfigRepository configRepository, final HttpCli super(configRepository, httpClient); } + @VisibleForTesting + InstagramOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + @Override protected String getScopes() { return SCOPES; diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java index 84ceccfd54e03..e4ddc1e1185a0 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java @@ -30,7 +30,8 @@ protected String getScope() { } @Override - protected List getDefaultOAuthOutputPath() { + @Deprecated + public List getDefaultOAuthOutputPath() { return List.of("authorization"); } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java new file mode 100644 index 0000000000000..77ed66adc77f4 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +public class MoreOAuthParametersTest { + + @Test + void testFlattenConfig() { + final JsonNode nestedConfig = Jsons.jsonNode(Map.of( + "field", "value1", + "top-level", Map.of( + "nested_field", "value2"))); + final JsonNode expectedConfig = Jsons.jsonNode(Map.of( + "field", "value1", + "nested_field", "value2")); + final JsonNode actualConfig = MoreOAuthParameters.flattenOAuthConfig(nestedConfig); + assertEquals(expectedConfig, actualConfig); + } + + @Test + void testFailureFlattenConfig() { + final JsonNode nestedConfig = Jsons.jsonNode(Map.of( + "field", "value1", + "top-level", Map.of( + "nested_field", "value2", + "field", "value3"))); + assertThrows(IllegalStateException.class, () -> MoreOAuthParameters.flattenOAuthConfig(nestedConfig)); + } + + private void maskAllValues(final ObjectNode node) { + for (final String key : Jsons.keys(node)) { + if (node.get(key).getNodeType() == JsonNodeType.OBJECT) { + maskAllValues((ObjectNode) node.get(key)); + } else { + node.set(key, MoreOAuthParameters.getSecretMask()); + } + } + } + + @Test + void testInjectUnnestedNode_Masked() { + final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); + final ObjectNode maskedOauthParams = Jsons.clone(oauthParams); + maskAllValues(maskedOauthParams); + final ObjectNode actual = generateJsonConfig(); + final ObjectNode expected = Jsons.clone(actual); + expected.setAll(maskedOauthParams); + + MoreOAuthParameters.mergeJsons(actual, oauthParams, MoreOAuthParameters.getSecretMask()); + assertEquals(expected, actual); + } + + @Test + void testInjectUnnestedNode_Unmasked() { + final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); + + final ObjectNode actual = generateJsonConfig(); + final ObjectNode expected = Jsons.clone(actual); + expected.setAll(oauthParams); + + MoreOAuthParameters.mergeJsons(actual, oauthParams); + + assertEquals(expected, actual); + } + + @Test + void testInjectNewNestedNode_Masked() { + final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); + final ObjectNode maskedOauthParams = Jsons.clone(oauthParams); + maskAllValues(maskedOauthParams); + final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() + .put("oauth_credentials", oauthParams) + .build()); + + // nested node does not exist in actual object + final ObjectNode actual = generateJsonConfig(); + final ObjectNode expected = Jsons.clone(actual); + expected.putObject("oauth_credentials").setAll(maskedOauthParams); + + MoreOAuthParameters.mergeJsons(actual, nestedConfig, MoreOAuthParameters.getSecretMask()); + assertEquals(expected, actual); + } + + @Test + @DisplayName("A nested config should be inserted with the same nesting structure") + void testInjectNewNestedNode_Unmasked() { + final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); + final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() + .put("oauth_credentials", oauthParams) + .build()); + + // nested node does not exist in actual object + final ObjectNode actual = generateJsonConfig(); + final ObjectNode expected = Jsons.clone(actual); + expected.putObject("oauth_credentials").setAll(oauthParams); + + MoreOAuthParameters.mergeJsons(actual, nestedConfig); + + assertEquals(expected, actual); + } + + @Test + @DisplayName("A nested node which partially exists in the main config should be merged into the main config, not overwrite the whole nested object") + void testInjectedPartiallyExistingNestedNode_Unmasked() { + final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); + final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() + .put("oauth_credentials", oauthParams) + .build()); + + // nested node partially exists in actual object + final ObjectNode actual = generateJsonConfig(); + actual.putObject("oauth_credentials").put("irrelevant_field", "_"); + final ObjectNode expected = Jsons.clone(actual); + ((ObjectNode) expected.get("oauth_credentials")).setAll(oauthParams); + + MoreOAuthParameters.mergeJsons(actual, nestedConfig); + + assertEquals(expected, actual); + } + + private ObjectNode generateJsonConfig() { + return (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() + .put("apiSecret", "123") + .put("client", "testing") + .build()); + } + + private Map generateOAuthParameters() { + return ImmutableMap.builder() + .put("api_secret", "mysecret") + .put("api_client", UUID.randomUUID().toString()) + .build(); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/AsanaOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/AsanaOAuthFlowTest.java index b1b4ee0427186..c1e571fa8b6bd 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/AsanaOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/AsanaOAuthFlowTest.java @@ -4,77 +4,18 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class AsanaOAuthFlowTest extends BaseOAuthFlowTest { -public class AsanaOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private AsanaOAuthFlow asanaOAuthFlow; - private HttpClient httpClient; - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; - } - - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - asanaOAuthFlow = new AsanaOAuthFlow(configRepository, httpClient, AsanaOAuthFlowTest::getConstantState); - } - - @Test - public void testGetSourceConsentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String consentUrl = - asanaOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://app.asana.com/-/oauth_authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state", - consentUrl); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new AsanaOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @Test - public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { - - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - asanaOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://app.asana.com/-/oauth_authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java new file mode 100644 index 0000000000000..8e1a3e979465b --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import io.airbyte.protocol.models.OAuthConfigSpecification; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public abstract class BaseOAuthFlowTest { + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private HttpClient httpClient; + private ConfigRepository configRepository; + private BaseOAuthFlow oauthFlow; + + private UUID workspaceId; + private UUID definitionId; + + protected HttpClient getHttpClient() { + return httpClient; + } + + protected ConfigRepository getConfigRepository() { + return configRepository; + } + + @BeforeEach + public void setup() throws JsonValidationException, IOException { + httpClient = mock(HttpClient.class); + configRepository = mock(ConfigRepository.class); + oauthFlow = getOAuthFlow(); + + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(getOAuthParamConfig()))); + when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withDestinationDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(getOAuthParamConfig()))); + } + + /** + * @return the oauth flow implementation to test + */ + protected abstract BaseOAuthFlow getOAuthFlow(); + + /** + * @return the expected consent URL + */ + protected abstract String getExpectedConsentUrl(); + + /** + * @return the instance wide config params for this oauth flow + */ + protected JsonNode getOAuthParamConfig() { + return Jsons.jsonNode(ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()); + } + + /** + * @return the full output expected to be returned by this oauth flow + all its instance wide + * variables + */ + protected Map getExpectedOutput() { + return Map.of( + "refresh_token", "refresh_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + /** + * @return the backward compatible path that is used in the deprecated oauth flows (should match + * getDefaultOAuthOutputPath()) + */ + protected List getExpectedOutputPath() { + return List.of("credentials"); + } + + /** + * @return the input configuration sent to oauth flow (values from connector config) + */ + protected JsonNode getInputOAuthConfiguration() { + return Jsons.emptyObject(); + } + + /** + * @return the output specification used to filter what the oauth flow should be returning + */ + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of( + "refresh_token", Map.of("type", "String"))); + } + + /** + * @return the output specification used to filter what the oauth flow should be returning + */ + protected JsonNode getOutputOAuthParameterSpecification() { + return Jsons.jsonNode(Map.of( + "client_id", Map.of("type", "String"))); + } + + /** + * @return the fitlered outputs once it is filtered by the output specifications + */ + protected Map getExpectedFilteredOutput() { + return Map.of( + "refresh_token", "refresh_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + + private OAuthConfigSpecification getoAuthConfigSpecification() { + return new OAuthConfigSpecification() + .withCompleteOauthOutputSpecification(getOutputOAuthSpecification()) + .withCompleteOauthServerOutputSpecification(getOutputOAuthParameterSpecification()); + } + + private OAuthConfigSpecification getEmptyOAuthConfigSpecification() { + return new OAuthConfigSpecification() + .withCompleteOauthOutputSpecification(Jsons.emptyObject()) + .withCompleteOauthServerOutputSpecification(Jsons.emptyObject()); + } + + protected String getConstantState() { + return "state"; + } + + @Test + public void testGetDefaultOutputPath() { + assertEquals(getExpectedOutputPath(), oauthFlow.getDefaultOAuthOutputPath()); + } + + @Test + public void testGetConsentUrlEmptyOAuthParameters() throws JsonValidationException, IOException { + when(configRepository.listSourceOAuthParam()).thenReturn(List.of()); + when(configRepository.listDestinationOAuthParam()).thenReturn(List.of()); + assertThrows(ConfigNotFoundException.class, () -> oauthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(ConfigNotFoundException.class, () -> oauthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + } + + @Test + public void testGetConsentUrlIncompleteOAuthParameters() throws IOException, JsonValidationException { + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.emptyObject()))); + when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withDestinationDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.emptyObject()))); + assertThrows(IllegalArgumentException.class, () -> oauthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(IllegalArgumentException.class, () -> oauthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + } + + @Test + public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { + final String consentUrl = oauthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals(getExpectedConsentUrl(), consentUrl); + } + + @Test + public void testGetDestinationConsentUrl() throws IOException, ConfigNotFoundException { + final String consentUrl = oauthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals(getExpectedConsentUrl(), consentUrl); + } + + @Test + public void testCompleteOAuthMissingCode() { + final Map queryParams = Map.of(); + assertThrows(IOException.class, () -> oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL)); + } + + @Test + public void testDeprecatedCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + Map actualRawQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + for (final String node : getExpectedOutputPath()) { + assertNotNull(actualRawQueryParams.get(node)); + actualRawQueryParams = (Map) actualRawQueryParams.get(node); + } + final Map expectedOutput = returnedCredentials; + final Map actualQueryParams = actualRawQueryParams; + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + + @Test + public void testDeprecatedCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, InterruptedException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + Map actualRawQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + for (final String node : getExpectedOutputPath()) { + assertNotNull(actualRawQueryParams.get(node)); + actualRawQueryParams = (Map) actualRawQueryParams.get(node); + } + final Map expectedOutput = returnedCredentials; + final Map actualQueryParams = actualRawQueryParams; + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + + @Test + public void testEmptyOutputCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + getInputOAuthConfiguration(), getEmptyOAuthConfigSpecification()); + assertEquals(0, actualQueryParams.size(), + String.format("Expected no values but got %s", actualQueryParams)); + } + + @Test + public void testEmptyOutputCompleteDestinationOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + getInputOAuthConfiguration(), getEmptyOAuthConfigSpecification()); + assertEquals(0, actualQueryParams.size(), + String.format("Expected no values but got %s", actualQueryParams)); + } + + @Test + public void testEmptyInputCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + Jsons.emptyObject(), getoAuthConfigSpecification()); + final Map expectedOutput = getExpectedFilteredOutput(); + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + + @Test + public void testEmptyInputCompleteDestinationOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + Jsons.emptyObject(), getoAuthConfigSpecification()); + final Map expectedOutput = getExpectedFilteredOutput(); + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + + @Test + public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + getInputOAuthConfiguration(), getoAuthConfigSpecification()); + final Map expectedOutput = getExpectedFilteredOutput(); + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + + @Test + public void testCompleteDestinationOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + final Map returnedCredentials = getExpectedOutput(); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, + getInputOAuthConfiguration(), getoAuthConfigSpecification()); + final Map expectedOutput = getExpectedFilteredOutput(); + assertEquals(expectedOutput.size(), actualQueryParams.size(), + String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java index fb5b4aa288d9e..b912ce90fcb7e 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java @@ -4,78 +4,42 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class GithubOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private GithubOAuthFlow githubOAuthFlow; - private HttpClient httpClient; - private static final String REDIRECT_URL = "https://airbyte.io"; +public class GithubOAuthFlowTest extends BaseOAuthFlowTest { - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new GithubOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode( - Map.of("credentials", - ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - githubOAuthFlow = new GithubOAuthFlow(configRepository, httpClient, GithubOAuthFlowTest::getConstantState); - + @Override + protected String getExpectedConsentUrl() { + return "https://github.com/login/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"; } - @Test - public void testGetSourceConsentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String consentUrl = githubOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals("https://github.com/login/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state", - consentUrl); + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); } - @Test - public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); + } - Map returnedCredentials = Map.of("access_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - githubOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HubspotOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HubspotOAuthFlowTest.java index 1a0ec8edd9bdd..40c4ec8cfc9b3 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HubspotOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HubspotOAuthFlowTest.java @@ -4,76 +4,18 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class HubspotOAuthFlowTest extends BaseOAuthFlowTest { -public class HubspotOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private HubspotOAuthFlow hubspotOAuthFlow; - private HttpClient httpClient; - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; - } - - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - hubspotOAuthFlow = new HubspotOAuthFlow(configRepository, httpClient, HubspotOAuthFlowTest::getConstantState); - - } - - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { - final String consentUrl = hubspotOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://app.hubspot.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scopes=content+crm.schemas.deals.read+crm.objects.owners.read+forms+tickets+e-commerce+crm.objects.companies.read+crm.lists.read+crm.objects.deals.read+crm.schemas.contacts.read+crm.objects.contacts.read+crm.schemas.companies.read+files+forms-uploaded-files+files.ui_hidden.read", - consentUrl); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new HubspotOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { - final var response = mock(HttpResponse.class); - var returnedCredentials = "{\"refresh_token\":\"refresh_token_response\"}"; - when(response.body()).thenReturn(returnedCredentials); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - hubspotOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", Jsons.deserialize(returnedCredentials))), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://app.hubspot.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scopes=content+crm.schemas.deals.read+crm.objects.owners.read+forms+tickets+e-commerce+crm.objects.companies.read+crm.lists.read+crm.objects.deals.read+crm.schemas.contacts.read+crm.objects.contacts.read+crm.schemas.companies.read+files+forms-uploaded-files+files.ui_hidden.read"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java index 8a109cd21f74d..0f71ba6a9e1bb 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/IntercomOAuthFlowTest.java @@ -4,77 +4,48 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; import java.util.List; import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class IntercomOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private IntercomOAuthFlow intercomoAuthFlow; - private HttpClient httpClient; - private static final String REDIRECT_URL = "https://airbyte.io"; +public class IntercomOAuthFlowTest extends BaseOAuthFlowTest { - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new IntercomOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - ConfigRepository configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode( - ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - intercomoAuthFlow = new IntercomOAuthFlow(configRepository, httpClient, IntercomOAuthFlowTest::getConstantState); + @Override + protected List getExpectedOutputPath() { + return List.of(); + } + @Override + protected String getExpectedConsentUrl() { + return "https://app.intercom.com/a/oauth/connect?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"; } - @Test - public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { - final String concentUrl = - intercomoAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals(concentUrl, - "https://app.intercom.com/a/oauth/connect?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"); + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); } - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); + } - Map returnedCredentials = Map.of("access_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - intercomoAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/QuickbooksOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/QuickbooksOAuthFlowTest.java new file mode 100644 index 0000000000000..f6999ddcc97e3 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/QuickbooksOAuthFlowTest.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import io.airbyte.oauth.BaseOAuthFlow; + +public class QuickbooksOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new QuickbooksOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://appcenter.intuit.com/app/connect/oauth2?client_id=test_client_id&scope=com.intuit.quickbooks.accounting&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"; + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SalesforceOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SalesforceOAuthFlowTest.java index dcaa0d672c73a..f7fd3b2e31c52 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SalesforceOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SalesforceOAuthFlowTest.java @@ -4,77 +4,24 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; +import io.airbyte.oauth.BaseOAuthFlow; import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class SalesforceOAuthFlowTest { - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private SalesforceOAuthFlow salesforceOAuthFlow; - private HttpClient httpClient; +public class SalesforceOAuthFlowTest extends BaseOAuthFlowTest { - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new SalesforceOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - salesforceOAuthFlow = new SalesforceOAuthFlow(configRepository, httpClient, SalesforceOAuthFlowTest::getConstantState); - + @Override + protected String getExpectedConsentUrl() { + return "https://login.salesforce.com/services/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"; } - @Test - public void testGetSourceConsentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String consentUrl = salesforceOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://login.salesforce.com/services/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state", - consentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { - - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - salesforceOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + @Override + protected List getExpectedOutputPath() { + return List.of(); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SlackOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SlackOAuthFlowTest.java index 3a27929387cf8..fdf391a94bec0 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SlackOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SlackOAuthFlowTest.java @@ -4,76 +4,18 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class SlackOAuthFlowTest extends BaseOAuthFlowTest { -public class SlackOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private SlackOAuthFlow slackOAuthFlow; - private HttpClient httpClient; - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new SlackOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - slackOAuthFlow = new SlackOAuthFlow(configRepository, httpClient, SlackOAuthFlowTest::getConstantState); - - } - - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { - final String consentUrl = slackOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals("https://slack.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scope=read", - consentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { - - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - slackOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://slack.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scope=read"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlowTest.java index 41a8f30250f1a..08976b47f1748 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SnapchatMarketingOAuthFlowTest.java @@ -4,77 +4,24 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; +import io.airbyte.oauth.BaseOAuthFlow; import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class SnapchatMarketingOAuthFlowTest { - private UUID workspaceId; - private UUID definitionId; - private ConfigRepository configRepository; - private SnapchatMarketingOAuthFlow snapchatOAuthFlow; - private HttpClient httpClient; +public class SnapchatMarketingOAuthFlowTest extends BaseOAuthFlowTest { - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new SnapchatMarketingOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - snapchatOAuthFlow = new SnapchatMarketingOAuthFlow(configRepository, httpClient, SnapchatMarketingOAuthFlowTest::getConstantState); - + @Override + protected List getExpectedOutputPath() { + return List.of(); } - @Test - public void testGetSourceConsentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String consentUrl = snapchatOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://accounts.snapchat.com/login/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=snapchat-marketing-api&state=state", - consentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { - - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - snapchatOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://accounts.snapchat.com/login/oauth2/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=snapchat-marketing-api&state=state"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlowTest.java index edc8833d6e5ac..83d9e7d9516ed 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/SurveymonkeyOAuthFlowTest.java @@ -4,111 +4,48 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; import java.util.List; import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class SurveymonkeyOAuthFlowTest { - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private SurveymonkeyOAuthFlow surveymonkeyOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - @BeforeEach - public void setup() throws JsonValidationException, IOException { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - surveymonkeyOAuthFlow = new SurveymonkeyOAuthFlow(configRepository, httpClient, SurveymonkeyOAuthFlowTest::getConstantState); +public class SurveymonkeyOAuthFlowTest extends BaseOAuthFlowTest { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new SurveymonkeyOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - private static String getConstantState() { - return "state"; + @Override + protected String getExpectedConsentUrl() { + return "https://api.surveymonkey.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state"; } - @Test - public void testGetSourceConsentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String consentUrl = surveymonkeyOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://api.surveymonkey.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state", - consentUrl); + @Override + protected List getExpectedOutputPath() { + return List.of(); } - @Test - public void testGetDestinationConcentUrl() throws IOException, InterruptedException, ConfigNotFoundException { - final String concentUrl = - surveymonkeyOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://api.surveymonkey.com/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&state=state", - concentUrl); + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); } - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { - - final Map returnedCredentials = Map.of("access_token", "access_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - surveymonkeyOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); } - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, InterruptedException { - - final Map returnedCredentials = Map.of("access_token", "access_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - surveymonkeyOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/TrelloOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/TrelloOAuthFlowTest.java index 8a4fa187a1f92..df88c307392d7 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/TrelloOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/TrelloOAuthFlowTest.java @@ -19,6 +19,7 @@ import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.MoreOAuthParameters; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; @@ -81,11 +82,18 @@ public void testGetSourceConsentUrl() throws IOException, InterruptedException, @Test public void testCompleteSourceAuth() throws IOException, InterruptedException, ConfigNotFoundException { - final Map expectedParams = Map.of("key", "test_client_id", "token", "test_token"); + final Map expectedParams = Map.of( + "key", "test_client_id", + "token", "test_token", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); final Map queryParams = Map.of("oauth_token", "token", "oauth_verifier", "verifier"); - final Map returnedParams = + final Map actualParams = trelloOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(returnedParams, expectedParams); + assertEquals(actualParams, expectedParams); + assertEquals(expectedParams.size(), actualParams.size(), + String.format("Expected %s values but got %s", expectedParams.size(), actualParams)); + expectedParams.forEach((key, value) -> assertEquals(value, actualParams.get(key))); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookMarketingOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookMarketingOAuthFlowTest.java new file mode 100644 index 0000000000000..b0b878c6b3121 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookMarketingOAuthFlowTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows.facebook; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; +import java.util.List; +import java.util.Map; + +public class FacebookMarketingOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new FacebookMarketingOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://www.facebook.com/v12.0/dialog/oauth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scope=ads_management%2Cads_read%2Cread_insights"; + } + + @Override + protected List getExpectedOutputPath() { + return List.of(); + } + + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); + } + + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlowTest.java deleted file mode 100644 index 306720140fa40..0000000000000 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlowTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.oauth.flows.facebook; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class FacebookOAuthFlowTest { - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private FacebookMarketingOAuthFlow facebookOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - - @BeforeEach - public void setup() throws JsonValidationException, IOException { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - facebookOAuthFlow = new FacebookMarketingOAuthFlow(configRepository, httpClient, FacebookOAuthFlowTest::getConstantState); - - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build())))); - } - - private static String getConstantState() { - return "state"; - } - - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { - final Map returnedCredentials = Map.of("access_token", "access_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - facebookOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); - } - - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("access_token", "access_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - facebookOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); - } - -} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlowTest.java new file mode 100644 index 0000000000000..66e1c56ded25b --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/FacebookPagesOAuthFlowTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows.facebook; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; +import java.util.List; +import java.util.Map; + +public class FacebookPagesOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new FacebookPagesOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://www.facebook.com/v12.0/dialog/oauth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scope=pages_manage_ads%2Cpages_manage_metadata%2Cpages_read_engagement%2Cpages_read_user_content"; + } + + @Override + protected List getExpectedOutputPath() { + return List.of(); + } + + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); + } + + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlowTest.java new file mode 100644 index 0000000000000..628049c3862a2 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/facebook/InstagramOAuthFlowTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows.facebook; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; +import java.util.List; +import java.util.Map; + +public class InstagramOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new InstagramOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://www.facebook.com/v12.0/dialog/oauth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state&scope=ads_management%2Cinstagram_basic%2Cinstagram_manage_insights%2Cread_insights"; + } + + @Override + protected List getExpectedOutputPath() { + return List.of(); + } + + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + @Override + protected JsonNode getOutputOAuthSpecification() { + return Jsons.jsonNode(Map.of("access_token", Map.of("type", "String"))); + } + + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java index 16e616a0996a1..35c7a37bb93ff 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java @@ -4,98 +4,19 @@ package io.airbyte.oauth.flows.google; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class GoogleAdsOAuthFlowTest extends BaseOAuthFlowTest { -public class GoogleAdsOAuthFlowTest { - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private GoogleAdsOAuthFlow googleAdsOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - - @BeforeEach - public void setup() throws JsonValidationException, IOException { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - googleAdsOAuthFlow = new GoogleAdsOAuthFlow(configRepository, httpClient, GoogleAdsOAuthFlowTest::getConstantState); - - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - } - - private static String getConstantState() { - return "state"; + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new GoogleAdsOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { - final String consentUrl = googleAdsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadwords&access_type=offline&state=state&include_granted_scopes=true&prompt=consent", - consentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAdsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); - } - - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAdsOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadwords&access_type=offline&state=state&include_granted_scopes=true&prompt=consent"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java index 12c6c6366e824..e639604b7fbdc 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java @@ -4,188 +4,19 @@ package io.airbyte.oauth.flows.google; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +public class GoogleAnalyticsOAuthFlowTest extends BaseOAuthFlowTest { -public class GoogleAnalyticsOAuthFlowTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(GoogleAnalyticsOAuthFlowTest.class); - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - private static final String REDIRECT_URL = "https://airbyte.io"; - private static final String EXPECTED_REDIRECT_URL = "https%3A%2F%2Fairbyte.io"; - private static final String EXPECTED_SCOPE = "https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fanalytics.readonly"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private GoogleAnalyticsOAuthFlow googleAnalyticsOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - - @BeforeEach - public void setup() { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - googleAnalyticsOAuthFlow = new GoogleAnalyticsOAuthFlow(configRepository, httpClient, GoogleAnalyticsOAuthFlowTest::getConstantState); - - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - } - - private static String getConstantState() { - return "state"; - } - - @Test - public void testGetConsentUrlEmptyOAuthParameters() { - assertThrows(ConfigNotFoundException.class, () -> googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - assertThrows(ConfigNotFoundException.class, () -> googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - } - - @Test - public void testGetConsentUrlIncompleteOAuthParameters() throws IOException, JsonValidationException { - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.emptyObject()))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.emptyObject()))); - assertThrows(IllegalArgumentException.class, () -> googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - assertThrows(IllegalArgumentException.class, () -> googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - } - - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException, JsonValidationException { - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", getClientId()) - .build()))))); - final String actualSourceUrl = googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - final String expectedSourceUrl = String.format( - "https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&redirect_uri=%s&response_type=code&scope=%s&access_type=offline&state=%s&include_granted_scopes=true&prompt=consent", - getClientId(), - EXPECTED_REDIRECT_URL, - EXPECTED_SCOPE, - getConstantState()); - LOGGER.info(expectedSourceUrl); - assertEquals(expectedSourceUrl, actualSourceUrl); - } - - @Test - public void testGetDestinationConsentUrl() throws IOException, ConfigNotFoundException, JsonValidationException { - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", getClientId()) - .build()))))); - // It would be better to make this comparison agnostic of the order of query params but the URI - // class' equals() method - // considers URLs with different qparam orders different URIs.. - final String actualDestinationUrl = googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL); - final String expectedDestinationUrl = String.format( - "https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&redirect_uri=%s&response_type=code&scope=%s&access_type=offline&state=%s&include_granted_scopes=true&prompt=consent", - getClientId(), - EXPECTED_REDIRECT_URL, - EXPECTED_SCOPE, - getConstantState()); - LOGGER.info(expectedDestinationUrl); - assertEquals(expectedDestinationUrl, actualDestinationUrl); - } - - @Test - public void testCompleteOAuthMissingCode() throws IOException, ConfigNotFoundException, JsonValidationException { - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", getClientId()) - .put("client_secret", "test_client_secret") - .build()))))); - final Map queryParams = Map.of(); - assertThrows(IOException.class, () -> googleAnalyticsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL)); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", getClientId()) - .put("client_secret", "test_client_secret") - .build()))))); - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAnalyticsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); - } - - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", getClientId()) - .put("client_secret", "test_client_secret") - .build()))))); - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAnalyticsOAuthFlow - .completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new GoogleAnalyticsOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - private String getClientId() throws IOException { - if (!Files.exists(CREDENTIALS_PATH)) { - return "test_client_id"; - } else { - final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); - final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); - return credentialsJson.get("credentials").get("client_id").asText(); - } + @Override + protected String getExpectedConsentUrl() { + return "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fanalytics.readonly&access_type=offline&state=state&include_granted_scopes=true&prompt=consent"; } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java index fa11895dfad72..136f4d993053a 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java @@ -4,101 +4,25 @@ package io.airbyte.oauth.flows.google; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class GoogleSearchConsoleOAuthFlowTest { - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private GoogleSearchConsoleOAuthFlow googleSearchConsoleOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - - @BeforeEach - public void setup() throws JsonValidationException, IOException { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - googleSearchConsoleOAuthFlow = new GoogleSearchConsoleOAuthFlow(configRepository, httpClient, GoogleSearchConsoleOAuthFlowTest::getConstantState); - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); +public class GoogleSearchConsoleOAuthFlowTest extends BaseOAuthFlowTest { - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("authorization", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("authorization", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new GoogleSearchConsoleOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - private static String getConstantState() { - return "state"; + @Override + protected String getExpectedConsentUrl() { + return "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fwebmasters.readonly&access_type=offline&state=state&include_granted_scopes=true&prompt=consent"; } - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { - final String consentUrl = googleSearchConsoleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fwebmasters.readonly&access_type=offline&state=state&include_granted_scopes=true&prompt=consent", - consentUrl); - } - - @Test - public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleSearchConsoleOAuthFlow - .completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), Jsons.serialize(actualQueryParams)); - } - - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleSearchConsoleOAuthFlow - .completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected List getExpectedOutputPath() { + return List.of("authorization"); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSheetsOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSheetsOAuthFlowTest.java index af74564d22edd..6ad6786b1024c 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSheetsOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSheetsOAuthFlowTest.java @@ -4,100 +4,19 @@ package io.airbyte.oauth.flows.google; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.flows.BaseOAuthFlowTest; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationOAuthParameter; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class GoogleSheetsOAuthFlowTest extends BaseOAuthFlowTest { -public class GoogleSheetsOAuthFlowTest { - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private HttpClient httpClient; - private ConfigRepository configRepository; - private GoogleSheetsOAuthFlow googleSheetsOAuthFlow; - - private UUID workspaceId; - private UUID definitionId; - - @BeforeEach - public void setup() throws JsonValidationException, IOException { - httpClient = mock(HttpClient.class); - configRepository = mock(ConfigRepository.class); - googleSheetsOAuthFlow = new GoogleSheetsOAuthFlow(configRepository, httpClient, GoogleSheetsOAuthFlowTest::getConstantState); - - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withDestinationDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - } - - private static String getConstantState() { - return "state"; - } - - @Test - public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundException { - final String consentUrl = googleSheetsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals( - "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fspreadsheets.readonly+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.readonly&access_type=offline&state=state&include_granted_scopes=true&prompt=consent", - consentUrl); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new GoogleSheetsOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @Test - public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleSheetsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); - } - - @Test - public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { - final Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - googleSheetsOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - - assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + @Override + protected String getExpectedConsentUrl() { + return "https://accounts.google.com/o/oauth2/v2/auth?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fspreadsheets.readonly+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.readonly&access_type=offline&state=state&include_granted_scopes=true&prompt=consent"; } } diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplier.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplier.java index ee9d58617591b..a1181d6611cf3 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplier.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplier.java @@ -4,33 +4,24 @@ package io.airbyte.scheduler.persistence.job_factory; -import static com.fasterxml.jackson.databind.node.JsonNodeType.OBJECT; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.MoreOAuthParameters; +import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.UUID; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class OAuthConfigSupplier { - private static final Logger LOGGER = LoggerFactory.getLogger(OAuthConfigSupplier.class); - - public static final String SECRET_MASK = "******"; final private ConfigRepository configRepository; private final boolean maskSecrets; private final TrackingClient trackingClient; @@ -41,6 +32,10 @@ public OAuthConfigSupplier(final ConfigRepository configRepository, final boolea this.trackingClient = trackingClient; } + public static boolean hasOAuthConfigSpecification(final ConnectorSpecification spec) { + return spec != null && spec.getAdvancedAuth() != null && spec.getAdvancedAuth().getOauthConfigSpecification() != null; + } + public JsonNode injectSourceOAuthParameters(final UUID sourceDefinitionId, final UUID workspaceId, final JsonNode sourceConnectorConfig) throws IOException { try { @@ -50,9 +45,14 @@ public JsonNode injectSourceOAuthParameters(final UUID sourceDefinitionId, final MoreOAuthParameters.getSourceOAuthParameter(configRepository.listSourceOAuthParam().stream(), workspaceId, sourceDefinitionId) .ifPresent( sourceOAuthParameter -> { - injectJsonNode((ObjectNode) sourceConnectorConfig, (ObjectNode) sourceOAuthParameter.getConfiguration()); - if (!maskSecrets) { - // when maskSecrets = true, no real oauth injections is happening + if (maskSecrets) { + // when maskSecrets = true, no real oauth injections is happening, only masked values + MoreOAuthParameters.mergeJsons( + (ObjectNode) sourceConnectorConfig, + (ObjectNode) sourceOAuthParameter.getConfiguration(), + MoreOAuthParameters.getSecretMask()); + } else { + MoreOAuthParameters.mergeJsons((ObjectNode) sourceConnectorConfig, (ObjectNode) sourceOAuthParameter.getConfiguration()); Exceptions.swallow(() -> trackingClient.track(workspaceId, "OAuth Injection - Backend", metadata)); } }); @@ -70,11 +70,15 @@ public JsonNode injectDestinationOAuthParameters(final UUID destinationDefinitio final ImmutableMap metadata = generateDestinationMetadata(destinationDefinitionId); MoreOAuthParameters.getDestinationOAuthParameter(configRepository.listDestinationOAuthParam().stream(), workspaceId, destinationDefinitionId) .ifPresent(destinationOAuthParameter -> { - injectJsonNode((ObjectNode) destinationConnectorConfig, - (ObjectNode) destinationOAuthParameter.getConfiguration()); - if (!maskSecrets) { - // when maskSecrets = true, no real oauth injections is happening - trackingClient.track(workspaceId, "OAuth Injection - Backend", metadata); + if (maskSecrets) { + // when maskSecrets = true, no real oauth injections is happening, only masked values + MoreOAuthParameters.mergeJsons( + (ObjectNode) destinationConnectorConfig, + (ObjectNode) destinationOAuthParameter.getConfiguration(), + MoreOAuthParameters.getSecretMask()); + } else { + MoreOAuthParameters.mergeJsons((ObjectNode) destinationConnectorConfig, (ObjectNode) destinationOAuthParameter.getConfiguration()); + Exceptions.swallow(() -> trackingClient.track(workspaceId, "OAuth Injection - Backend", metadata)); } }); return destinationConnectorConfig; @@ -83,41 +87,6 @@ public JsonNode injectDestinationOAuthParameters(final UUID destinationDefinitio } } - @VisibleForTesting - void injectJsonNode(final ObjectNode mainConfig, final ObjectNode fromConfig) { - // TODO this method might make sense to have as a general utility in Jsons - for (final String key : Jsons.keys(fromConfig)) { - if (fromConfig.get(key).getNodeType() == OBJECT) { - // nested objects are merged rather than overwrite the contents of the equivalent object in config - if (mainConfig.get(key) == null) { - injectJsonNode(mainConfig.putObject(key), (ObjectNode) fromConfig.get(key)); - } else if (mainConfig.get(key).getNodeType() == OBJECT) { - injectJsonNode((ObjectNode) mainConfig.get(key), (ObjectNode) fromConfig.get(key)); - } else { - throw new IllegalStateException("Can't merge an object node into a non-object node!"); - } - } else { - if (maskSecrets) { - // TODO secrets should be masked with the correct type - // https://github.com/airbytehq/airbyte/issues/5990 - // In the short-term this is not world-ending as all secret fields are currently strings - LOGGER.debug(String.format("Masking instance wide parameter %s in config", key)); - mainConfig.set(key, Jsons.jsonNode(SECRET_MASK)); - } else { - if (!mainConfig.has(key) || isSecretMask(mainConfig.get(key).asText())) { - LOGGER.debug(String.format("injecting instance wide parameter %s into config", key)); - mainConfig.set(key, fromConfig.get(key)); - } - } - } - - } - } - - private static boolean isSecretMask(final String input) { - return Strings.isNullOrEmpty(input.replaceAll("\\*", "")); - } - private ImmutableMap generateSourceMetadata(final UUID sourceDefinitionId) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(sourceDefinitionId); diff --git a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java index 40710bc43eb19..f3f5509e0acd5 100644 --- a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java +++ b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java @@ -14,7 +14,6 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.analytics.TrackingClient; @@ -23,13 +22,13 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.MoreOAuthParameters; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; public class OAuthConfigSupplierTest { @@ -140,7 +139,7 @@ void testInjectMaskedOAuthParameters() throws JsonValidationException, IOExcepti final JsonNode actualConfig = maskingSupplier.injectSourceOAuthParameters(sourceDefinitionId, workspaceId, Jsons.clone(config)); final ObjectNode expectedConfig = ((ObjectNode) Jsons.clone(config)); for (final String key : oauthParameters.keySet()) { - expectedConfig.set(key, Jsons.jsonNode(OAuthConfigSupplier.SECRET_MASK)); + expectedConfig.set(key, MoreOAuthParameters.getSecretMask()); } assertEquals(expectedConfig, actualConfig); assertNoTracking(); @@ -160,105 +159,6 @@ private Map generateOAuthParameters() { .build(); } - private void maskAllValues(final ObjectNode node) { - for (final String key : Jsons.keys(node)) { - if (node.get(key).getNodeType() == JsonNodeType.OBJECT) { - maskAllValues((ObjectNode) node.get(key)); - } else { - node.set(key, Jsons.jsonNode(OAuthConfigSupplier.SECRET_MASK)); - } - } - } - - @Test - void testInjectUnnestedNode_Masked() { - final OAuthConfigSupplier supplier = new OAuthConfigSupplier(configRepository, true, trackingClient); - final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); - final ObjectNode maskedOauthParams = Jsons.clone(oauthParams); - maskAllValues(maskedOauthParams); - final ObjectNode actual = generateJsonConfig(); - final ObjectNode expected = Jsons.clone(actual); - expected.setAll(maskedOauthParams); - - supplier.injectJsonNode(actual, oauthParams); - assertEquals(expected, actual); - assertNoTracking(); - } - - @Test - void testInjectUnnestedNode_Unmasked() { - final OAuthConfigSupplier supplier = new OAuthConfigSupplier(configRepository, false, trackingClient); - final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); - - final ObjectNode actual = generateJsonConfig(); - final ObjectNode expected = Jsons.clone(actual); - expected.setAll(oauthParams); - - supplier.injectJsonNode(actual, oauthParams); - - assertEquals(expected, actual); - assertNoTracking(); - } - - @Test - void testInjectNewNestedNode_Masked() { - final OAuthConfigSupplier supplier = new OAuthConfigSupplier(configRepository, true, trackingClient); - final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); - final ObjectNode maskedOauthParams = Jsons.clone(oauthParams); - maskAllValues(maskedOauthParams); - final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() - .put("oauth_credentials", oauthParams) - .build()); - - // nested node does not exist in actual object - final ObjectNode actual = generateJsonConfig(); - final ObjectNode expected = Jsons.clone(actual); - expected.putObject("oauth_credentials").setAll(maskedOauthParams); - - supplier.injectJsonNode(actual, nestedConfig); - assertEquals(expected, actual); - assertNoTracking(); - } - - @Test - @DisplayName("A nested config should be inserted with the same nesting structure") - void testInjectNewNestedNode_Unmasked() { - final OAuthConfigSupplier supplier = new OAuthConfigSupplier(configRepository, false, trackingClient); - final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); - final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() - .put("oauth_credentials", oauthParams) - .build()); - - // nested node does not exist in actual object - final ObjectNode actual = generateJsonConfig(); - final ObjectNode expected = Jsons.clone(actual); - expected.putObject("oauth_credentials").setAll(oauthParams); - - supplier.injectJsonNode(actual, nestedConfig); - assertEquals(expected, actual); - assertNoTracking(); - } - - @Test - @DisplayName("A nested node which partially exists in the main config should be merged into the main config, not overwrite the whole nested object") - void testInjectedPartiallyExistingNestedNode_Unmasked() { - final OAuthConfigSupplier supplier = new OAuthConfigSupplier(configRepository, false, trackingClient); - final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); - final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() - .put("oauth_credentials", oauthParams) - .build()); - - // nested node partially exists in actual object - final ObjectNode actual = generateJsonConfig(); - actual.putObject("oauth_credentials").put("irrelevant_field", "_"); - final ObjectNode expected = Jsons.clone(actual); - ((ObjectNode) expected.get("oauth_credentials")).setAll(oauthParams); - - supplier.injectJsonNode(actual, nestedConfig); - assertEquals(expected, actual); - assertNoTracking(); - } - private void assertNoTracking() { verify(trackingClient, times(0)).track(any(), anyString(), anyMap()); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index f49ebfea13870..7bd64e60610b3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -112,8 +112,6 @@ import io.airbyte.server.handlers.SourceDefinitionsHandler; import io.airbyte.server.handlers.SourceHandler; import io.airbyte.server.handlers.WebBackendConnectionsHandler; -import io.airbyte.server.handlers.WebBackendDestinationsHandler; -import io.airbyte.server.handlers.WebBackendSourcesHandler; import io.airbyte.server.handlers.WorkspacesHandler; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; @@ -137,8 +135,6 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { private final SchedulerHandler schedulerHandler; private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; - private final WebBackendSourcesHandler webBackendSourcesHandler; - private final WebBackendDestinationsHandler webBackendDestinationsHandler; private final HealthCheckHandler healthCheckHandler; private final ArchiveHandler archiveHandler; private final LogsHandler logsHandler; @@ -193,7 +189,7 @@ public ConfigurationApi(final ConfigRepository configRepository, sourceHandler = new SourceHandler(configRepository, schemaValidator, specFetcher, connectionsHandler); workspacesHandler = new WorkspacesHandler(configRepository, connectionsHandler, destinationHandler, sourceHandler); jobHistoryHandler = new JobHistoryHandler(jobPersistence, workerEnvironment, logConfigs); - oAuthHandler = new OAuthHandler(configRepository, httpClient, trackingClient); + oAuthHandler = new OAuthHandler(configRepository, httpClient, trackingClient, specFetcher); webBackendConnectionsHandler = new WebBackendConnectionsHandler( connectionsHandler, sourceHandler, @@ -201,8 +197,6 @@ public ConfigurationApi(final ConfigRepository configRepository, jobHistoryHandler, schedulerHandler, operationsHandler); - webBackendSourcesHandler = new WebBackendSourcesHandler(sourceHandler, configRepository, trackingClient); - webBackendDestinationsHandler = new WebBackendDestinationsHandler(destinationHandler, configRepository, trackingClient); healthCheckHandler = new HealthCheckHandler(configRepository); archiveHandler = new ArchiveHandler( airbyteVersion, @@ -633,16 +627,6 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne return execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); } - @Override - public SourceRead webBackendCreateSource(final SourceCreate sourceCreate) { - return execute(() -> webBackendSourcesHandler.webBackendCreateSource(sourceCreate)); - } - - @Override - public DestinationRead webBackendCreateDestination(final DestinationCreate destinationCreate) { - return execute(() -> webBackendDestinationsHandler.webBackendCreateDestination(destinationCreate)); - } - // ARCHIVES @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java index c4e888f9e31c8..25095de760ef3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java @@ -4,9 +4,14 @@ package io.airbyte.server.converters; +import io.airbyte.api.model.AdvancedAuth; +import io.airbyte.api.model.AdvancedAuth.AuthFlowTypeEnum; import io.airbyte.api.model.AuthSpecification; import io.airbyte.api.model.OAuth2Specification; +import io.airbyte.api.model.OAuthConfigSpecification; +import io.airbyte.commons.enums.Enums; import io.airbyte.protocol.models.ConnectorSpecification; +import java.util.List; import java.util.Optional; public class OauthModelConverter { @@ -25,8 +30,29 @@ public static Optional getAuthSpec(final ConnectorSpecificati .oauthFlowInitParameters(incomingAuthSpec.getOauth2Specification().getOauthFlowInitParameters()) .oauthFlowOutputParameters(incomingAuthSpec.getOauth2Specification().getOauthFlowOutputParameters())); } + return Optional.of(authSpecification); + } - return Optional.ofNullable(authSpecification); + public static Optional getAdvancedAuth(final ConnectorSpecification spec) { + if (spec.getAdvancedAuth() == null) { + return Optional.empty(); + } + final io.airbyte.protocol.models.AdvancedAuth incomingAdvancedAuth = spec.getAdvancedAuth(); + final AdvancedAuth advancedAuth = new AdvancedAuth(); + if (List.of(io.airbyte.protocol.models.AdvancedAuth.AuthFlowType.OAUTH_1_0, io.airbyte.protocol.models.AdvancedAuth.AuthFlowType.OAUTH_2_0) + .contains(incomingAdvancedAuth.getAuthFlowType())) { + final io.airbyte.protocol.models.OAuthConfigSpecification incomingOAuthConfigSpecification = incomingAdvancedAuth.getOauthConfigSpecification(); + advancedAuth + .authFlowType(Enums.convertTo(incomingAdvancedAuth.getAuthFlowType(), AuthFlowTypeEnum.class)) + .predicateKey(incomingAdvancedAuth.getPredicateKey()) + .predicateValue(incomingAdvancedAuth.getPredicateValue()) + .oauthConfigSpecification(new OAuthConfigSpecification() + .oauthUserInputFromConnectorConfigSpecification(incomingOAuthConfigSpecification.getOauthUserInputFromConnectorConfigSpecification()) + .completeOAuthOutputSpecification(incomingOAuthConfigSpecification.getCompleteOauthOutputSpecification()) + .completeOAuthServerInputSpecification(incomingOAuthConfigSpecification.getCompleteOauthServerInputSpecification()) + .completeOAuthServerOutputSpecification(incomingOAuthConfigSpecification.getCompleteOauthServerOutputSpecification())); + } + return Optional.of(advancedAuth); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index 02b4ab9d538da..225afd29f6ea7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -22,7 +22,10 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.OAuthFlowImplementation; import io.airbyte.oauth.OAuthImplementationFactory; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.scheduler.persistence.job_factory.OAuthConfigSupplier; import io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata; +import io.airbyte.server.converters.SpecFetcher; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.http.HttpClient; @@ -38,11 +41,16 @@ public class OAuthHandler { private final ConfigRepository configRepository; private final OAuthImplementationFactory oAuthImplementationFactory; private final TrackingClient trackingClient; + private final SpecFetcher specFetcher; - public OAuthHandler(final ConfigRepository configRepository, final HttpClient httpClient, final TrackingClient trackingClient) { + public OAuthHandler(final ConfigRepository configRepository, + final HttpClient httpClient, + final TrackingClient trackingClient, + final SpecFetcher specFetcher) { this.configRepository = configRepository; this.oAuthImplementationFactory = new OAuthImplementationFactory(configRepository, httpClient); this.trackingClient = trackingClient; + this.specFetcher = specFetcher; } public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest sourceDefinitionIdRequestBody) @@ -86,14 +94,26 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent public Map completeSourceOAuth(final CompleteSourceOauthRequest oauthSourceRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(oauthSourceRequestBody.getSourceDefinitionId()); - final OAuthFlowImplementation oAuthFlowImplementation = - oAuthImplementationFactory.create(sourceDefinition); + final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(sourceDefinition); + final ConnectorSpecification spec = specFetcher.getSpec(sourceDefinition); final ImmutableMap metadata = generateSourceMetadata(oauthSourceRequestBody.getSourceDefinitionId()); - final Map result = oAuthFlowImplementation.completeSourceOAuth( - oauthSourceRequestBody.getWorkspaceId(), - oauthSourceRequestBody.getSourceDefinitionId(), - oauthSourceRequestBody.getQueryParams(), - oauthSourceRequestBody.getRedirectUrl()); + final Map result; + if (OAuthConfigSupplier.hasOAuthConfigSpecification(spec)) { + result = oAuthFlowImplementation.completeSourceOAuth( + oauthSourceRequestBody.getWorkspaceId(), + oauthSourceRequestBody.getSourceDefinitionId(), + oauthSourceRequestBody.getQueryParams(), + oauthSourceRequestBody.getRedirectUrl(), + oauthSourceRequestBody.getoAuthInputConfiguration(), + spec.getAdvancedAuth().getOauthConfigSpecification()); + } else { + // deprecated but this path is kept for connectors that don't define OAuth Spec yet + result = oAuthFlowImplementation.completeSourceOAuth( + oauthSourceRequestBody.getWorkspaceId(), + oauthSourceRequestBody.getSourceDefinitionId(), + oauthSourceRequestBody.getQueryParams(), + oauthSourceRequestBody.getRedirectUrl()); + } try { trackingClient.track(oauthSourceRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { @@ -106,14 +126,26 @@ public Map completeDestinationOAuth(final CompleteDestinationOAu throws JsonValidationException, ConfigNotFoundException, IOException { final StandardDestinationDefinition destinationDefinition = configRepository.getStandardDestinationDefinition(oauthDestinationRequestBody.getDestinationDefinitionId()); - final OAuthFlowImplementation oAuthFlowImplementation = - oAuthImplementationFactory.create(destinationDefinition); + final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(destinationDefinition); + final ConnectorSpecification spec = specFetcher.getSpec(destinationDefinition); final ImmutableMap metadata = generateDestinationMetadata(oauthDestinationRequestBody.getDestinationDefinitionId()); - final Map result = oAuthFlowImplementation.completeDestinationOAuth( - oauthDestinationRequestBody.getWorkspaceId(), - oauthDestinationRequestBody.getDestinationDefinitionId(), - oauthDestinationRequestBody.getQueryParams(), - oauthDestinationRequestBody.getRedirectUrl()); + final Map result; + if (OAuthConfigSupplier.hasOAuthConfigSpecification(spec)) { + result = oAuthFlowImplementation.completeDestinationOAuth( + oauthDestinationRequestBody.getWorkspaceId(), + oauthDestinationRequestBody.getDestinationDefinitionId(), + oauthDestinationRequestBody.getQueryParams(), + oauthDestinationRequestBody.getRedirectUrl(), + oauthDestinationRequestBody.getoAuthInputConfiguration(), + spec.getAdvancedAuth().getOauthConfigSpecification()); + } else { + // deprecated but this path is kept for connectors that don't define OAuth Spec yet + result = oAuthFlowImplementation.completeDestinationOAuth( + oauthDestinationRequestBody.getWorkspaceId(), + oauthDestinationRequestBody.getDestinationDefinitionId(), + oauthDestinationRequestBody.getQueryParams(), + oauthDestinationRequestBody.getRedirectUrl()); + } try { trackingClient.track(oauthDestinationRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 378a793282f22..3d972fb2fe28d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import io.airbyte.api.model.AdvancedAuth; import io.airbyte.api.model.AuthSpecification; import io.airbyte.api.model.CheckConnectionRead; import io.airbyte.api.model.CheckConnectionRead.StatusEnum; @@ -264,9 +265,10 @@ public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final .sourceDefinitionId(sourceDefinitionId); final Optional authSpec = OauthModelConverter.getAuthSpec(spec); - if (authSpec.isPresent()) { - specRead.setAuthSpecification(authSpec.get()); - } + authSpec.ifPresent(specRead::setAuthSpecification); + + final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); + advancedAuth.ifPresent(specRead::setAdvancedAuth); return specRead; } @@ -289,9 +291,10 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification( .destinationDefinitionId(destinationDefinitionId); final Optional authSpec = OauthModelConverter.getAuthSpec(spec); - if (authSpec.isPresent()) { - specRead.setAuthSpecification(authSpec.get()); - } + authSpec.ifPresent(specRead::setAuthSpecification); + + final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); + advancedAuth.ifPresent(specRead::setAdvancedAuth); return specRead; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendDestinationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendDestinationsHandler.java deleted file mode 100644 index a13c0b42eff10..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendDestinationsHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.handlers; - -import io.airbyte.analytics.TrackingClient; -import io.airbyte.api.model.DestinationCreate; -import io.airbyte.api.model.DestinationRead; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.scheduler.persistence.job_factory.OAuthConfigSupplier; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; - -public class WebBackendDestinationsHandler { - - private final DestinationHandler destinationHandler; - private final OAuthConfigSupplier oAuthConfigSupplier; - - public WebBackendDestinationsHandler(final DestinationHandler destinationHandler, - final ConfigRepository configRepository, - final TrackingClient trackingClient) { - this.destinationHandler = destinationHandler; - oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, true, trackingClient); - } - - public DestinationRead webBackendCreateDestination(final DestinationCreate destinationCreate) - throws JsonValidationException, ConfigNotFoundException, IOException { - destinationCreate.connectionConfiguration( - oAuthConfigSupplier.injectDestinationOAuthParameters( - destinationCreate.getDestinationDefinitionId(), - destinationCreate.getWorkspaceId(), - destinationCreate.getConnectionConfiguration())); - return destinationHandler.createDestination(destinationCreate); - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendSourcesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendSourcesHandler.java deleted file mode 100644 index ed1a94703341b..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendSourcesHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.handlers; - -import io.airbyte.analytics.TrackingClient; -import io.airbyte.api.model.SourceCreate; -import io.airbyte.api.model.SourceRead; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.scheduler.persistence.job_factory.OAuthConfigSupplier; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; - -public class WebBackendSourcesHandler { - - private final SourceHandler sourceHandler; - private final OAuthConfigSupplier oAuthConfigSupplier; - - public WebBackendSourcesHandler(final SourceHandler sourceHandler, final ConfigRepository configRepository, final TrackingClient trackingClient) { - this.sourceHandler = sourceHandler; - oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, true, trackingClient); - } - - public SourceRead webBackendCreateSource(final SourceCreate sourceCreate) throws JsonValidationException, ConfigNotFoundException, IOException { - sourceCreate.connectionConfiguration( - oAuthConfigSupplier.injectSourceOAuthParameters( - sourceCreate.getSourceDefinitionId(), - sourceCreate.getWorkspaceId(), - sourceCreate.getConnectionConfiguration())); - return sourceHandler.createSource(sourceCreate); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java index 67c74aea83179..df0aa82975905 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @@ -15,6 +15,7 @@ import io.airbyte.config.DestinationOAuthParameter; import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.server.converters.SpecFetcher; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.http.HttpClient; @@ -30,17 +31,19 @@ class OAuthHandlerTest { - ConfigRepository configRepository; - OAuthHandler handler; - TrackingClient trackingClient; + private ConfigRepository configRepository; + private OAuthHandler handler; + private TrackingClient trackingClient; private HttpClient httpClient; + private SpecFetcher specFetcher; @BeforeEach public void init() { configRepository = Mockito.mock(ConfigRepository.class); trackingClient = mock(TrackingClient.class); httpClient = Mockito.mock(HttpClient.class); - handler = new OAuthHandler(configRepository, httpClient, trackingClient); + specFetcher = mock(SpecFetcher.class); + handler = new OAuthHandler(configRepository, httpClient, trackingClient, specFetcher); } @Test diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourcesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourcesHandlerTest.java deleted file mode 100644 index f8a8356f22c9e..0000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourcesHandlerTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.handlers; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.analytics.TrackingClient; -import io.airbyte.api.model.SourceCreate; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.scheduler.persistence.job_factory.OAuthConfigSupplier; -import io.airbyte.server.helpers.SourceHelpers; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class WebBackendSourcesHandlerTest { - - private SourceHandler sourceHandler; - private ConfigRepository configRepository; - private WebBackendSourcesHandler webBackendSourcesHandler; - private TrackingClient trackingClient; - - @BeforeEach - public void setup() throws JsonValidationException, ConfigNotFoundException, IOException { - sourceHandler = mock(SourceHandler.class); - configRepository = mock(ConfigRepository.class); - trackingClient = mock(TrackingClient.class); - webBackendSourcesHandler = new WebBackendSourcesHandler(sourceHandler, configRepository, trackingClient); - when(configRepository.getStandardSourceDefinition(any())).thenReturn(new StandardSourceDefinition() - .withSourceDefinitionId(UUID.randomUUID()) - .withName("test") - .withDockerImageTag("dev")); - } - - @Test - public void testWebBackendCreateSourceEmptyOAuthParameters() throws JsonValidationException, ConfigNotFoundException, IOException { - final UUID sourceDefinitionId = UUID.randomUUID(); - final SourceConnection sourceConnection = SourceHelpers.generateSource(sourceDefinitionId); - final SourceCreate sourceCreate = new SourceCreate() - .name(sourceConnection.getName()) - .workspaceId(sourceConnection.getWorkspaceId()) - .sourceDefinitionId(sourceDefinitionId) - .connectionConfiguration(sourceConnection.getConfiguration()); - webBackendSourcesHandler.webBackendCreateSource(Jsons.clone(sourceCreate)); - verify(sourceHandler).createSource(sourceCreate); - assertNoTracking(); - } - - @Test - public void testWebBackendCreateSourceWithMaskedOAuthParameters() throws JsonValidationException, ConfigNotFoundException, IOException { - final UUID sourceDefinitionId = UUID.randomUUID(); - final SourceConnection sourceConnection = SourceHelpers.generateSource(sourceDefinitionId); - final SourceCreate sourceCreate = new SourceCreate() - .name(sourceConnection.getName()) - .workspaceId(sourceConnection.getWorkspaceId()) - .sourceDefinitionId(sourceDefinitionId) - .connectionConfiguration(sourceConnection.getConfiguration()); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of( - new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(sourceDefinitionId) - .withWorkspaceId(null) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() - .put("api_secret", "mysecret") - .put("api_client", UUID.randomUUID().toString()) - .build())))); - final SourceCreate expectedSourceCreate = Jsons.clone(sourceCreate); - ((ObjectNode) expectedSourceCreate.getConnectionConfiguration()) - .put("api_secret", OAuthConfigSupplier.SECRET_MASK) - .put("api_client", OAuthConfigSupplier.SECRET_MASK); - webBackendSourcesHandler.webBackendCreateSource(Jsons.clone(sourceCreate)); - verify(sourceHandler).createSource(expectedSourceCreate); - assertNoTracking(); - } - - private void assertNoTracking() { - verify(trackingClient, times(0)).track(any(), anyString(), anyMap()); - } - -} diff --git a/airbyte-webapp/src/core/resources/Destination.tsx b/airbyte-webapp/src/core/resources/Destination.tsx index 3db297f50b758..0041cc65b7c73 100644 --- a/airbyte-webapp/src/core/resources/Destination.tsx +++ b/airbyte-webapp/src/core/resources/Destination.tsx @@ -44,6 +44,7 @@ export class DestinationResource extends BaseResource implements Destination { }; } + // TODO: remove? static recreateShape( this: T ): MutateShape> { @@ -74,11 +75,7 @@ export class DestinationResource extends BaseResource implements Destination { _: Readonly>, body: Readonly> ): Promise => - await this.fetch( - "post", - `${super.rootUrl()}web_backend/destinations/create`, - body - ), + await this.fetch("post", `${super.rootUrl()}destinations/create`, body), }; } } diff --git a/airbyte-webapp/src/core/resources/Source.tsx b/airbyte-webapp/src/core/resources/Source.tsx index 9a64e9e9a75e4..4af4b0b1e74ef 100644 --- a/airbyte-webapp/src/core/resources/Source.tsx +++ b/airbyte-webapp/src/core/resources/Source.tsx @@ -45,6 +45,7 @@ export class SourceResource extends BaseResource implements Source { // TODO: fix detailShape here as it is actually createShape // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types + // TODO: remove? static recreateShape(this: T) { return { ...super.detailShape(), @@ -73,11 +74,7 @@ export class SourceResource extends BaseResource implements Source { _: Readonly>, body: Readonly> ): Promise => - await this.fetch( - "post", - `${super.rootUrl()}web_backend/sources/create`, - body - ), + await this.fetch("post", `${super.rootUrl()}sources/create`, body), }; } } diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 3a93ccfe5b888..fd262a71dd6d8 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -339,8 +339,6 @@

SourceDefinitionSpecificationWebBackend

  • post /v1/web_backend/connections/create
    • -
  • post /v1/web_backend/destinations/create
    • -
  • post /v1/web_backend/sources/create
  • post /v1/web_backend/connections/get
  • post /v1/web_backend/connections/list
  • post /v1/web_backend/connections/search
    • @@ -2534,7 +2532,7 @@

    Example data

    "predicateValue" : "predicateValue", "oauthConfigSpecification" : { }, "predicateKey" : [ "predicateKey", "predicateKey" ], - "authFlowType" : "oauth1.0" + "authFlowType" : "oauth2.0" }, "authSpecification" : { "auth_type" : "oauth2.0", @@ -4966,7 +4964,7 @@

    Example data

    "predicateValue" : "predicateValue", "oauthConfigSpecification" : { }, "predicateKey" : [ "predicateKey", "predicateKey" ], - "authFlowType" : "oauth1.0" + "authFlowType" : "oauth2.0" }, "authSpecification" : { "auth_type" : "oauth2.0", @@ -5165,130 +5163,6 @@

    422

    InvalidInputExceptionInfo
    -
    -
    - Up - 
    post /v1/web_backend/destinations/create
    -
    Create a destination (webBackendCreateDestination)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/json
      • -
    - -

    Request body

    -
    -
    DestinationCreate DestinationCreate (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - DestinationRead - -
    - - - -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "connectionConfiguration" : {
-    "user" : "charles"
-  },
-  "destinationName" : "destinationName",
-  "name" : "name",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-}
    - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/json
      • -
    - -

    Responses

    -

    200

    - Successful operation - DestinationRead -

    422

    - Input failed validation - InvalidInputExceptionInfo -
    -
    -
    -
    - Up - 
    post /v1/web_backend/sources/create
    -
    Create a source (webBackendCreateSource)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/json
      • -
    - -

    Request body

    -
    -
    SourceCreate SourceCreate (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - SourceRead - -
    - - - -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "connectionConfiguration" : {
-    "user" : "charles"
-  },
-  "name" : "name",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "sourceName" : "sourceName",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-}
    - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/json
      • -
    - -

    Responses

    -

    200

    - Successful operation - SourceRead -

    422

    - Input failed validation - InvalidInputExceptionInfo -
    -
    Up @@ -6716,7 +6590,7 @@

    AdvancedAuth -
    authFlowType (optional)
    String
    Enum:
    -
    oauth1.0
    oauth2.0
    +
    oauth2.0
    oauth1.0
    predicateKey (optional)
    array[String] Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.
    predicateValue (optional)
    String Value of the predicate_key fields for the advanced auth to be applicable.
    oauthConfigSpecification (optional)
    OAuthConfigSpecification