CDK: add option to source to skip config validation on read/discover (a…

…irbytehq#6978)

* Add connector check_config_against_spec parameter

* Bump CDK version.
Add changelog record.

* Always enable spec check on check cmd

* Update airbyte-cdk/python/CHANGELOG.md

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.1.28
+Added `check_config_against_spec` parameter to `Connector` abstract class 
+to allow skipping validating the input config against the spec for non-`check` calls
+
 ## 0.1.27
 Improving unit test for logger
 

airbyte-cdk/python/airbyte_cdk/connector.py

@@ -25,6 +25,9 @@ def __init__(self, spec_string):
 
 
 class Connector(ABC):
+    # configure whether the `check_config_against_spec_or_exit()` needs to be called
+    check_config_against_spec: bool = True
+
     # can be overridden to change an input config
     def configure(self, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]:
         """

airbyte-cdk/python/airbyte_cdk/destinations/destination.py

@@ -91,7 +91,8 @@ def run_cmd(self, parsed_args: argparse.Namespace) -> Iterable[AirbyteMessage]:
             yield AirbyteMessage(type=Type.SPEC, spec=spec)
             return
         config = self.read_config(config_path=parsed_args.config)
-        check_config_against_spec_or_exit(config, spec, self.logger)
+        if self.check_config_against_spec or cmd == "check":
+            check_config_against_spec_or_exit(config, spec, self.logger)
 
         if cmd == "check":
             yield self._run_check(config=config)

airbyte-cdk/python/airbyte_cdk/entrypoint.py

@@ -72,7 +72,8 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
                 # Remove internal flags from config before validating so
                 # jsonschema's additionalProperties flag wont fail the validation
                 config, internal_config = split_config(config)
-                check_config_against_spec_or_exit(config, source_spec, self.logger)
+                if self.source.check_config_against_spec or cmd == "check":
+                    check_config_against_spec_or_exit(config, source_spec, self.logger)
                 # Put internal flags back to config dict
                 config.update(internal_config.dict())
 

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.27",
+    version="0.1.28",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",