Python Developer Experience (airbytehq#684)

* add symlinks for intellij includes

* add python support for exchangerate

* add python support for other modules

* get working across all projects

* add pydantic deps

* revert testing code

* mypy

* use python 3.7 in github actions

* remove unused envs plugin

* increase line length

* increase line length to 140 (run formatting)

* don't ignore most init.py

* does this happen to work on github actions?

* try to fix generation

* try to have GA chown

* exclude re-generation

.github/workflows/gradle.yml

@@ -36,6 +36,10 @@ jobs:
         with:
           node-version: '14.7'
 
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
       - name: Write Integration Test Credentials
         run: ./tools/bin/ci_credentials.sh
         env:
@@ -46,16 +50,15 @@ jobs:
       - name: Build
         run: ./gradlew --no-daemon build --scan
 
-
       - name: Ensure no file change
         run: git status --porcelain && test -z "$(git status --porcelain)"
 
       - name: Run Integration Tests
-        run: ./gradlew --no-daemon integrationTest --scan
+        run: ./gradlew --no-daemon integrationTest --scan -x generateProtocolClassFiles
 
       - name: Build Core Docker Images
         if: success() && github.ref == 'refs/heads/master'
-        run: ./gradlew --no-daemon composeBuild --scan
+        run: ./gradlew --no-daemon composeBuild --scan -x generateProtocolClassFiles
         env:
           GIT_REVISION: ${{ github.sha }}
 

.gitignore

@@ -10,3 +10,5 @@ secrets
 # Python
 *.egg-info
 __pycache__
+.venv
+.mypy_cache

airbyte-integrations/bases/base-python/airbyte_protocol/__init__.py

@@ -22,19 +22,41 @@
 SOFTWARE.
 """
 
-from .integration import *
+from .integration import AirbyteCheckResponse, AirbyteSpec, ConfigContainer, Destination, Integration, Source
 from .logger import AirbyteLogger
-from .models import AirbyteCatalog
-from .models import AirbyteConnectionStatus
-from .models import Status
-from .models import AirbyteLogMessage
-from .models import AirbyteMessage
-from .models import AirbyteRecordMessage
-from .models import AirbyteStateMessage
-from .models import AirbyteStream
-from .models import Type
-from .models import ConnectorSpecification
+from .models import (
+    AirbyteCatalog,
+    AirbyteConnectionStatus,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStateMessage,
+    AirbyteStream,
+    ConnectorSpecification,
+    Status,
+    Type,
+)
 
 # Must be the last one because the way we load the connector module creates a circular
 # dependency and models might not have been loaded yet
-from .entrypoint import AirbyteEntrypoint
+from .entrypoint import AirbyteEntrypoint  # noqa isort:skip
+
+__all__ = [
+    "AirbyteSpec",
+    "AirbyteCheckResponse",
+    "ConfigContainer",
+    "Integration",
+    "Source",
+    "Destination",
+    "AirbyteCatalog",
+    "AirbyteConnectionStatus",
+    "AirbyteLogMessage",
+    "AirbyteMessage",
+    "AirbyteRecordMessage",
+    "AirbyteStateMessage",
+    "AirbyteStream",
+    "ConnectorSpecification",
+    "Status",
+    "Type",
+    "AirbyteLogger",
+]

airbyte-integrations/bases/base-python/airbyte_protocol/entrypoint.py

@@ -27,14 +27,14 @@
 import os.path
 import sys
 import tempfile
-import json
+
+from airbyte_protocol import AirbyteMessage, Status, Type
 
 from .integration import ConfigContainer, Source
 from .logger import AirbyteLogger
-from airbyte_protocol import AirbyteMessage, AirbyteConnectionStatus, Type, Status
 
-impl_module = os.environ.get('AIRBYTE_IMPL_MODULE', Source.__module__)
-impl_class = os.environ.get('AIRBYTE_IMPL_PATH', Source.__name__)
+impl_module = os.environ.get("AIRBYTE_IMPL_MODULE", Source.__module__)
+impl_class = os.environ.get("AIRBYTE_IMPL_PATH", Source.__name__)
 module = importlib.import_module(impl_module)
 impl = getattr(module, impl_class)
 
@@ -49,35 +49,32 @@ def start(self, args):
         # set up parent parsers
         parent_parser = argparse.ArgumentParser(add_help=False)
         main_parser = argparse.ArgumentParser()
-        subparsers = main_parser.add_subparsers(title='commands', dest='command')
+        subparsers = main_parser.add_subparsers(title="commands", dest="command")
 
         # spec
         subparsers.add_parser("spec", help="outputs the json configuration specification", parents=[parent_parser])
 
         # check
-        check_parser = subparsers.add_parser("check", help="checks the config can be used to connect",
-                                             parents=[parent_parser])
-        required_check_parser = check_parser.add_argument_group('required named arguments')
-        required_check_parser.add_argument('--config', type=str, required=True,
-                                           help='path to the json configuration file')
+        check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", parents=[parent_parser])
+        required_check_parser = check_parser.add_argument_group("required named arguments")
+        required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
 
         # discover
-        discover_parser = subparsers.add_parser("discover", help="outputs a catalog describing the source's schema",
-                                                parents=[parent_parser])
-        required_discover_parser = discover_parser.add_argument_group('required named arguments')
-        required_discover_parser.add_argument('--config', type=str, required=True,
-                                              help='path to the json configuration file')
+        discover_parser = subparsers.add_parser(
+            "discover", help="outputs a catalog describing the source's schema", parents=[parent_parser]
+        )
+        required_discover_parser = discover_parser.add_argument_group("required named arguments")
+        required_discover_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
 
         # read
-        read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT",
-                                            parents=[parent_parser])
+        read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", parents=[parent_parser])
 
-        read_parser.add_argument('--state', type=str, required=False, help='path to the json-encoded state file')
-        required_read_parser = read_parser.add_argument_group('required named arguments')
-        required_read_parser.add_argument('--config', type=str, required=True,
-                                          help='path to the json configuration file')
-        required_read_parser.add_argument('--catalog', type=str, required=True,
-                                          help='path to the catalog used to determine which data to read')
+        read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file")
+        required_read_parser = read_parser.add_argument_group("required named arguments")
+        required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
+        required_read_parser.add_argument(
+            "--catalog", type=str, required=True, help="path to the catalog used to determine which data to read"
+        )
 
         # parse the args
         parsed_args = main_parser.parse_args(args)
@@ -91,21 +88,22 @@ def start(self, args):
 
         with tempfile.TemporaryDirectory() as temp_dir:
             if cmd == "spec":
-                message = AirbyteMessage(type='SPEC', spec=self.source.spec(logger))
+                message = AirbyteMessage(type="SPEC", spec=self.source.spec(logger))
                 print(message.json(exclude_unset=True))
                 sys.exit(0)
 
             raw_config = self.source.read_config(parsed_args.config)
 
-            rendered_config_path = os.path.join(temp_dir, 'config.json')
+            rendered_config_path = os.path.join(temp_dir, "config.json")
             rendered_config = self.source.transform_config(raw_config)
             self.source.write_config(rendered_config, rendered_config_path)
 
             config_container = ConfigContainer(
                 raw_config=raw_config,
                 rendered_config=rendered_config,
                 raw_config_path=parsed_args.config,
-                rendered_config_path=rendered_config_path)
+                rendered_config_path=rendered_config_path,
+            )
 
             if cmd == "check":
                 check_result = self.source.check(logger, config_container)

airbyte-integrations/bases/base-python/airbyte_protocol/integration.py

@@ -29,6 +29,7 @@
 
 from .models import AirbyteCatalog, AirbyteMessage, ConnectorSpecification
 
+
 class AirbyteSpec(object):
     @staticmethod
     def from_file(file):
@@ -59,11 +60,11 @@ def __init__(self):
         pass
 
     def spec(self, logger) -> ConnectorSpecification:
-        raw_spec = pkgutil.get_data(self.__class__.__module__.split('.')[0], 'spec.json')
+        raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json")
         return ConnectorSpecification.parse_obj(json.loads(raw_spec))
 
     def read_config(self, config_path):
-        with open(config_path, 'r') as file:
+        with open(config_path, "r") as file:
             contents = file.read()
         return json.loads(contents)
 
@@ -72,7 +73,7 @@ def transform_config(self, raw_config):
         return raw_config
 
     def write_config(self, config_object, path):
-        with open(path, 'w') as fh:
+        with open(path, "w") as fh:
             fh.write(json.dumps(config_object))
 
     def check(self, logger, config_container) -> AirbyteCheckResponse:

airbyte-integrations/bases/base-python/airbyte_protocol/models/airbyte_message.py

@@ -34,44 +34,44 @@
 
 
 class Type(Enum):
-    RECORD = 'RECORD'
-    STATE = 'STATE'
-    LOG = 'LOG'
-    SPEC = 'SPEC'
-    CONNECTION_STATUS = 'CONNECTION_STATUS'
-    CATALOG = 'CATALOG'
+    RECORD = "RECORD"
+    STATE = "STATE"
+    LOG = "LOG"
+    SPEC = "SPEC"
+    CONNECTION_STATUS = "CONNECTION_STATUS"
+    CATALOG = "CATALOG"
 
 
 class AirbyteRecordMessage(BaseModel):
-    stream: str = Field(..., description='the name of the stream for this record')
-    data: Dict[str, Any] = Field(..., description='the record data')
+    stream: str = Field(..., description="the name of the stream for this record")
+    data: Dict[str, Any] = Field(..., description="the record data")
     emitted_at: int = Field(
         ...,
-        description='when the data was emitted from the source. epoch in millisecond.',
+        description="when the data was emitted from the source. epoch in millisecond.",
     )
 
 
 class AirbyteStateMessage(BaseModel):
-    data: Dict[str, Any] = Field(..., description='the state data')
+    data: Dict[str, Any] = Field(..., description="the state data")
 
 
 class Level(Enum):
-    FATAL = 'FATAL'
-    ERROR = 'ERROR'
-    WARN = 'WARN'
-    INFO = 'INFO'
-    DEBUG = 'DEBUG'
-    TRACE = 'TRACE'
+    FATAL = "FATAL"
+    ERROR = "ERROR"
+    WARN = "WARN"
+    INFO = "INFO"
+    DEBUG = "DEBUG"
+    TRACE = "TRACE"
 
 
 class AirbyteLogMessage(BaseModel):
-    level: Level = Field(..., description='the type of logging')
-    message: str = Field(..., description='the log message')
+    level: Level = Field(..., description="the type of logging")
+    message: str = Field(..., description="the log message")
 
 
 class Status(Enum):
-    SUCCEEDED = 'SUCCEEDED'
-    FAILED = 'FAILED'
+    SUCCEEDED = "SUCCEEDED"
+    FAILED = "FAILED"
 
 
 class AirbyteConnectionStatus(BaseModel):
@@ -81,17 +81,15 @@ class AirbyteConnectionStatus(BaseModel):
 
 class AirbyteStream(BaseModel):
     name: str = Field(..., description="Stream's name.")
-    json_schema: Dict[str, Any] = Field(
-        ..., description='Stream schema using Json Schema specs.'
-    )
+    json_schema: Dict[str, Any] = Field(..., description="Stream schema using Json Schema specs.")
 
 
 class ConnectorSpecification(BaseModel):
     documentationUrl: Optional[AnyUrl] = None
     changelogUrl: Optional[AnyUrl] = None
     connectionSpecification: Dict[str, Any] = Field(
         ...,
-        description='ConnectorDefinition specific blob. Must be a valid JSON string.',
+        description="ConnectorDefinition specific blob. Must be a valid JSON string.",
     )
 
 
@@ -100,21 +98,19 @@ class AirbyteCatalog(BaseModel):
 
 
 class AirbyteMessage(BaseModel):
-    type: Type = Field(..., description='Message type')
+    type: Type = Field(..., description="Message type")
     log: Optional[AirbyteLogMessage] = Field(
         None,
-        description='log message: any kind of logging you want the platform to know about.',
+        description="log message: any kind of logging you want the platform to know about.",
     )
     spec: Optional[ConnectorSpecification] = None
     connectionStatus: Optional[AirbyteConnectionStatus] = None
     catalog: Optional[AirbyteCatalog] = Field(
         None,
-        description='log message: any kind of logging you want the platform to know about.',
-    )
-    record: Optional[AirbyteRecordMessage] = Field(
-        None, description='record message: the record'
+        description="log message: any kind of logging you want the platform to know about.",
     )
+    record: Optional[AirbyteRecordMessage] = Field(None, description="record message: the record")
     state: Optional[AirbyteStateMessage] = Field(
         None,
-        description='schema message: the state. Must be the last message produced. The platform uses this information',
+        description="schema message: the state. Must be the last message produced. The platform uses this information",
     )

airbyte-integrations/bases/base-python/build.gradle

@@ -1,10 +1,10 @@
+project.ext.pyModule = 'airbyte_protocol'
+apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
 apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
 
 task generateProtocolClassFiles(type: Exec) {
     environment 'ROOT_DIR', rootDir.absolutePath
     commandLine 'bin/generate-protocol-files.sh'
-
-    outputs.upToDateWhen { false }
     dependsOn ':airbyte-integrations:bases:base-python:code-generator:buildImage'
 }
 

airbyte-integrations/bases/base-python/setup.py

@@ -25,25 +25,15 @@
 import setuptools
 
 setuptools.setup(
-    name='airbyte-protocol',
-    description='Contains classes representing the schema of the Airbyte protocol.',
-    author='Airbyte',
-    author_email='contact@airbyte.io',
-    url='https://github.com/airbytehq/airbyte',
-
+    name="airbyte-protocol",
+    description="Contains classes representing the schema of the Airbyte protocol.",
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    url="https://github.com/airbytehq/airbyte",
     packages=setuptools.find_packages(),
-    package_data={
-        '': ['models/yaml/*.yaml']
-    },
-
-    install_requires=[
-        'PyYAML==5.3.1',
-        'pydantic==1.6.1'
-    ],
-
+    package_data={"": ["models/yaml/*.yaml"]},
+    install_requires=["PyYAML==5.3.1", "pydantic==1.6.1"],
     entry_points={
-        'console_scripts': [
-            'base-python=airbyte_protocol.entrypoint:main'
-        ],
-    }
+        "console_scripts": ["base-python=airbyte_protocol.entrypoint:main"],
+    },
 )

airbyte-integrations/bases/base-singer/airbyte_protocol

@@ -0,0 +1 @@
+../base-python/airbyte_protocol

airbyte-integrations/bases/base-singer/base_singer/__init__.py

@@ -22,5 +22,5 @@
 SOFTWARE.
 """
 
-from .singer_helpers import *
-from .source import *
+from .singer_helpers import *  # noqa
+from .source import *  # noqa