From b34d559df3d3d369396510adb5bdce7dd2dc2d24 Mon Sep 17 00:00:00 2001 From: Jared Rhizor Date: Fri, 23 Oct 2020 12:43:18 -0700 Subject: [PATCH] Python Developer Experience (#684) * add symlinks for intellij includes * add python support for exchangerate * add python support for other modules * get working across all projects * add pydantic deps * revert testing code * mypy * use python 3.7 in github actions * remove unused envs plugin * increase line length * increase line length to 140 (run formatting) * don't ignore most init.py * does this happen to work on github actions? * try to fix generation * try to have GA chown * exclude re-generation --- .github/workflows/gradle.yml | 9 ++- .gitignore | 2 + .../base-python/airbyte_protocol/__init__.py | 46 +++++++++++---- .../airbyte_protocol/entrypoint.py | 50 ++++++++-------- .../airbyte_protocol/integration.py | 7 ++- .../models/airbyte_message.py | 58 +++++++++---------- .../bases/base-python/build.gradle | 4 +- .../bases/base-python/setup.py | 28 +++------ .../bases/base-singer/airbyte_protocol | 1 + .../bases/base-singer/base_singer/__init__.py | 4 +- .../base-singer/base_singer/singer_helpers.py | 34 +++++------ .../bases/base-singer/base_singer/source.py | 6 +- .../bases/base-singer/build.gradle | 2 + .../bases/base-singer/setup.py | 14 ++--- .../python-source/airbyte_protocol | 1 + .../python-source/build.gradle | 2 + .../python-source/main_dev.py | 1 - .../python-source/requirements.txt | 2 +- .../python-source/setup.py | 18 +++--- .../template_python_source/__init__.py | 4 +- .../template_python_source/source.py | 27 +++------ .../singer-source/airbyte_protocol | 1 + .../singer-source/base_singer | 1 + .../singer-source/build.gradle | 2 + .../singer-source/main_dev.py | 2 +- .../singer-source/requirements.txt | 4 +- .../singer-source/setup.py | 25 ++++---- .../template_singer_source/__init__.py | 4 +- .../template_singer_source/source.py | 2 +- .../airbyte_protocol | 1 + .../base_singer | 1 + .../build.gradle | 2 + .../main_dev.py | 2 +- .../source-exchangeratesapi-singer/setup.py | 25 ++++---- .../__init__.py | 4 +- .../source_exchangeratesapi_singer/source.py | 5 +- .../source-postgres-singer/airbyte_protocol | 1 + .../source-postgres-singer/base_singer | 1 + .../source-postgres-singer/build.gradle | 3 + .../source-postgres-singer/main_dev.py | 2 +- .../postgres_singer_source/__init__.py | 4 +- .../postgres_singer_source/source.py | 20 +++---- .../source-postgres-singer/setup.py | 27 ++++----- .../source-stripe-singer/airbyte_protocol | 1 + .../source-stripe-singer/base_singer | 1 + .../source-stripe-singer/build.gradle | 2 + .../source-stripe-singer/main_dev.py | 2 +- .../connectors/source-stripe-singer/setup.py | 26 ++++----- .../source_stripe_singer/__init__.py | 3 +- .../source_stripe_singer/source.py | 2 +- build.gradle | 3 +- .../gradle/commons/integrations/python.gradle | 47 +++++++++++++++ tools/python/.flake8 | 9 +++ tools/python/.isort.cfg | 8 +++ tools/python/.mypy.ini | 34 +++++++++++ 55 files changed, 351 insertions(+), 246 deletions(-) create mode 120000 airbyte-integrations/bases/base-singer/airbyte_protocol create mode 120000 airbyte-integrations/connector-templates/python-source/airbyte_protocol create mode 120000 airbyte-integrations/connector-templates/singer-source/airbyte_protocol create mode 120000 airbyte-integrations/connector-templates/singer-source/base_singer create mode 120000 airbyte-integrations/connectors/source-exchangeratesapi-singer/airbyte_protocol create mode 120000 airbyte-integrations/connectors/source-exchangeratesapi-singer/base_singer create mode 120000 airbyte-integrations/connectors/source-postgres-singer/airbyte_protocol create mode 120000 airbyte-integrations/connectors/source-postgres-singer/base_singer create mode 120000 airbyte-integrations/connectors/source-stripe-singer/airbyte_protocol create mode 120000 airbyte-integrations/connectors/source-stripe-singer/base_singer create mode 100644 tools/gradle/commons/integrations/python.gradle create mode 100644 tools/python/.flake8 create mode 100644 tools/python/.isort.cfg create mode 100644 tools/python/.mypy.ini diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index a58eacb450c28..c1e7c25b9eafb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -36,6 +36,10 @@ jobs: with: node-version: '14.7' + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Write Integration Test Credentials run: ./tools/bin/ci_credentials.sh env: @@ -46,16 +50,15 @@ jobs: - name: Build run: ./gradlew --no-daemon build --scan - - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" - name: Run Integration Tests - run: ./gradlew --no-daemon integrationTest --scan + run: ./gradlew --no-daemon integrationTest --scan -x generateProtocolClassFiles - name: Build Core Docker Images if: success() && github.ref == 'refs/heads/master' - run: ./gradlew --no-daemon composeBuild --scan + run: ./gradlew --no-daemon composeBuild --scan -x generateProtocolClassFiles env: GIT_REVISION: ${{ github.sha }} diff --git a/.gitignore b/.gitignore index 6300868a0fd33..178425b5c6769 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ secrets # Python *.egg-info __pycache__ +.venv +.mypy_cache diff --git a/airbyte-integrations/bases/base-python/airbyte_protocol/__init__.py b/airbyte-integrations/bases/base-python/airbyte_protocol/__init__.py index 34ba008928485..2b0b9007fda98 100644 --- a/airbyte-integrations/bases/base-python/airbyte_protocol/__init__.py +++ b/airbyte-integrations/bases/base-python/airbyte_protocol/__init__.py @@ -22,19 +22,41 @@ SOFTWARE. """ -from .integration import * +from .integration import AirbyteCheckResponse, AirbyteSpec, ConfigContainer, Destination, Integration, Source from .logger import AirbyteLogger -from .models import AirbyteCatalog -from .models import AirbyteConnectionStatus -from .models import Status -from .models import AirbyteLogMessage -from .models import AirbyteMessage -from .models import AirbyteRecordMessage -from .models import AirbyteStateMessage -from .models import AirbyteStream -from .models import Type -from .models import ConnectorSpecification +from .models import ( + AirbyteCatalog, + AirbyteConnectionStatus, + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConnectorSpecification, + Status, + Type, +) # Must be the last one because the way we load the connector module creates a circular # dependency and models might not have been loaded yet -from .entrypoint import AirbyteEntrypoint +from .entrypoint import AirbyteEntrypoint # noqa isort:skip + +__all__ = [ + "AirbyteSpec", + "AirbyteCheckResponse", + "ConfigContainer", + "Integration", + "Source", + "Destination", + "AirbyteCatalog", + "AirbyteConnectionStatus", + "AirbyteLogMessage", + "AirbyteMessage", + "AirbyteRecordMessage", + "AirbyteStateMessage", + "AirbyteStream", + "ConnectorSpecification", + "Status", + "Type", + "AirbyteLogger", +] diff --git a/airbyte-integrations/bases/base-python/airbyte_protocol/entrypoint.py b/airbyte-integrations/bases/base-python/airbyte_protocol/entrypoint.py index 91c435e695e46..6d89efd64dfa3 100644 --- a/airbyte-integrations/bases/base-python/airbyte_protocol/entrypoint.py +++ b/airbyte-integrations/bases/base-python/airbyte_protocol/entrypoint.py @@ -27,14 +27,14 @@ import os.path import sys import tempfile -import json + +from airbyte_protocol import AirbyteMessage, Status, Type from .integration import ConfigContainer, Source from .logger import AirbyteLogger -from airbyte_protocol import AirbyteMessage, AirbyteConnectionStatus, Type, Status -impl_module = os.environ.get('AIRBYTE_IMPL_MODULE', Source.__module__) -impl_class = os.environ.get('AIRBYTE_IMPL_PATH', Source.__name__) +impl_module = os.environ.get("AIRBYTE_IMPL_MODULE", Source.__module__) +impl_class = os.environ.get("AIRBYTE_IMPL_PATH", Source.__name__) module = importlib.import_module(impl_module) impl = getattr(module, impl_class) @@ -49,35 +49,32 @@ def start(self, args): # set up parent parsers parent_parser = argparse.ArgumentParser(add_help=False) main_parser = argparse.ArgumentParser() - subparsers = main_parser.add_subparsers(title='commands', dest='command') + subparsers = main_parser.add_subparsers(title="commands", dest="command") # spec subparsers.add_parser("spec", help="outputs the json configuration specification", parents=[parent_parser]) # check - check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", - parents=[parent_parser]) - required_check_parser = check_parser.add_argument_group('required named arguments') - required_check_parser.add_argument('--config', type=str, required=True, - help='path to the json configuration file') + check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", parents=[parent_parser]) + required_check_parser = check_parser.add_argument_group("required named arguments") + required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") # discover - discover_parser = subparsers.add_parser("discover", help="outputs a catalog describing the source's schema", - parents=[parent_parser]) - required_discover_parser = discover_parser.add_argument_group('required named arguments') - required_discover_parser.add_argument('--config', type=str, required=True, - help='path to the json configuration file') + discover_parser = subparsers.add_parser( + "discover", help="outputs a catalog describing the source's schema", parents=[parent_parser] + ) + required_discover_parser = discover_parser.add_argument_group("required named arguments") + required_discover_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") # read - read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", - parents=[parent_parser]) + read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", parents=[parent_parser]) - read_parser.add_argument('--state', type=str, required=False, help='path to the json-encoded state file') - required_read_parser = read_parser.add_argument_group('required named arguments') - required_read_parser.add_argument('--config', type=str, required=True, - help='path to the json configuration file') - required_read_parser.add_argument('--catalog', type=str, required=True, - help='path to the catalog used to determine which data to read') + read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file") + required_read_parser = read_parser.add_argument_group("required named arguments") + required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") + required_read_parser.add_argument( + "--catalog", type=str, required=True, help="path to the catalog used to determine which data to read" + ) # parse the args parsed_args = main_parser.parse_args(args) @@ -91,13 +88,13 @@ def start(self, args): with tempfile.TemporaryDirectory() as temp_dir: if cmd == "spec": - message = AirbyteMessage(type='SPEC', spec=self.source.spec(logger)) + message = AirbyteMessage(type="SPEC", spec=self.source.spec(logger)) print(message.json(exclude_unset=True)) sys.exit(0) raw_config = self.source.read_config(parsed_args.config) - rendered_config_path = os.path.join(temp_dir, 'config.json') + rendered_config_path = os.path.join(temp_dir, "config.json") rendered_config = self.source.transform_config(raw_config) self.source.write_config(rendered_config, rendered_config_path) @@ -105,7 +102,8 @@ def start(self, args): raw_config=raw_config, rendered_config=rendered_config, raw_config_path=parsed_args.config, - rendered_config_path=rendered_config_path) + rendered_config_path=rendered_config_path, + ) if cmd == "check": check_result = self.source.check(logger, config_container) diff --git a/airbyte-integrations/bases/base-python/airbyte_protocol/integration.py b/airbyte-integrations/bases/base-python/airbyte_protocol/integration.py index a2e42715d8702..78b791c5bced1 100644 --- a/airbyte-integrations/bases/base-python/airbyte_protocol/integration.py +++ b/airbyte-integrations/bases/base-python/airbyte_protocol/integration.py @@ -29,6 +29,7 @@ from .models import AirbyteCatalog, AirbyteMessage, ConnectorSpecification + class AirbyteSpec(object): @staticmethod def from_file(file): @@ -59,11 +60,11 @@ def __init__(self): pass def spec(self, logger) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split('.')[0], 'spec.json') + raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") return ConnectorSpecification.parse_obj(json.loads(raw_spec)) def read_config(self, config_path): - with open(config_path, 'r') as file: + with open(config_path, "r") as file: contents = file.read() return json.loads(contents) @@ -72,7 +73,7 @@ def transform_config(self, raw_config): return raw_config def write_config(self, config_object, path): - with open(path, 'w') as fh: + with open(path, "w") as fh: fh.write(json.dumps(config_object)) def check(self, logger, config_container) -> AirbyteCheckResponse: diff --git a/airbyte-integrations/bases/base-python/airbyte_protocol/models/airbyte_message.py b/airbyte-integrations/bases/base-python/airbyte_protocol/models/airbyte_message.py index 2f3a4b5f42f7b..b6da958ee0f42 100644 --- a/airbyte-integrations/bases/base-python/airbyte_protocol/models/airbyte_message.py +++ b/airbyte-integrations/bases/base-python/airbyte_protocol/models/airbyte_message.py @@ -34,44 +34,44 @@ class Type(Enum): - RECORD = 'RECORD' - STATE = 'STATE' - LOG = 'LOG' - SPEC = 'SPEC' - CONNECTION_STATUS = 'CONNECTION_STATUS' - CATALOG = 'CATALOG' + RECORD = "RECORD" + STATE = "STATE" + LOG = "LOG" + SPEC = "SPEC" + CONNECTION_STATUS = "CONNECTION_STATUS" + CATALOG = "CATALOG" class AirbyteRecordMessage(BaseModel): - stream: str = Field(..., description='the name of the stream for this record') - data: Dict[str, Any] = Field(..., description='the record data') + stream: str = Field(..., description="the name of the stream for this record") + data: Dict[str, Any] = Field(..., description="the record data") emitted_at: int = Field( ..., - description='when the data was emitted from the source. epoch in millisecond.', + description="when the data was emitted from the source. epoch in millisecond.", ) class AirbyteStateMessage(BaseModel): - data: Dict[str, Any] = Field(..., description='the state data') + data: Dict[str, Any] = Field(..., description="the state data") class Level(Enum): - FATAL = 'FATAL' - ERROR = 'ERROR' - WARN = 'WARN' - INFO = 'INFO' - DEBUG = 'DEBUG' - TRACE = 'TRACE' + FATAL = "FATAL" + ERROR = "ERROR" + WARN = "WARN" + INFO = "INFO" + DEBUG = "DEBUG" + TRACE = "TRACE" class AirbyteLogMessage(BaseModel): - level: Level = Field(..., description='the type of logging') - message: str = Field(..., description='the log message') + level: Level = Field(..., description="the type of logging") + message: str = Field(..., description="the log message") class Status(Enum): - SUCCEEDED = 'SUCCEEDED' - FAILED = 'FAILED' + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" class AirbyteConnectionStatus(BaseModel): @@ -81,9 +81,7 @@ class AirbyteConnectionStatus(BaseModel): class AirbyteStream(BaseModel): name: str = Field(..., description="Stream's name.") - json_schema: Dict[str, Any] = Field( - ..., description='Stream schema using Json Schema specs.' - ) + json_schema: Dict[str, Any] = Field(..., description="Stream schema using Json Schema specs.") class ConnectorSpecification(BaseModel): @@ -91,7 +89,7 @@ class ConnectorSpecification(BaseModel): changelogUrl: Optional[AnyUrl] = None connectionSpecification: Dict[str, Any] = Field( ..., - description='ConnectorDefinition specific blob. Must be a valid JSON string.', + description="ConnectorDefinition specific blob. Must be a valid JSON string.", ) @@ -100,21 +98,19 @@ class AirbyteCatalog(BaseModel): class AirbyteMessage(BaseModel): - type: Type = Field(..., description='Message type') + type: Type = Field(..., description="Message type") log: Optional[AirbyteLogMessage] = Field( None, - description='log message: any kind of logging you want the platform to know about.', + description="log message: any kind of logging you want the platform to know about.", ) spec: Optional[ConnectorSpecification] = None connectionStatus: Optional[AirbyteConnectionStatus] = None catalog: Optional[AirbyteCatalog] = Field( None, - description='log message: any kind of logging you want the platform to know about.', - ) - record: Optional[AirbyteRecordMessage] = Field( - None, description='record message: the record' + description="log message: any kind of logging you want the platform to know about.", ) + record: Optional[AirbyteRecordMessage] = Field(None, description="record message: the record") state: Optional[AirbyteStateMessage] = Field( None, - description='schema message: the state. Must be the last message produced. The platform uses this information', + description="schema message: the state. Must be the last message produced. The platform uses this information", ) diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle index fc3b39a0b6845..eb03b0f797e88 100644 --- a/airbyte-integrations/bases/base-python/build.gradle +++ b/airbyte-integrations/bases/base-python/build.gradle @@ -1,10 +1,10 @@ +project.ext.pyModule = 'airbyte_protocol' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') task generateProtocolClassFiles(type: Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/generate-protocol-files.sh' - - outputs.upToDateWhen { false } dependsOn ':airbyte-integrations:bases:base-python:code-generator:buildImage' } diff --git a/airbyte-integrations/bases/base-python/setup.py b/airbyte-integrations/bases/base-python/setup.py index 2fd7bf8cd4de7..b044a50bb0962 100644 --- a/airbyte-integrations/bases/base-python/setup.py +++ b/airbyte-integrations/bases/base-python/setup.py @@ -25,25 +25,15 @@ import setuptools setuptools.setup( - name='airbyte-protocol', - description='Contains classes representing the schema of the Airbyte protocol.', - author='Airbyte', - author_email='contact@airbyte.io', - url='https://github.com/airbytehq/airbyte', - + name="airbyte-protocol", + description="Contains classes representing the schema of the Airbyte protocol.", + author="Airbyte", + author_email="contact@airbyte.io", + url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), - package_data={ - '': ['models/yaml/*.yaml'] - }, - - install_requires=[ - 'PyYAML==5.3.1', - 'pydantic==1.6.1' - ], - + package_data={"": ["models/yaml/*.yaml"]}, + install_requires=["PyYAML==5.3.1", "pydantic==1.6.1"], entry_points={ - 'console_scripts': [ - 'base-python=airbyte_protocol.entrypoint:main' - ], - } + "console_scripts": ["base-python=airbyte_protocol.entrypoint:main"], + }, ) diff --git a/airbyte-integrations/bases/base-singer/airbyte_protocol b/airbyte-integrations/bases/base-singer/airbyte_protocol new file mode 120000 index 0000000000000..4e6d7ee94ae82 --- /dev/null +++ b/airbyte-integrations/bases/base-singer/airbyte_protocol @@ -0,0 +1 @@ +../base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/bases/base-singer/base_singer/__init__.py b/airbyte-integrations/bases/base-singer/base_singer/__init__.py index a7519d57b55cb..11253e6516796 100644 --- a/airbyte-integrations/bases/base-singer/base_singer/__init__.py +++ b/airbyte-integrations/bases/base-singer/base_singer/__init__.py @@ -22,5 +22,5 @@ SOFTWARE. """ -from .singer_helpers import * -from .source import * +from .singer_helpers import * # noqa +from .source import * # noqa diff --git a/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py index 56889ba298788..9f793c606a7b5 100644 --- a/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py @@ -22,25 +22,21 @@ SOFTWARE. """ +import json import os import selectors import subprocess -from airbyte_protocol import AirbyteCatalog -from airbyte_protocol import AirbyteMessage -from airbyte_protocol import AirbyteRecordMessage -from airbyte_protocol import AirbyteStateMessage -from airbyte_protocol import AirbyteStream from dataclasses import dataclass from datetime import datetime -from typing import Generator, List, DefaultDict +from typing import DefaultDict, Generator -import json +from airbyte_protocol import AirbyteCatalog, AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream def to_json(string): try: return json.loads(string) - except ValueError as e: + except ValueError: return False @@ -62,12 +58,15 @@ class SingerHelper: def _transform_types(stream_properties: DefaultDict): for field_name in stream_properties: field_object = stream_properties[field_name] - field_object['type'] = SingerHelper._parse_type(field_object['type']) + field_object["type"] = SingerHelper._parse_type(field_object["type"]) @staticmethod - def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: - completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + def get_catalogs( + logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog) + ) -> Catalogs: + completed_process = subprocess.run( + shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + ) for line in completed_process.stderr.splitlines(): logger.log_by_prefix(line, "ERROR") @@ -102,9 +101,9 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x if out_json is not None and is_message(out_json): transformed_json = transform(out_json) if transformed_json is not None: - if transformed_json.get('type') == "SCHEMA" or transformed_json.get('type') == "ACTIVATE_VERSION": + if transformed_json.get("type") == "SCHEMA" or transformed_json.get("type") == "ACTIVATE_VERSION": pass - elif transformed_json.get('type') == "STATE": + elif transformed_json.get("type") == "STATE": out_record = AirbyteStateMessage(data=transformed_json["value"]) out_message = AirbyteMessage(type="STATE", state=out_record) yield transform(out_message) @@ -114,7 +113,8 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x out_record = AirbyteRecordMessage( stream=stream_name, data=transformed_json["record"], - emitted_at=int(datetime.now().timestamp()) * 1000) + emitted_at=int(datetime.now().timestamp()) * 1000, + ) out_message = AirbyteMessage(type="RECORD", record=out_record) yield transform(out_message) else: @@ -124,7 +124,7 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x @staticmethod def create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog) -> str: - combined_catalog_path = os.path.join('singer_rendered_catalog.json') + combined_catalog_path = os.path.join("singer_rendered_catalog.json") masked_singer_streams = [] stream_to_airbyte_schema = {} @@ -150,7 +150,7 @@ def create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_sing masked_singer_streams += [singer_stream] combined_catalog = {"streams": masked_singer_streams} - with open(combined_catalog_path, 'w') as fh: + with open(combined_catalog_path, "w") as fh: fh.write(json.dumps(combined_catalog)) return combined_catalog_path diff --git a/airbyte-integrations/bases/base-singer/base_singer/source.py b/airbyte-integrations/bases/base-singer/base_singer/source.py index 4b0426794611d..a47a7d1454b4f 100644 --- a/airbyte-integrations/bases/base-singer/base_singer/source.py +++ b/airbyte-integrations/bases/base-singer/base_singer/source.py @@ -22,16 +22,14 @@ SOFTWARE. """ -from airbyte_protocol import AirbyteCatalog -from airbyte_protocol import AirbyteMessage -from airbyte_protocol import Source from typing import Generator +from airbyte_protocol import AirbyteCatalog, AirbyteMessage, Source + from .singer_helpers import SingerHelper class SingerSource(Source): - def discover_cmd(self, logger, config_path) -> str: raise Exception("Not Implemented") diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index 4b2c8b5ab8d83..542401f0341c0 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -1,3 +1,5 @@ +project.ext.pyModule = 'base_singer' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' diff --git a/airbyte-integrations/bases/base-singer/setup.py b/airbyte-integrations/bases/base-singer/setup.py index a18505d23eb72..26ce603450a0e 100644 --- a/airbyte-integrations/bases/base-singer/setup.py +++ b/airbyte-integrations/bases/base-singer/setup.py @@ -22,15 +22,13 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='base-singer', - description='Contains helpers for handling Singer sources and destinations.', - author='Airbyte', - author_email='contact@airbyte.io', - + name="base-singer", + description="Contains helpers for handling Singer sources and destinations.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - - install_requires=['airbyte-protocol'] + install_requires=["airbyte-protocol"], ) diff --git a/airbyte-integrations/connector-templates/python-source/airbyte_protocol b/airbyte-integrations/connector-templates/python-source/airbyte_protocol new file mode 120000 index 0000000000000..124c03ba9ebbc --- /dev/null +++ b/airbyte-integrations/connector-templates/python-source/airbyte_protocol @@ -0,0 +1 @@ +../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/python-source/build.gradle b/airbyte-integrations/connector-templates/python-source/build.gradle index a0c5bb8bfd425..5a28e46b610c9 100644 --- a/airbyte-integrations/connector-templates/python-source/build.gradle +++ b/airbyte-integrations/connector-templates/python-source/build.gradle @@ -1,3 +1,5 @@ +project.ext.pyModule = 'template_python_source' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') buildImage.dependsOn ":airbyte-integrations:bases:base-python:buildImage" diff --git a/airbyte-integrations/connector-templates/python-source/main_dev.py b/airbyte-integrations/connector-templates/python-source/main_dev.py index c943a50d13101..b56990dad7bf4 100644 --- a/airbyte-integrations/connector-templates/python-source/main_dev.py +++ b/airbyte-integrations/connector-templates/python-source/main_dev.py @@ -25,7 +25,6 @@ import sys from airbyte_protocol.entrypoint import launch - from template_python_source import TemplatePythonSource if __name__ == "__main__": diff --git a/airbyte-integrations/connector-templates/python-source/requirements.txt b/airbyte-integrations/connector-templates/python-source/requirements.txt index d92984a47ae4b..f8204a82d0328 100644 --- a/airbyte-integrations/connector-templates/python-source/requirements.txt +++ b/airbyte-integrations/connector-templates/python-source/requirements.txt @@ -1,2 +1,2 @@ --e ../../base-python +-e ../../bases/base-python -e . diff --git a/airbyte-integrations/connector-templates/python-source/setup.py b/airbyte-integrations/connector-templates/python-source/setup.py index eb5114f713627..79f8ce2f23280 100644 --- a/airbyte-integrations/connector-templates/python-source/setup.py +++ b/airbyte-integrations/connector-templates/python-source/setup.py @@ -22,18 +22,14 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='template-python-source', - description='Source implementation template', - author='Airbyte', - author_email='contact@airbyte.io', - + name="template-python-source", + description="Source implementation template", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - - install_requires=['airbyte-protocol'] + package_data={"": ["*.json"]}, + install_requires=["airbyte-protocol", "pydantic==1.6.1"], ) diff --git a/airbyte-integrations/connector-templates/python-source/template_python_source/__init__.py b/airbyte-integrations/connector-templates/python-source/template_python_source/__init__.py index 3f5220e13e201..59e195f1db1a6 100644 --- a/airbyte-integrations/connector-templates/python-source/template_python_source/__init__.py +++ b/airbyte-integrations/connector-templates/python-source/template_python_source/__init__.py @@ -22,4 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import TemplatePythonSource + +__all__ = ["TemplatePythonSource"] diff --git a/airbyte-integrations/connector-templates/python-source/template_python_source/source.py b/airbyte-integrations/connector-templates/python-source/template_python_source/source.py index cca2a4de27f2c..67965e32a9583 100644 --- a/airbyte-integrations/connector-templates/python-source/template_python_source/source.py +++ b/airbyte-integrations/connector-templates/python-source/template_python_source/source.py @@ -26,13 +26,7 @@ import time from typing import Generator -from airbyte_protocol import AirbyteCatalog -from airbyte_protocol import AirbyteCheckResponse -from airbyte_protocol import AirbyteMessage -from airbyte_protocol import AirbyteRecordMessage -from airbyte_protocol import AirbyteSpec -from airbyte_protocol import AirbyteStateMessage -from airbyte_protocol import Source +from airbyte_protocol import AirbyteCatalog, AirbyteCheckResponse, AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, Source class TemplatePythonSource(Source): @@ -40,21 +34,18 @@ def __init__(self): pass def check(self, logger, config_container) -> AirbyteCheckResponse: - logger.info(f'Checking configuration ({config_container.rendered_config_path})...') + logger.info(f"Checking configuration ({config_container.rendered_config_path})...") return AirbyteCheckResponse(True, {}) def discover(self, logger, config_container) -> AirbyteCatalog: - logger.info(f'Discovering ({config_container.rendered_config_path})...') - return AirbyteCatalog.from_json(pkgutil.get_data(__name__, 'catalog.json')) + logger.info(f"Discovering ({config_container.rendered_config_path})...") + return AirbyteCatalog.from_json(pkgutil.get_data(__name__, "catalog.json")) def read(self, logger, config_container, catalog_path, state_path=None) -> Generator[AirbyteMessage, None, None]: - logger.info(f'Reading ({config_container.rendered_config_path}, {catalog_path}, {state_path})...') + logger.info(f"Reading ({config_container.rendered_config_path}, {catalog_path}, {state_path})...") - message = AirbyteRecordMessage( - stream='love_airbyte', - data={'love': True}, - emitted_at=int(time.time() * 1000)) - yield AirbyteMessage(type='RECORD', record=message) + message = AirbyteRecordMessage(stream="love_airbyte", data={"love": True}, emitted_at=int(time.time() * 1000)) + yield AirbyteMessage(type="RECORD", record=message) - state = AirbyteStateMessage(data={'love_cursor': 'next_version'}) - yield AirbyteMessage(type='STATE', state=state) + state = AirbyteStateMessage(data={"love_cursor": "next_version"}) + yield AirbyteMessage(type="STATE", state=state) diff --git a/airbyte-integrations/connector-templates/singer-source/airbyte_protocol b/airbyte-integrations/connector-templates/singer-source/airbyte_protocol new file mode 120000 index 0000000000000..124c03ba9ebbc --- /dev/null +++ b/airbyte-integrations/connector-templates/singer-source/airbyte_protocol @@ -0,0 +1 @@ +../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/singer-source/base_singer b/airbyte-integrations/connector-templates/singer-source/base_singer new file mode 120000 index 0000000000000..873ffcbc1ac36 --- /dev/null +++ b/airbyte-integrations/connector-templates/singer-source/base_singer @@ -0,0 +1 @@ +../../bases/base-singer/base_singer \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/singer-source/build.gradle b/airbyte-integrations/connector-templates/singer-source/build.gradle index 1b8b5e471c6b0..3c8ede33df73e 100644 --- a/airbyte-integrations/connector-templates/singer-source/build.gradle +++ b/airbyte-integrations/connector-templates/singer-source/build.gradle @@ -1,3 +1,5 @@ +project.ext.pyModule = 'template_singer_source' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') diff --git a/airbyte-integrations/connector-templates/singer-source/main_dev.py b/airbyte-integrations/connector-templates/singer-source/main_dev.py index 7c3f584738333..b064bfcaa3572 100644 --- a/airbyte-integrations/connector-templates/singer-source/main_dev.py +++ b/airbyte-integrations/connector-templates/singer-source/main_dev.py @@ -23,8 +23,8 @@ """ import sys -from airbyte_protocol.entrypoint import launch +from airbyte_protocol.entrypoint import launch from template_singer_source import TemplateSingerSource if __name__ == "__main__": diff --git a/airbyte-integrations/connector-templates/singer-source/requirements.txt b/airbyte-integrations/connector-templates/singer-source/requirements.txt index 2bddfcb9617fc..e269d2d23e34b 100644 --- a/airbyte-integrations/connector-templates/singer-source/requirements.txt +++ b/airbyte-integrations/connector-templates/singer-source/requirements.txt @@ -1,3 +1,3 @@ --e ../../base-python --e ../../base-singer +-e ../../bases/base-python +-e ../../bases/base-singer -e . diff --git a/airbyte-integrations/connector-templates/singer-source/setup.py b/airbyte-integrations/connector-templates/singer-source/setup.py index 475d3dcec88b0..d666bbde8fe15 100644 --- a/airbyte-integrations/connector-templates/singer-source/setup.py +++ b/airbyte-integrations/connector-templates/singer-source/setup.py @@ -22,22 +22,19 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='template-singer-source', - description='Singer source implementation template', - author='Airbyte', - author_email='contact@airbyte.io', - + name="template-singer-source", + description="Singer source implementation template", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - + package_data={"": ["*.json"]}, install_requires=[ - 'tap-exchangeratesapi==0.1.1', - 'base_singer', - 'airbyte_protocol' - ] + "tap-exchangeratesapi==0.1.1", + "pydantic==1.6.1", + "base_singer", + "airbyte_protocol", + ], ) diff --git a/airbyte-integrations/connector-templates/singer-source/template_singer_source/__init__.py b/airbyte-integrations/connector-templates/singer-source/template_singer_source/__init__.py index 3f5220e13e201..8d6a99d65c5d7 100644 --- a/airbyte-integrations/connector-templates/singer-source/template_singer_source/__init__.py +++ b/airbyte-integrations/connector-templates/singer-source/template_singer_source/__init__.py @@ -22,4 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import TemplateSingerSource + +__all__ = ["TemplateSingerSource"] diff --git a/airbyte-integrations/connector-templates/singer-source/template_singer_source/source.py b/airbyte-integrations/connector-templates/singer-source/template_singer_source/source.py index b24b58c9b6072..f8378aa65216d 100644 --- a/airbyte-integrations/connector-templates/singer-source/template_singer_source/source.py +++ b/airbyte-integrations/connector-templates/singer-source/template_singer_source/source.py @@ -38,7 +38,7 @@ def check(self, logger, config_container) -> AirbyteCheckResponse: return AirbyteCheckResponse(code == 200, {}) def discover_cmd(self, logger, config_path) -> str: - return "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'" + return 'tap-exchangeratesapi | grep \'"type": "SCHEMA"\' | head -1 | jq -c \'{"streams":[{"stream": .stream, "schema": .schema}]}\'' def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str: config_option = f"--config {config_path}" diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/airbyte_protocol b/airbyte-integrations/connectors/source-exchangeratesapi-singer/airbyte_protocol new file mode 120000 index 0000000000000..124c03ba9ebbc --- /dev/null +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/airbyte_protocol @@ -0,0 +1 @@ +../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/base_singer b/airbyte-integrations/connectors/source-exchangeratesapi-singer/base_singer new file mode 120000 index 0000000000000..873ffcbc1ac36 --- /dev/null +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/base_singer @@ -0,0 +1 @@ +../../bases/base-singer/base_singer \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle index 0847a630c47a1..4183845c9061f 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +project.ext.pyModule = 'source_exchangeratesapi_singer' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/main_dev.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/main_dev.py index 67621e7555ee8..ed54a5740805d 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/main_dev.py +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/main_dev.py @@ -23,8 +23,8 @@ """ import sys -from airbyte_protocol.entrypoint import launch +from airbyte_protocol.entrypoint import launch from source_exchangeratesapi_singer import SourceExchangeRatesApiSinger if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py index 24ed6e89aad53..037d9f8f38bb1 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py @@ -22,22 +22,19 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='source-exchangeratesapi-singer', - description='Source implementation for the exchange rates API.', - author='Airbyte', - author_email='contact@airbyte.io', - + name="source-exchangeratesapi-singer", + description="Source implementation for the exchange rates API.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - + package_data={"": ["*.json"]}, install_requires=[ - 'tap-exchangeratesapi==0.1.1', - 'base_singer', - 'airbyte_protocol' - ] + "tap-exchangeratesapi==0.1.1", + "pydantic==1.6.1", + "base_singer", + "airbyte_protocol", + ], ) diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/__init__.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/__init__.py index 3f5220e13e201..c278891ed418c 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/__init__.py +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/__init__.py @@ -22,4 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import SourceExchangeRatesApiSinger + +__all__ = ["SourceExchangeRatesApiSinger"] diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/source.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/source.py index 152633906f06b..01684c43168f4 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/source.py +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/source_exchangeratesapi_singer/source.py @@ -36,14 +36,13 @@ def check(self, logger, config_path) -> AirbyteConnectionStatus: try: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() logger.info(f"Ping response code: {code}") - return AirbyteConnectionStatus(status=Status.SUCCEEDED if (code==200) else Status.FAILED) + return AirbyteConnectionStatus(status=Status.SUCCEEDED if (code == 200) else Status.FAILED) except Exception as e: return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}") def discover_cmd(self, logger, config_path) -> str: - return "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'" + return 'tap-exchangeratesapi | grep \'"type": "SCHEMA"\' | head -1 | jq -c \'{"streams":[{"stream": .stream, "schema": .schema}]}\'' def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str: state_option = f"--state {state_path}" if state_path else "" return f"tap-exchangeratesapi --config {config_path} {state_option}" - diff --git a/airbyte-integrations/connectors/source-postgres-singer/airbyte_protocol b/airbyte-integrations/connectors/source-postgres-singer/airbyte_protocol new file mode 120000 index 0000000000000..124c03ba9ebbc --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-singer/airbyte_protocol @@ -0,0 +1 @@ +../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres-singer/base_singer b/airbyte-integrations/connectors/source-postgres-singer/base_singer new file mode 120000 index 0000000000000..873ffcbc1ac36 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-singer/base_singer @@ -0,0 +1 @@ +../../bases/base-singer/base_singer \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres-singer/build.gradle b/airbyte-integrations/connectors/source-postgres-singer/build.gradle index fd557727dd242..b8187c280b8ef 100644 --- a/airbyte-integrations/connectors/source-postgres-singer/build.gradle +++ b/airbyte-integrations/connectors/source-postgres-singer/build.gradle @@ -2,8 +2,11 @@ plugins { id 'java' } +project.ext.pyModule = 'postgres_singer_source' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') + dependencies { integrationTestImplementation 'org.apache.commons:commons-dbcp2:2.7.0' integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind' diff --git a/airbyte-integrations/connectors/source-postgres-singer/main_dev.py b/airbyte-integrations/connectors/source-postgres-singer/main_dev.py index 2289ca0fe0174..290df945b0848 100644 --- a/airbyte-integrations/connectors/source-postgres-singer/main_dev.py +++ b/airbyte-integrations/connectors/source-postgres-singer/main_dev.py @@ -23,8 +23,8 @@ """ import sys -from airbyte_protocol.entrypoint import launch +from airbyte_protocol.entrypoint import launch from postgres_singer_source import PostgresSingerSource if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/__init__.py b/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/__init__.py index 3f5220e13e201..8def10e1f6710 100644 --- a/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/__init__.py +++ b/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/__init__.py @@ -22,4 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import PostgresSingerSource + +__all__ = ["PostgresSingerSource"] diff --git a/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/source.py b/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/source.py index c3f2ea1fd92b4..3b72ec95fc689 100644 --- a/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/source.py +++ b/airbyte-integrations/connectors/source-postgres-singer/postgres_singer_source/source.py @@ -22,21 +22,15 @@ SOFTWARE. """ -import urllib.request -import psycopg2 - from typing import Generator -from airbyte_protocol import AirbyteCatalog -from airbyte_protocol import AirbyteConnectionStatus -from airbyte_protocol import Status -from airbyte_protocol import AirbyteMessage -from airbyte_protocol import Source -from airbyte_protocol import ConfigContainer -from base_singer import SingerHelper, SingerSource - +import psycopg2 +from airbyte_protocol import AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, ConfigContainer, Status +from base_singer import SingerSource TAP_CMD = "PGCLIENTENCODING=UTF8 tap-postgres" + + class PostgresSingerSource(SingerSource): def __init__(self): pass @@ -44,7 +38,7 @@ def __init__(self): def check(self, logger, config_container: ConfigContainer) -> AirbyteConnectionStatus: config = config_container.rendered_config try: - params="dbname='{dbname}' user='{user}' host='{host}' password='{password}' port='{port}'".format(**config) + params = "dbname='{dbname}' user='{user}' host='{host}' password='{password}' port='{port}'".format(**config) psycopg2.connect(params) return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: @@ -56,7 +50,7 @@ def transform_config(self, raw_config): # It should be equal to the dbname option in all cases. # See https://github.com/singer-io/tap-postgres source code for more information rendered_config = dict(raw_config) - rendered_config['filter_dbs'] = raw_config['dbname'] + rendered_config["filter_dbs"] = raw_config["dbname"] return rendered_config def discover_cmd(self, logger, config_path) -> AirbyteCatalog: diff --git a/airbyte-integrations/connectors/source-postgres-singer/setup.py b/airbyte-integrations/connectors/source-postgres-singer/setup.py index 59ddcc7d0911e..6d73982bcc4d2 100644 --- a/airbyte-integrations/connectors/source-postgres-singer/setup.py +++ b/airbyte-integrations/connectors/source-postgres-singer/setup.py @@ -22,23 +22,20 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='postgres-singer-source', - description='Postgres Singer source', - author='Airbyte', - author_email='contact@airbyte.io', - + name="postgres-singer-source", + description="Postgres Singer source", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - + package_data={"": ["*.json"]}, install_requires=[ - 'psycopg2==2.7.4', - 'tap-postgres==0.1.0', - 'base_singer', - 'airbyte_protocol' - ] + "psycopg2==2.8.4", + "tap-postgres==0.1.0", + "pydantic==1.6.1", + "base_singer", + "airbyte_protocol", + ], ) diff --git a/airbyte-integrations/connectors/source-stripe-singer/airbyte_protocol b/airbyte-integrations/connectors/source-stripe-singer/airbyte_protocol new file mode 120000 index 0000000000000..124c03ba9ebbc --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe-singer/airbyte_protocol @@ -0,0 +1 @@ +../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe-singer/base_singer b/airbyte-integrations/connectors/source-stripe-singer/base_singer new file mode 120000 index 0000000000000..873ffcbc1ac36 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe-singer/base_singer @@ -0,0 +1 @@ +../../bases/base-singer/base_singer \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe-singer/build.gradle b/airbyte-integrations/connectors/source-stripe-singer/build.gradle index a56c10b7fa1c1..a56ed12a5a1d5 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/build.gradle +++ b/airbyte-integrations/connectors/source-stripe-singer/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +project.ext.pyModule = 'source_stripe_singer' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') diff --git a/airbyte-integrations/connectors/source-stripe-singer/main_dev.py b/airbyte-integrations/connectors/source-stripe-singer/main_dev.py index 955b4a75df97f..315543476f1bf 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/main_dev.py +++ b/airbyte-integrations/connectors/source-stripe-singer/main_dev.py @@ -23,8 +23,8 @@ """ import sys -from airbyte_protocol.entrypoint import launch +from airbyte_protocol.entrypoint import launch from source_stripe_singer import SourceStripeSinger if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-stripe-singer/setup.py b/airbyte-integrations/connectors/source-stripe-singer/setup.py index ed22b5225221f..f38472b17e1d4 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/setup.py +++ b/airbyte-integrations/connectors/source-stripe-singer/setup.py @@ -22,23 +22,19 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='source_stripe_singer', - description='Source implementation for Stripe.', - author='Airbyte', - author_email='contact@airbyte.io', - + name="source_stripe_singer", + description="Source implementation for Stripe.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - + package_data={"": ["*.json"]}, install_requires=[ - 'tap-stripe==1.4.4', - 'requests', - 'base_singer', - 'airbyte_protocol' - ] + "tap-stripe==1.4.4", + "requests", + "base_singer", + "airbyte_protocol", + ], ) diff --git a/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/__init__.py b/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/__init__.py index 11ea21a7cdd1f..6973233b01c6e 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/__init__.py +++ b/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/__init__.py @@ -22,5 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import SourceStripeSinger +__all__ = ["SourceStripeSinger"] diff --git a/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/source.py b/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/source.py index bc784c8ca836a..5718d3891ad73 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/source.py +++ b/airbyte-integrations/connectors/source-stripe-singer/source_stripe_singer/source.py @@ -34,7 +34,7 @@ def __init__(self): def check(self, logger, config_container) -> AirbyteConnectionStatus: try: json_config = config_container.rendered_config - r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], '')) + r = requests.get("https://api.stripe.com/v1/customers", auth=(json_config["client_secret"], "")) if r.status_code == 200: return AirbyteConnectionStatus(status=Status.SUCCEEDED) else: diff --git a/build.gradle b/build.gradle index d7f5e5e01261e..a9bb5acd62629 100644 --- a/build.gradle +++ b/build.gradle @@ -3,6 +3,7 @@ plugins { id 'java' id 'pmd' id 'com.diffplug.spotless' version '5.6.1' + id 'ru.vyarus.use-python' version '2.2.0' apply false // id 'de.aaschmid.cpd' version '3.1' } @@ -71,7 +72,7 @@ spotless { } python { target '**/*.py' - targetExclude "**/build/**/*", "**/.gradle/**/*" + targetExclude "**/build/**/*", "**/.gradle/**/*", "**/.venv/**/*" licenseHeaderFile createPythonLicenseWith(rootProject.file('LICENSE')), '(from|import|# generated)' } format 'styling', { diff --git a/tools/gradle/commons/integrations/python.gradle b/tools/gradle/commons/integrations/python.gradle new file mode 100644 index 0000000000000..d4680cc652aae --- /dev/null +++ b/tools/gradle/commons/integrations/python.gradle @@ -0,0 +1,47 @@ +apply plugin: 'ru.vyarus.use-python' + +python { + envPath = '.venv' + minPythonVersion = '3.7' + scope = VIRTUALENV + installVirtualenv = true + pip 'flake8:3.8.4' + pip 'black:20.8b1' + pip 'mypy:0.790' + pip 'isort:5.6.4' +} + +task installReqs(type: PythonTask) { + module = "pip" + command = "install -r requirements.txt" +} + +task blackFormat(type: PythonTask) { + module = "black" + // the line length should match .isort.cfg + command = ". --line-length 140" +} + +task isortFormat(type: PythonTask) { + module = "isort" + command = ". --settings-file ${rootProject.file('tools/python/.isort.cfg').absolutePath}" +} + +task flakeCheck(type: PythonTask, dependsOn: blackFormat) { + module = "flake8" + command = ". --config ${rootProject.file('tools/python/.flake8').absolutePath}" +} + +if(project.ext.has('pyModule')) { + task mypyCheck(type: PythonTask, dependsOn: installReqs) { + module = "mypy" + command = "-m ${project.ext.pyModule} --config-file ${rootProject.file('tools/python/.mypy.ini').absolutePath}" + } + + check.dependsOn mypyCheck +} + +build.dependsOn installReqs +check.dependsOn blackFormat +check.dependsOn isortFormat +check.dependsOn flakeCheck diff --git a/tools/python/.flake8 b/tools/python/.flake8 new file mode 100644 index 0000000000000..a18b92871182f --- /dev/null +++ b/tools/python/.flake8 @@ -0,0 +1,9 @@ +[flake8] +exclude = + .venv, + models # generated protocol models +extend-ignore = + E203, # whitespace before ':' (conflicts with Black) + E231, # Bad trailing comma (conflicts with Black) + E501, # line too long (conflicts with Black) + W503, # line break before binary operator (conflicts with Black) diff --git a/tools/python/.isort.cfg b/tools/python/.isort.cfg new file mode 100644 index 0000000000000..9c615d0fd7b14 --- /dev/null +++ b/tools/python/.isort.cfg @@ -0,0 +1,8 @@ +[settings] +# This is to make isort compatible with Black. See +# https://black.readthedocs.io/en/stable/the_black_code_style.html#how-black-wraps-lines. +line_length=140 +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True diff --git a/tools/python/.mypy.ini b/tools/python/.mypy.ini new file mode 100644 index 0000000000000..1ffad0c4b3343 --- /dev/null +++ b/tools/python/.mypy.ini @@ -0,0 +1,34 @@ +[mypy] +# Optionals +no_implicit_optional = True + +# Strictness +allow_untyped_globals = False +allow_redefinition = False +implicit_reexport = False +strict_equality = True + +# Warnings +warn_unused_ignores = True +warn_no_return = True +warn_return_any = True +warn_redundant_casts = True +warn_unreachable = True + +# Error output +show_column_numbers = True +show_error_context = True +show_error_codes = True +show_traceback = True +pretty = True +color_output = True +error_summary = True + +[mypy-colors] +ignore_missing_imports = True + +[mypy-translate] +ignore_missing_imports = True + +[mypy-pytest] +ignore_missing_imports = True