From c948f18186983a16f3bab9282a0c035131e7588b Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Thu, 30 May 2024 10:00:20 -0700 Subject: [PATCH] destination-duckdb: [autopull] base image + poetry + up_to_date (#38486) Co-authored-by: Augustin Co-authored-by: alafanechere --- .../connectors/destination-duckdb/README.md | 62 ++++++++++++++++--- .../destination-duckdb/metadata.yaml | 10 ++- .../destination-duckdb/pyproject.toml | 2 +- docs/integrations/destinations/duckdb.md | 1 + 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/destination-duckdb/README.md b/airbyte-integrations/connectors/destination-duckdb/README.md index 8857bce3546c..b143a5afda11 100644 --- a/airbyte-integrations/connectors/destination-duckdb/README.md +++ b/airbyte-integrations/connectors/destination-duckdb/README.md @@ -56,22 +56,70 @@ cat integration_tests/messages.jsonl| python main.py write --config integration_ ### Locally running the connector docker image -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: ```bash -airbyte-ci connectors --name=destination-duckdb build [--architecture=...] +airbyte-ci connectors --name destination-duckdb build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-duckdb:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -An image will be built with the tag `airbyte/destination-duckdb:dev`. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-duckdb:latest -**Via `docker build`:** +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: ```bash docker build -t airbyte/destination-duckdb:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-duckdb:dev spec ``` - #### Run Then run any of the connector commands as follows: @@ -114,4 +162,4 @@ You've checked out the repo, implemented a million dollar feature, and you're re 4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/duckdb.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml index aa541135b32e..a9e7cb5495c3 100644 --- a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml @@ -1,10 +1,10 @@ data: connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: database connectorType: destination definitionId: 94bd199c-2ff0-4aa2-b98e-17f0acb72610 - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 dockerRepository: airbyte/destination-duckdb githubIssueLabel: destination-duckdb icon: duckdb.svg @@ -19,7 +19,11 @@ data: releases: breakingChanges: 0.3.0: - message: "This version uses the DuckDB 0.9.1 database driver, which is not backwards compatible with prior versions. MotherDuck users can upgrade their database by visiting https://app.motherduck.com/ and accepting the upgrade. For more information, see the connector migration guide." + message: + "This version uses the DuckDB 0.9.1 database driver, which is not + backwards compatible with prior versions. MotherDuck users can upgrade their + database by visiting https://app.motherduck.com/ and accepting the upgrade. + For more information, see the connector migration guide." upgradeDeadline: "2023-10-31" resourceRequirements: jobSpecific: diff --git a/airbyte-integrations/connectors/destination-duckdb/pyproject.toml b/airbyte-integrations/connectors/destination-duckdb/pyproject.toml index 5353adc12e22..0a7387667f43 100644 --- a/airbyte-integrations/connectors/destination-duckdb/pyproject.toml +++ b/airbyte-integrations/connectors/destination-duckdb/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "destination-duckdb" -version = "0.3.5" +version = "0.3.6" description = "Destination implementation for Duckdb." authors = ["Simon Späti, Airbyte"] license = "MIT" diff --git a/docs/integrations/destinations/duckdb.md b/docs/integrations/destinations/duckdb.md index fc43df5b4011..73bb1d08f2b8 100644 --- a/docs/integrations/destinations/duckdb.md +++ b/docs/integrations/destinations/duckdb.md @@ -106,6 +106,7 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.3.6 | 2024-05-21 | [38486](https://github.com/airbytehq/airbyte/pull/38486) | [autopull] base image + poetry + up_to_date | | 0.3.5 | 2024-04-23 | [#37515](https://github.com/airbytehq/airbyte/pull/37515) | Add resource requirements declaration to `metatadat.yml`. | | :------ | :--------- | :------------------------------------------------------- | :--------------------- | | 0.3.4 | 2024-04-16 | [#36715](https://github.com/airbytehq/airbyte/pull/36715) | Improve ingestion performance using pyarrow inmem view for writing to DuckDB. |