Merge branch 'main' into aj/feat/unprivileged-discover-for-declarative-static-schemas

Author: aaronsteers

.github/pr-welcome-community.md

@@ -0,0 +1,32 @@
+## 👋 Welcome to the Airbyte Python CDK!
+
+Thank you for your contribution from **{{ .repo_name }}**! We're excited to have you in the Airbyte community.
+
+### Testing This CDK Version
+
+You can test this version of the CDK using the following:
+
+```bash
+# Run the CLI from this branch:
+uvx 'git+https://github.com/airbytehq/airbyte-python-cdk.git@{{ .branch_name }}#egg=airbyte-python-cdk[dev]' --help
+
+# Update a connector to use the CDK from this branch ref:
+cd airbyte-integrations/connectors/source-example
+poe use-cdk-branch {{ .branch_name }}
+```
+
+### Helpful Resources
+
+- [Contributing Guidelines](https://docs.airbyte.com/contributing-to-airbyte/)
+- [CDK API Reference](https://airbytehq.github.io/airbyte-python-cdk/)
+
+### PR Slash Commands
+
+As needed or by request, Airbyte Maintainers can execute the following slash commands on your PR:
+
+- `/autofix` - Fixes most formatting and linting issues
+- `/poetry-lock` - Updates poetry.lock file
+- `/test` - Runs connector tests with the updated CDK
+
+If you have any questions, feel free to ask in the PR comments or join our [Slack community](https://airbytehq.slack.com/).
+

.github/pr-welcome-internal.md

@@ -0,0 +1,31 @@
+## 👋 Greetings, Airbyte Team Member!
+
+Here are some helpful tips and reminders for your convenience.
+
+### Testing This CDK Version
+
+You can test this version of the CDK using the following:
+
+```bash
+# Run the CLI from this branch:
+uvx 'git+https://github.com/airbytehq/airbyte-python-cdk.git@{{ .branch_name }}#egg=airbyte-python-cdk[dev]' --help
+
+# Update a connector to use the CDK from this branch ref:
+cd airbyte-integrations/connectors/source-example
+poe use-cdk-branch {{ .branch_name }}
+```
+
+### Helpful Resources
+
+- [CDK API Reference](https://airbytehq.github.io/airbyte-python-cdk/)
+
+### PR Slash Commands
+
+Airbyte Maintainers can execute the following slash commands on your PR:
+
+- `/autofix` - Fixes most formatting and linting issues
+- `/poetry-lock` - Updates poetry.lock file
+- `/test` - Runs connector tests with the updated CDK
+- `/poe <command>` - Runs any poe command in the CDK environment
+
+[📝 _Edit this welcome message._](https://github.com/airbytehq/airbyte-python-cdk/blob/main/.github/pr-welcome-internal.md)

.github/workflows/connector-tests.yml

@@ -72,28 +72,26 @@ jobs:
             cdk_extra: n/a
           - connector: source-shopify
             cdk_extra: n/a
-          # Chargebee is being flaky:
-          # - connector: source-chargebee
-          #   cdk_extra: n/a
-          # This one is behind in CDK updates and can't be used as tests until it is updated:
-          # - connector: destination-pinecone
-          #   cdk_extra: vector-db-based
           - connector: source-google-drive
             cdk_extra: file-based
           - connector: destination-motherduck
-            cdk_extra: sql
-          # ZenDesk currently failing (as of 2024-12-02)
-          # TODO: Re-enable once fixed
-          # - connector: source-zendesk-support
+            # For now, we mark as 'n/a' to always test this connector
+            cdk_extra: n/a  # change to 'sql' to test less often
+          # source-amplitude failing for unrelated issue "date too far back"
+          # e.g. https://github.com/airbytehq/airbyte-python-cdk/actions/runs/16053716569/job/45302638848?pr=639
+          # - connector: source-amplitude
           #   cdk_extra: n/a
-          # TODO: These are manifest connectors and won't work as expected until we
-          #       add `--use-local-cdk` support for manifest connectors.
-          - connector: source-amplitude
+          - connector: source-intercom
             cdk_extra: n/a
           - connector: source-pokeapi
             cdk_extra: n/a
 
-    name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
+          # CDK Tests cannot build the Connector object (constructor args not optional).
+          # - connector: source-zendesk-support
+          #   cdk_extra: n/a
+
+    name: "${{ needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' && 'Skipped Check' || 'Check' }}: ${{matrix.connector}}"
+    if: needs.cdk_changes.outputs['src'] == 'true'
     permissions:
       checks: write
       contents: write # Required for creating commit statuses
@@ -125,110 +123,89 @@ jobs:
           repository: airbytehq/airbyte
           ref: master
           path: airbyte
+
       - name: Set up Python
+        if: steps.no_changes.outputs.status != 'cancelled'
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      # Create initial pending status for test report
-      - name: Create Pending Test Report Status
+
+      - name: Set up `uv`
+        if: steps.no_changes.outputs.status != 'cancelled'
+        uses: astral-sh/setup-uv@v6.1.0
+
+      - name: Set up `poe`
+        if: steps.no_changes.outputs.status != 'cancelled'
+        run: |
+          uv tool install poethepoet
+
+      - name: Set up Poetry
+        if: steps.no_changes.outputs.status != 'cancelled'
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "2.0.1"
+
+      - name: Get Connector Language
+        if: steps.no_changes.outputs.status != 'cancelled'
+        working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }}
+        run: |
+          # Get the language of the connector from the metadata file
+          CONNECTOR_LANGUAGE=$(poe -qq get-language)
+          echo "CONNECTOR_LANGUAGE=$CONNECTOR_LANGUAGE" | tee -a $GITHUB_ENV
+
+      - name: Install CDK with Poetry
         if: steps.no_changes.outputs.status != 'cancelled'
-        env:
-          GH_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
         run: |
-          HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
-          gh api \
-            --method POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            repos/${{ github.repository }}/statuses/$HEAD_SHA \
-            -f state="pending" \
-            -f description="Running connector tests..." \
-            -f context="${{ matrix.connector }} Test Report"
-
-      - name: Test Connector
+          cd airbyte-python-cdk
+          poetry install --all-extras
+
+      - name: Fetch Connector Secrets
         if: steps.no_changes.outputs.status != 'cancelled'
         timeout-minutes: 90
+        working-directory: airbyte-python-cdk
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
         run: |
-          cd airbyte
-          make tools.airbyte-ci.install
-          airbyte-ci \
-          --ci-report-bucket-name=airbyte-ci-reports-multi \
-          connectors \
-          --name ${{matrix.connector}} \
-          --use-local-cdk \
-          test \
-          --fail-fast \
-          --skip-step qa_checks \
-          --skip-step connector_live_tests
-
-      - name: Evaluate Test
-        id: evaluate_output
-        if: always() && steps.no_changes.outputs.status != 'cancelled'
+          poetry run airbyte-cdk secrets fetch ${{ matrix.connector }}
+
+      - name: Bump to Dev Branch CDK [Python Connectors]
+        if: env.CONNECTOR_LANGUAGE == 'python'
+        working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }}
         run: |
-          # save job output json file as ci step output
-          json_output_file=$(find airbyte/airbyte-ci/connectors/pipelines/pipeline_reports -name 'output.json' -print -quit)
-          job_output=$(cat ${json_output_file})
-          success=$(echo ${job_output} | jq -r '.success')
-          failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"')
-          run_duration=$(echo ${job_output} | jq -r '.run_duration')
-          html_report_url=$(echo ${job_output} | jq -r '.html_report_url')
-          echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY
-          echo "- [HTML Report](${html_report_url})" >> $GITHUB_STEP_SUMMARY
-          echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY
-          echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY
-          if [ "${success}" != "true" ]; then
-            echo "- Failed Step: ${failed_step}" >> $GITHUB_STEP_SUMMARY
-          fi
-          echo -e "\n[Download Job Output](${{steps.upload_job_output.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY
-          if [ "${success}" != "true" ]; then
-            echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. "
-            exit 1
-          fi
-          echo "See the execution report for details: ${html_report_url}"
-          echo "success=${success}" >> $GITHUB_OUTPUT
-          echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT
+          echo "Using CDK ref ${{ github.event.pull_request.head.sha || github.sha }}"
+          poe use-cdk-branch ${{ github.event.pull_request.head.sha || github.sha }}
+          poetry install --all-extras
 
-      # Update the test report status with results
-      - name: Update Test Report Status
-        if: always() && steps.no_changes.outputs.status != 'cancelled' && steps.evaluate_output.outcome == 'success'
+      - name: Run Unit Tests [Python Connectors]
+        if: env.CONNECTOR_LANGUAGE == 'python'
+        working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }}
+        run: |
+          poe test-unit-tests
+
+      - name: Run FAST Standard Tests [Python Connectors]
+        if: env.CONNECTOR_LANGUAGE == 'python'
+        working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }}
         env:
-          GH_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
         run: |
-          HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
-          gh api \
-            --method POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            repos/${{ github.repository }}/statuses/$HEAD_SHA \
-            -f state="${{ steps.evaluate_output.outputs.success == 'true' && 'success' || 'failure' }}" \
-            -f target_url="${{ steps.evaluate_output.outputs.html_report_url }}" \
-            -f description="Click Details to view the test report" \
-            -f context="${{ matrix.connector }} Test Report"
-
-      # Create failure status if report generation failed
-      - name: Create Report Generation Failed Status
-        if: always() && steps.no_changes.outputs.status != 'cancelled' && steps.evaluate_output.outcome != 'success'
+          poetry run airbyte-cdk connector test
+
+      - name: Run FAST Standard Tests [Manifest-Only Connectors]
+        if: env.CONNECTOR_LANGUAGE == 'manifest-only'
+        working-directory: airbyte-python-cdk
         env:
-          GH_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
         run: |
-          HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
-          gh api \
-            --method POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            repos/${{ github.repository }}/statuses/$HEAD_SHA \
-            -f state="failure" \
-            -f description="Failed to run connector tests." \
-            -f context="${{ matrix.connector }} Test Report"
-
-      # Upload the job output to the artifacts
-      - name: Upload Job Output
-        id: upload_job_output
-        if: always() && steps.no_changes.outputs.status != 'cancelled'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{matrix.connector}}-job-output
-          path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports
+          poetry run airbyte-cdk connector test ${{ matrix.connector }}
+
+      - name: Container Build and Test [All Connectors]
+        if: steps.no_changes.outputs.status != 'cancelled'
+        working-directory: airbyte-python-cdk
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
+        run: |
+          poetry run airbyte-cdk image test ${{ matrix.connector }}

.github/workflows/pypi_publish.yml

@@ -292,7 +292,6 @@ jobs:
             set -euo pipefail
             PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
             sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile"
-            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile"
             sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
             echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
             cd oss/airbyte-connector-builder-server

.github/workflows/pytest_fast.yml

@@ -70,6 +70,15 @@ jobs:
           poetry run coverage html -d htmlcov
           poetry run coverage xml -o htmlcov/coverage.xml
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled()
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (Fast)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Upload coverage to GitHub Artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/pytest_matrix.yml

@@ -93,6 +93,15 @@ jobs:
           poetry run coverage html -d htmlcov
           poetry run coverage xml -o htmlcov/coverage.xml
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled() && steps.changes.outputs.src == 'true' && matrix.python-version == '3.11'
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (Full)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Upload coverage to GitHub Artifacts
         if: always() && steps.changes.outputs.src == 'true'
         uses: actions/upload-artifact@v4

.github/workflows/python_lint.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Install dependencies
         run: poetry install --all-extras
 
-      # Job-specifc step(s):
+      # Job-specific step(s):
       - name: Run lint check
         run: poetry run ruff check .
 
@@ -49,9 +49,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --all-extras
 
-      # Job-specifc step(s):
+      # Job-specific step(s):
       - name: Check code format
-        run: poetry run ruff format --check .
+        run: poetry run ruff format --diff .
 
   mypy-check:
     name: MyPy Check

.github/workflows/test-command.yml

@@ -113,6 +113,15 @@ jobs:
           --verbose
           -m "flaky and not super_slow"
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always() && !cancelled() && matrix.python-version == '3.11'
+        continue-on-error: true
+        with:
+          check_name: "PyTest Results (On-Demand)"
+          large_files: true
+          files: |
+            build/test-results/**/*.xml
       - name: Post CI Success to GitHub
         run: |
           curl --request POST \