Merge branch 'main' into aj/ci/tests/new-smoke-tests-dsl-config-dialect

Author: aaronsteers

.github/workflows/publish.yml

@@ -342,8 +342,7 @@ jobs:
       url: https://github.com/airbytehq/airbyte-platform-internal/pulls?q=is%3Apr+automatic-cdk-release+
     needs:
       - build
-      - publish_cdk
-      - publish_sdm
+      - publish_manifest_server
     if: >
       (github.event_name == 'push' &&
        needs.build.outputs.IS_PRERELEASE == 'false' &&
@@ -355,9 +354,6 @@ jobs:
       IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
       - name: Authenticate as GitHub App
         uses: actions/create-github-app-token@v2
         id: get-app-token
@@ -372,24 +368,20 @@ jobs:
           repository: airbytehq/airbyte-platform-internal
           token: ${{ steps.get-app-token.outputs.token }}
       - name: Update Builder's CDK version to ${{ env.VERSION }}
-        # PyPI servers aren't immediately updated so we may need to retry a few times.
-        uses: nick-fields/retry@v3
-        with:
-          shell: bash
-          max_attempts: 5
-          retry_wait_seconds: 30
-          timeout_minutes: 7
-          command: |
-            set -euo pipefail
-            PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
-            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile"
-            sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
-            sed -i "s/tag: ${PREVIOUS_VERSION}/tag: ${VERSION}/g" "oss/charts/v2/airbyte/values.yaml"
-            sed -i "s/refs\/tags\/v${PREVIOUS_VERSION}/refs\/tags\/v${VERSION}/g" "oss/airbyte-api/manifest-server-api/build.gradle.kts"
-            echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
-            cd oss/airbyte-connector-builder-server
-            python -m pip install --no-cache-dir pip-tools
-            pip-compile --upgrade
+        # The manifest-server Docker image already includes the CDK version.
+        # We just need to update the image tag in the Helm values file.
+        run: |
+          set -euo pipefail
+          VALUES_FILE="oss/charts/v2/airbyte/values.yaml"
+          # Get the current manifest-server tag from the values.yaml file
+          PREVIOUS_VERSION=$(grep -A15 "^manifestServer:" "$VALUES_FILE" | grep -A4 "image:" | grep "tag:" | awk '{print $2}')
+          echo "Previous version: ${PREVIOUS_VERSION}"
+          echo "New version: ${VERSION}"
+          # Update the manifest-server tag in the Helm values file (preserves formatting)
+          sed -i "/^manifestServer:/,/^[a-zA-Z]/ s/tag: ${PREVIOUS_VERSION}/tag: ${VERSION}/" "$VALUES_FILE"
+          # Also update the CDK version in the manifest-server-api build file
+          sed -i "s/refs\/tags\/v${PREVIOUS_VERSION}/refs\/tags\/v${VERSION}/g" "oss/airbyte-api/manifest-server-api/build.gradle.kts"
+          echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
       - name: Create Pull Request
         id: create-pull-request
         uses: peter-evans/create-pull-request@v7

Dockerfile

@@ -5,7 +5,7 @@
 # A new version of source-declarative-manifest is built for every new Airbyte CDK release, and their versions are kept in sync.
 #
 
-FROM docker.io/airbyte/python-connector-base:4.0.2@sha256:9fdb1888c4264cf6fee473649ecb593f56f58e5d0096a87ee0b231777e2e3e73
+FROM docker.io/airbyte/python-connector-base:4.1.0@sha256:1d1aa21d34e851df4e8a87b391c27724c06e2597608e7161f4d167be853bd7b6
 
 WORKDIR /airbyte/integration_code
 
@@ -24,8 +24,8 @@ RUN pip install dist/*.whl
 RUN mkdir -p source_declarative_manifest \
     && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n    run()' > main.py \
     && touch source_declarative_manifest/__init__.py \
-    && cp /usr/local/lib/python3.11/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \
-    && cp /usr/local/lib/python3.11/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/
+    && cp /usr/local/lib/python3.13/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \
+    && cp /usr/local/lib/python3.13/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/
 
 # Remove unnecessary build files
 RUN rm -rf dist/ pyproject.toml poetry.lock README.md

airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -8,9 +8,9 @@
 from typing import Any, Dict, List, Mapping, Optional, Tuple
 
 import dpath
-from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
-from langchain.utils import stringify_dict
 from langchain_core.documents.base import Document
+from langchain_core.utils.strings import stringify_dict
+from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 
 from airbyte_cdk.destinations.vector_db_based.config import (
     ProcessingConfigModel,

airbyte_cdk/destinations/vector_db_based/embedder.py

@@ -7,10 +7,12 @@
 from dataclasses import dataclass
 from typing import List, Optional, Union, cast
 
-from langchain.embeddings.cohere import CohereEmbeddings
-from langchain.embeddings.fake import FakeEmbeddings
-from langchain.embeddings.localai import LocalAIEmbeddings
-from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain_community.embeddings import (
+    CohereEmbeddings,
+    FakeEmbeddings,
+    LocalAIEmbeddings,
+    OpenAIEmbeddings,
+)
 
 from airbyte_cdk.destinations.vector_db_based.config import (
     AzureOpenAIEmbeddingConfigModel,
@@ -140,7 +142,9 @@ def __init__(self, config: CohereEmbeddingConfigModel):
         super().__init__()
         # Client is set internally
         self.embeddings = CohereEmbeddings(
-            cohere_api_key=config.cohere_key, model="embed-english-light-v2.0"
+            cohere_api_key=config.cohere_key,
+            model="embed-english-light-v2.0",
+            user_agent="airbyte-cdk",
         )  # type: ignore
 
     def check(self) -> Optional[str]:

airbyte_cdk/manifest_server/Dockerfile

@@ -5,7 +5,7 @@
 # Example:
 # docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
 
-FROM python:3.12-slim-bookworm
+FROM python:3.13.9-slim-bookworm@sha256:4c9fe962f6ce46ecf3633a7e9d0a9fb7f5622121ee00d628eff206da024147c9
 
 # Install git (needed for dynamic versioning) and poetry
 RUN apt-get update && \

airbyte_cdk/manifest_server/api_models/__init__.py

@@ -10,6 +10,7 @@
     CheckResponse,
     DiscoverRequest,
     DiscoverResponse,
+    ErrorResponse,
     FullResolveRequest,
     ManifestResponse,
     RequestContext,
@@ -40,6 +41,7 @@
     "CheckResponse",
     "DiscoverRequest",
     "DiscoverResponse",
+    "ErrorResponse",
     # Stream models
     "AuxiliaryRequest",
     "HttpRequest",

airbyte_cdk/manifest_server/api_models/manifest.py

@@ -83,3 +83,9 @@ class FullResolveRequest(BaseModel):
     config: ConnectorConfig
     stream_limit: int = Field(default=100, ge=1, le=100)
     context: Optional[RequestContext] = None
+
+
+class ErrorResponse(BaseModel):
+    """Error response for API requests."""
+
+    detail: str

airbyte_cdk/manifest_server/app.py

@@ -11,7 +11,7 @@
 app = FastAPI(
     title="Manifest Server",
     description="A service for running low-code Airbyte connectors",
-    version="0.1.0",
+    version="0.2.0",
     contact={
         "name": "Airbyte",
         "url": "https://airbyte.com",

airbyte_cdk/manifest_server/command_processor/utils.py

@@ -64,16 +64,7 @@ def build_source(
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
 ) -> ConcurrentDeclarativeSource:
-    # We enforce a concurrency level of 1 so that the stream is processed on a single thread
-    # to retain ordering for the grouping of the builder message responses.
     definition = copy.deepcopy(manifest)
-    if "concurrency_level" in definition:
-        definition["concurrency_level"]["default_concurrency"] = 1
-    else:
-        definition["concurrency_level"] = {
-            "type": "ConcurrencyLevel",
-            "default_concurrency": 1,
-        }
 
     should_normalize = should_normalize_manifest(manifest)
     if should_normalize:

airbyte_cdk/manifest_server/openapi.yaml

@@ -8,7 +8,7 @@ info:
   contact:
     name: Airbyte
     url: https://airbyte.com/
-  version: 0.1.0
+  version: 0.2.0
 paths:
   /health/:
     get:
@@ -62,6 +62,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/StreamReadResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -90,6 +96,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/CheckResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -118,6 +130,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/DiscoverResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -146,6 +164,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -180,6 +204,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -353,6 +383,16 @@ components:
       - catalog
       title: DiscoverResponse
       description: Response to discover a manifest.
+    ErrorResponse:
+      properties:
+        detail:
+          type: string
+          title: Detail
+      type: object
+      required:
+      - detail
+      title: ErrorResponse
+      description: Error response for API requests.
     FullResolveRequest:
       properties:
         manifest: