Merge branch 'main' into tolik0/file-based/fix-memory-issue-in-excel-parser

Author: aaronsteers

.github/actions/check-docker-tag/action.yml

@@ -25,4 +25,4 @@ runs:
           echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
           exit 1
         fi
-        echo "No existing tag '$tag' found. Proceeding with publish."
+        echo "No existing tag '$tag' found. Proceeding with publish."

.github/workflows/pytest_matrix.yml

@@ -28,10 +28,11 @@ jobs:
     strategy:
       matrix:
         python-version: [
-            "3.10",
-            "3.11",
-            # "3.12",  # `requests-cache` blocker: https://github.com/airbytehq/airbyte-python-cdk/issues/299
-          ]
+          "3.10",
+          "3.11",
+          "3.12",
+          "3.13",
+        ]
         os: [
             Ubuntu,
             # Windows,  # For now, we don't include Windows in the test matrix.

.github/workflows/test-command.yml

@@ -50,7 +50,12 @@ jobs:
     needs: [start-workflow]
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: [
+          "3.10",
+          "3.11",
+          "3.12",
+          "3.13",
+        ]
         os: [
             Ubuntu,
             # Windows,  # For now, we don't include Windows in the test matrix.

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -62,10 +62,10 @@ def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
 
 def create_source(
     config: Mapping[str, Any],
-    limits: TestLimits,
-    catalog: Optional[ConfiguredAirbyteCatalog],
-    state: Optional[List[AirbyteStateMessage]],
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+    limits: TestLimits | None = None,
+    catalog: ConfiguredAirbyteCatalog | None = None,
+    state: List[AirbyteStateMessage] | None = None,
+) -> ConcurrentDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
 
     # We enforce a concurrency level of 1 so that the stream is processed on a single thread
@@ -88,7 +88,7 @@ def create_source(
 
 
 def read_stream(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
@@ -127,7 +127,7 @@ def read_stream(
 
 
 def resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
 ) -> AirbyteMessage:
     try:
         return AirbyteMessage(
@@ -146,7 +146,7 @@ def resolve_manifest(
 
 
 def full_resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]], limits: TestLimits
+    source: ConcurrentDeclarativeSource, limits: TestLimits
 ) -> AirbyteMessage:
     try:
         manifest = {**source.resolved_manifest}

airbyte_cdk/connector_builder/main.py

@@ -34,7 +34,7 @@
 
 def get_config_and_catalog_from_args(
     args: List[str],
-) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]:
+) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]]:
     # TODO: Add functionality for the `debug` logger.
     #  Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`.
     parsed_args = AirbyteEntrypoint.parse_args(args)
@@ -70,7 +70,7 @@ def get_config_and_catalog_from_args(
 
 
 def handle_connector_builder_request(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     command: str,
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],

airbyte_cdk/connector_builder/test_reader/reader.py

@@ -85,7 +85,7 @@ def __init__(
 
     def run_test_read(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         stream_name: str,
@@ -383,7 +383,7 @@ def _get_latest_config_update(
 
     def _read_stream(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         state: List[AirbyteStateMessage],

airbyte_cdk/manifest_server/Dockerfile

@@ -11,7 +11,7 @@ FROM python:3.12-slim-bookworm
 RUN apt-get update && \
     apt-get install -y git && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install poetry==1.8.3
+    pip install poetry==2.0.1
 
 # Configure poetry to not create virtual environments and disable interactive mode
 ENV POETRY_NO_INTERACTION=1 \
@@ -42,4 +42,4 @@ USER airbyte:airbyte
 
 EXPOSE 8080
 
-CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

airbyte_cdk/manifest_server/README.md

@@ -37,26 +37,31 @@ The server will start on `http://localhost:8000` by default.
 ## API Endpoints
 
 ### `/v1/manifest/test_read`
+
 Test reading from a specific stream in the manifest.
 
 **POST** - Test stream reading with configurable limits for records, pages, and slices.
 
 ### `/v1/manifest/check`
+
 Check configuration against a manifest.
 
 **POST** - Validates connector configuration and returns success/failure status with message.
 
 ### `/v1/manifest/discover`
+
 Discover streams from a manifest.
 
 **POST** - Returns the catalog of available streams from the manifest.
 
-### `/v1/manifest/resolve` 
+### `/v1/manifest/resolve`
+
 Resolve a manifest to its final configuration.
 
 **POST** - Returns the resolved manifest without dynamic stream generation.
 
 ### `/v1/manifest/full_resolve`
+
 Fully resolve a manifest including dynamic streams.
 
 **POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
@@ -68,6 +73,7 @@ The manifest server supports custom Python components, but this feature is **dis
 ### Enabling Custom Components
 
 To allow custom Python components in your manifest files, set the environment variable:
+
 ```bash
 export AIRBYTE_ENABLE_UNSAFE_CODE=true
 ```
@@ -77,27 +83,33 @@ export AIRBYTE_ENABLE_UNSAFE_CODE=true
 The manifest server supports optional JWT bearer token authentication:
 
 ### Configuration
+
 Set the environment variable to enable authentication:
+
 ```bash
 export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
 ```
 
 ### Usage
+
 When authentication is enabled, include a valid JWT token in the Authorization header:
+
 ```bash
 curl -H "Authorization: Bearer <your-jwt-token>" \
   http://localhost:8000/v1/manifest/test_read
 ```
 
 ### Behavior
-- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through
 - **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
 
 ## OpenAPI Specification
 
 The manifest server provides an OpenAPI specification for API client generation:
 
 ### Generating the OpenAPI Spec
+
 ```bash
 # Generate OpenAPI YAML (default location)
 manifest-server generate-openapi
@@ -107,6 +119,7 @@ manifest-server generate-openapi --output /path/to/openapi.yaml
 ```
 
 The generated OpenAPI specification is consumed by other applications and tools to:
+
 - Generate API clients in various programming languages
 - Create SDK bindings for the manifest server
 - Provide API documentation and validation
@@ -115,6 +128,7 @@ The generated OpenAPI specification is consumed by other applications and tools
 ### Interactive API Documentation
 
 When running, interactive API documentation is available at:
+
 - Swagger UI: `http://localhost:8000/docs`
 - ReDoc: `http://localhost:8000/redoc`
 
@@ -139,4 +153,26 @@ docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
 docker run -p 8080:8080 manifest-server
 ```
 
-Note: The container runs on port 8080 by default.
+Note: The container runs on port 8080 by default.
+
+## Datadog APM
+
+The manifest server supports Datadog APM tracing for monitoring and observability:
+
+### Configuration
+
+To enable Datadog tracing, set the environment variable:
+
+```bash
+export DD_ENABLED=true
+```
+
+This requires the `ddtrace` dependency, which is included in the `manifest-server` extra. For additional configuration options via environment variables, see [ddtrace configuration](https://ddtrace.readthedocs.io/en/stable/configuration.html).
+
+### Usage
+
+```bash
+# Run with Datadog tracing enabled
+DD_ENABLED=true manifest-server start
+```
+

airbyte_cdk/manifest_server/app.py

@@ -1,3 +1,9 @@
+import os
+
+if os.getenv("DD_ENABLED", "false").lower() == "true":
+    # Auto-instrumentation should be imported as early as possible.
+    import ddtrace.auto  # noqa: F401
+
 from fastapi import FastAPI
 
 from .routers import capabilities, health, manifest

airbyte_cdk/manifest_server/cli/_common.py

@@ -7,6 +7,7 @@
 
 # Import server dependencies with graceful fallback
 try:
+    import ddtrace  # noqa: F401
     import fastapi  # noqa: F401
     import uvicorn  # noqa: F401