feat(metadata-service): Add schema support for smoke test scenarios

Add JSON schema definitions to support the new smoke tests configuration
format introduced in airbytehq/airbyte-python-cdk#775.

Changes:
- Created SmokeTestScenario.yaml schema defining the structure for individual
  smoke test scenarios with properties: name, config_file, config_settings,
  expect_failure, only_streams, exclude_streams, suggested_streams_only,
  and configured_catalog_path
- Updated ConnectorTestSuiteOptions.yaml to add 'smokeTests' to the suite enum
  and added 'scenarios' property to support the new smoke test format
- Regenerated Python models and bundled JSON schema

This enables connectors to define smoke tests in their metadata.yaml file under
data.connectorTestSuitesOptions with the following structure:

  connectorTestSuitesOptions:
    - suite: smokeTests
      scenarios:
        - name: default
          config_file: secrets/config_oauth.json
          config_settings:
            start_date: "2025-01-01T00:00:00Z"
          only_streams:
            - users

Related: airbytehq/airbyte-python-cdk#775
Co-Authored-By: AJ Steers <aj@airbyte.io>

Author: devin-ai-integration[bot]

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json

@@ -72,7 +72,8 @@
                   "unitTests",
                   "integrationTests",
                   "acceptanceTests",
-                  "liveTests"
+                  "liveTests",
+                  "smokeTests"
                 ]
               },
               "testSecrets": {
@@ -147,6 +148,64 @@
                     }
                   }
                 }
+              },
+              "scenarios": {
+                "description": "List of smoke test scenarios (only applicable when suite is 'smokeTests')",
+                "type": "array",
+                "items": {
+                  "$schema": "http://json-schema.org/draft-07/schema#",
+                  "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml",
+                  "title": "SmokeTestScenario",
+                  "description": "A single smoke test scenario configuration for a connector.",
+                  "type": "object",
+                  "required": [
+                    "name"
+                  ],
+                  "additionalProperties": false,
+                  "properties": {
+                    "name": {
+                      "description": "Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')",
+                      "type": "string"
+                    },
+                    "config_file": {
+                      "description": "Relative path to the config file to use for this scenario",
+                      "type": "string"
+                    },
+                    "config_settings": {
+                      "description": "Optional dictionary of config settings to override or supplement config_file settings",
+                      "type": "object",
+                      "additionalProperties": true
+                    },
+                    "expect_failure": {
+                      "description": "Whether the scenario is expected to fail",
+                      "type": "boolean",
+                      "default": false
+                    },
+                    "only_streams": {
+                      "description": "List of stream names to include in the scenario (if specified, only these streams will be tested)",
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "exclude_streams": {
+                      "description": "List of stream names to exclude from the scenario",
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "suggested_streams_only": {
+                      "description": "Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)",
+                      "type": "boolean",
+                      "default": false
+                    },
+                    "configured_catalog_path": {
+                      "description": "Path to a pre-configured catalog file for the scenario",
+                      "type": "string"
+                    }
+                  }
+                }
               }
             }
           }

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py

@@ -38,6 +38,40 @@ class Config:
     id: str = Field(..., description="The connection ID")
 
 
+class SmokeTestScenario(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: str = Field(
+        ...,
+        description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')",
+    )
+    config_file: Optional[str] = Field(
+        None, description="Relative path to the config file to use for this scenario"
+    )
+    config_settings: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Optional dictionary of config settings to override or supplement config_file settings",
+    )
+    expect_failure: Optional[bool] = Field(
+        False, description="Whether the scenario is expected to fail"
+    )
+    only_streams: Optional[List[str]] = Field(
+        None,
+        description="List of stream names to include in the scenario (if specified, only these streams will be tested)",
+    )
+    exclude_streams: Optional[List[str]] = Field(
+        None, description="List of stream names to exclude from the scenario"
+    )
+    suggested_streams_only: Optional[bool] = Field(
+        False,
+        description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)",
+    )
+    configured_catalog_path: Optional[str] = Field(
+        None, description="Path to a pre-configured catalog file for the scenario"
+    )
+
+
 class ReleaseStage(BaseModel):
     __root__: Literal["alpha", "beta", "generally_available", "custom"] = Field(
         ...,
@@ -278,16 +312,20 @@ class ConnectorTestSuiteOptions(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    suite: Literal["unitTests", "integrationTests", "acceptanceTests", "liveTests"] = (
-        Field(..., description="Name of the configured test suite")
-    )
+    suite: Literal[
+        "unitTests", "integrationTests", "acceptanceTests", "liveTests", "smokeTests"
+    ] = Field(..., description="Name of the configured test suite")
     testSecrets: Optional[List[Secret]] = Field(
         None, description="List of secrets required to run the test suite"
     )
     testConnections: Optional[List[TestConnections]] = Field(
         None,
         description="List of sandbox cloud connections that tests can be run against",
     )
+    scenarios: Optional[List[SmokeTestScenario]] = Field(
+        None,
+        description="List of smoke test scenarios (only applicable when suite is 'smokeTests')",
+    )
 
 
 class ActorDefinitionResourceRequirements(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorTestSuiteOptions.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from typing import List, Literal, Optional
+from typing import Any, Dict, List, Literal, Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -29,6 +29,40 @@ class Config:
     id: str = Field(..., description="The connection ID")
 
 
+class SmokeTestScenario(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: str = Field(
+        ...,
+        description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')",
+    )
+    config_file: Optional[str] = Field(
+        None, description="Relative path to the config file to use for this scenario"
+    )
+    config_settings: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Optional dictionary of config settings to override or supplement config_file settings",
+    )
+    expect_failure: Optional[bool] = Field(
+        False, description="Whether the scenario is expected to fail"
+    )
+    only_streams: Optional[List[str]] = Field(
+        None,
+        description="List of stream names to include in the scenario (if specified, only these streams will be tested)",
+    )
+    exclude_streams: Optional[List[str]] = Field(
+        None, description="List of stream names to exclude from the scenario"
+    )
+    suggested_streams_only: Optional[bool] = Field(
+        False,
+        description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)",
+    )
+    configured_catalog_path: Optional[str] = Field(
+        None, description="Path to a pre-configured catalog file for the scenario"
+    )
+
+
 class Secret(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -45,13 +79,17 @@ class ConnectorTestSuiteOptions(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    suite: Literal["unitTests", "integrationTests", "acceptanceTests", "liveTests"] = (
-        Field(..., description="Name of the configured test suite")
-    )
+    suite: Literal[
+        "unitTests", "integrationTests", "acceptanceTests", "liveTests", "smokeTests"
+    ] = Field(..., description="Name of the configured test suite")
     testSecrets: Optional[List[Secret]] = Field(
         None, description="List of secrets required to run the test suite"
     )
     testConnections: Optional[List[TestConnections]] = Field(
         None,
         description="List of sandbox cloud connections that tests can be run against",
     )
+    scenarios: Optional[List[SmokeTestScenario]] = Field(
+        None,
+        description="List of smoke test scenarios (only applicable when suite is 'smokeTests')",
+    )

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SmokeTestScenario.py

@@ -0,0 +1,42 @@
+# generated by datamodel-codegen:
+#   filename:  SmokeTestScenario.yaml
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Extra, Field
+
+
+class SmokeTestScenario(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: str = Field(
+        ...,
+        description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')",
+    )
+    config_file: Optional[str] = Field(
+        None, description="Relative path to the config file to use for this scenario"
+    )
+    config_settings: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Optional dictionary of config settings to override or supplement config_file settings",
+    )
+    expect_failure: Optional[bool] = Field(
+        False, description="Whether the scenario is expected to fail"
+    )
+    only_streams: Optional[List[str]] = Field(
+        None,
+        description="List of stream names to include in the scenario (if specified, only these streams will be tested)",
+    )
+    exclude_streams: Optional[List[str]] = Field(
+        None, description="List of stream names to exclude from the scenario"
+    )
+    suggested_streams_only: Optional[bool] = Field(
+        False,
+        description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)",
+    )
+    configured_catalog_path: Optional[str] = Field(
+        None, description="Path to a pre-configured catalog file for the scenario"
+    )

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py

@@ -25,6 +25,7 @@
 from .RolloutConfiguration import *
 from .Secret import *
 from .SecretStore import *
+from .SmokeTestScenario import *
 from .SourceFileInfo import *
 from .SuggestedStreams import *
 from .SupportLevel import *

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestSuiteOptions.yaml

@@ -16,6 +16,7 @@ properties:
       - "integrationTests"
       - "acceptanceTests"
       - "liveTests"
+      - "smokeTests"
   testSecrets:
     description: "List of secrets required to run the test suite"
     type: array
@@ -26,3 +27,8 @@ properties:
     type: array
     items:
       "$ref": "TestConnections.yaml"
+  scenarios:
+    description: "List of smoke test scenarios (only applicable when suite is 'smokeTests')"
+    type: array
+    items:
+      "$ref": "SmokeTestScenario.yaml"

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml

@@ -0,0 +1,41 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml
+title: SmokeTestScenario
+description: A single smoke test scenario configuration for a connector.
+type: object
+required:
+  - name
+additionalProperties: false
+properties:
+  name:
+    description: "Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')"
+    type: string
+  config_file:
+    description: "Relative path to the config file to use for this scenario"
+    type: string
+  config_settings:
+    description: "Optional dictionary of config settings to override or supplement config_file settings"
+    type: object
+    additionalProperties: true
+  expect_failure:
+    description: "Whether the scenario is expected to fail"
+    type: boolean
+    default: false
+  only_streams:
+    description: "List of stream names to include in the scenario (if specified, only these streams will be tested)"
+    type: array
+    items:
+      type: string
+  exclude_streams:
+    description: "List of stream names to exclude from the scenario"
+    type: array
+    items:
+      type: string
+  suggested_streams_only:
+    description: "Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)"
+    type: boolean
+    default: false
+  configured_catalog_path:
+    description: "Path to a pre-configured catalog file for the scenario"
+    type: string