Metadata: add tags field (#26320)

* Add optional tags field * Remove duplicate icons * Add programming tags to all * Update docs * supportUrl -> documentationUrl * Ensure one language tag is applied * Add keyvalue check * rebase and fix tests * Format * Add cache buster * Improve test * Automated Commit - Formatting Changes * Update error * Fix missing tags * Fix scaffold --------- Co-authored-by: Octavia Squidington III <octavia-squidington-iii@sers.noreply.github.com> Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>
airbytehq · May 26, 2023 · 1dabc62 · 1dabc62
1 parent c5d7961
commit 1dabc62
Show file tree

Hide file tree

Showing 410 changed files with 1,448 additions and 413 deletions.
diff --git a/...s/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/...s/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py
@@ -105,7 +105,7 @@ class Config:
     supportsDbt: Optional[bool] = None
     supportsNormalization: Optional[bool] = None
     license: Optional[str] = None
-    supportUrl: Optional[AnyUrl] = None
+    documentationUrl: Optional[AnyUrl] = None
     connectorSubtype: Optional[str] = None
     allowedHosts: Optional[AllowedHosts] = None
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None
@@ -131,7 +131,7 @@ class Data(BaseModel):
     supportsDbt: Optional[bool] = None
     supportsNormalization: Optional[bool] = None
     license: str
-    supportUrl: AnyUrl
+    documentationUrl: Optional[AnyUrl] = None
     githubIssueLabel: str
     maxSecondsBetweenMessages: Optional[int] = Field(
         None,
@@ -148,6 +148,10 @@ class Data(BaseModel):
         "api", "database", "file", "custom", "message_queue", "unknown"
     ]
     releaseStage: Literal["alpha", "beta", "generally_available", "source"]
+    tags: Optional[List[str]] = Field(
+        None,
+        description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
+    )
     registries: Optional[Registry] = None
     allowedHosts: Optional[AllowedHosts] = None
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None

diff --git a/...a_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py b/...a_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py
@@ -120,6 +120,10 @@ class Config:
         None,
         description="The date when this connector was first released, in yyyy-mm-dd format.",
     )
+    tags: Optional[List[str]] = Field(
+        None,
+        description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
+    )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
     protocolVersion: Optional[str] = Field(
         None, description="the Airbyte Protocol version supported by the connector"

diff --git a/.../connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py b/.../connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py
@@ -171,6 +171,10 @@ class Config:
         None,
         description="The date when this connector was first released, in yyyy-mm-dd format.",
     )
+    tags: Optional[List[str]] = Field(
+        None,
+        description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
+    )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
     protocolVersion: Optional[str] = Field(
         None, description="the Airbyte Protocol version supported by the connector"

diff --git a/...ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py b/...ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py
@@ -103,7 +103,7 @@ class Config:
     supportsDbt: Optional[bool] = None
     supportsNormalization: Optional[bool] = None
     license: Optional[str] = None
-    supportUrl: Optional[AnyUrl] = None
+    documentationUrl: Optional[AnyUrl] = None
     connectorSubtype: Optional[str] = None
     allowedHosts: Optional[AllowedHosts] = None
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None

diff --git a/...ctors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml b/...ctors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml
@@ -48,7 +48,7 @@ properties:
         type: boolean
       license:
         type: string
-      supportUrl:
+      documentationUrl:
         type: string
         format: uri
       githubIssueLabel:
@@ -79,7 +79,11 @@ properties:
           - beta
           - generally_available
           - source
-
+      tags:
+        type: array
+        description: "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc."
+        items:
+          type: string
       registries:
         anyOf:
           - type: object

diff --git a/...adata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml b/...adata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml
@@ -49,6 +49,11 @@ properties:
     description: The date when this connector was first released, in yyyy-mm-dd format.
     type: string
     format: date
+  tags:
+    type: array
+    description: "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc."
+    items:
+      type: string
   resourceRequirements:
     "$ref": ActorDefinitionResourceRequirements.yaml
   protocolVersion:

diff --git a/...yte-ci/connectors/metadata_service/lib/metadata_service/models/src/RegistryOverrides.yaml b/...yte-ci/connectors/metadata_service/lib/metadata_service/models/src/RegistryOverrides.yaml
@@ -23,7 +23,7 @@ properties:
     type: boolean
   license:
     type: string
-  supportUrl:
+  documentationUrl:
     type: string
     format: uri
   connectorSubtype:

diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py
@@ -42,13 +42,47 @@ def validate_metadata_images_in_dockerhub(metadata_definition: ConnectorMetadata
     return True, None
 
 
+def validate_at_least_one_langauge_tag(metadata_definition: ConnectorMetadataDefinitionV0) -> ValidationResult:
+    """Ensure that there is at least one tag in the data.tags field that matches language:<LANG>."""
+    if not any([tag.startswith("language:") for tag in metadata_definition.data.tags]):
+        return False, "At least one tag must be of the form language:<LANG>"
+
+    return True, None
+
+
+def validate_all_tags_are_keyvalue_pairs(metadata_definition: ConnectorMetadataDefinitionV0) -> ValidationResult:
+    """Ensure that all tags are of the form <KEY>:<VALUE>."""
+    for tag in metadata_definition.data.tags:
+        if ":" not in tag:
+            return False, f"Tag {tag} is not of the form <KEY>:<VALUE>"
+
+    return True, None
+
+
+def pre_upload_validations(metadata_definition: ConnectorMetadataDefinitionV0) -> ValidationResult:
+    """
+    Runs all validations that should be run before uploading a connector to the registry.
+    """
+    validations = [
+        validate_all_tags_are_keyvalue_pairs,
+        validate_at_least_one_langauge_tag,
+    ]
+
+    for validation in validations:
+        valid, error = validation(metadata_definition)
+        if not valid:
+            return False, error
+
+    return True, None
+
+
 def validate_metadata_file(file_path: pathlib.Path) -> ValidationResult:
     """
     Validates a metadata YAML file against a metadata Pydantic model.
     """
     try:
         metadata = yaml.safe_load(file_path.read_text())
-        ConnectorMetadataDefinitionV0.parse_obj(metadata)
-        return True, None
+        metadata_model = ConnectorMetadataDefinitionV0.parse_obj(metadata)
+        return pre_upload_validations(metadata_model)
     except ValidationError as e:
         return False, e
diff --git a/...rvice/lib/tests/fixtures/metadata_upload/invalid/metadata_cloud_image_does_not_exist.yaml b/...rvice/lib/tests/fixtures/metadata_upload/invalid/metadata_cloud_image_does_not_exist.yaml
@@ -23,3 +23,5 @@ data:
       enabled: true
       dockerRepository: airbyte/image-exists-3
       dockerImageTag: 0.0.1-exists
+  tags:
+    - language:java
diff --git a/...ervice/lib/tests/fixtures/metadata_upload/invalid/metadata_main_image_does_not_exist.yaml b/...ervice/lib/tests/fixtures/metadata_upload/invalid/metadata_main_image_does_not_exist.yaml
@@ -19,3 +19,5 @@ data:
       enabled: true
     oss:
       enabled: true
+  tags:
+    - language:java
diff --git a/...b/tests/fixtures/metadata_upload/invalid/metadata_normalization_image_does_not_exist.yaml b/...b/tests/fixtures/metadata_upload/invalid/metadata_normalization_image_does_not_exist.yaml
@@ -23,3 +23,5 @@ data:
       enabled: true
       dockerRepository: airbyte/image-exists-4
       dockerImageTag: 0.0.1-exists
+  tags:
+    - language:java
diff --git a/...service/lib/tests/fixtures/metadata_upload/invalid/metadata_oss_image_does_not_exist.yaml b/...service/lib/tests/fixtures/metadata_upload/invalid/metadata_oss_image_does_not_exist.yaml
@@ -23,3 +23,5 @@ data:
       enabled: true
       dockerRepository: airbyte/image-does-not-exist-4
       dockerImageTag: 0.0.1-exists
+  tags:
+    - language:java
diff --git a/.../metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist.yaml b/.../metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist.yaml
@@ -23,3 +23,5 @@ data:
       enabled: true
       dockerRepository: airbyte/image-exists-4
       dockerImageTag: 0.0.1-exists
+  tags:
+    - language:java
diff --git a/...ts/fixtures/metadata_upload/valid/metadata_main_image_does_not_exist_but_is_overrode.yaml b/...ts/fixtures/metadata_upload/valid/metadata_main_image_does_not_exist_but_is_overrode.yaml
@@ -23,3 +23,5 @@ data:
       enabled: true
       dockerRepository: airbyte/image-exists-4
       dockerImageTag: 0.0.1-exists
+  tags:
+    - language:java
diff --git a/...rs/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_empty_tags.yaml b/...rs/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_empty_tags.yaml
@@ -0,0 +1,13 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/source-alloydb-strict-encrypt
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 2.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  tags: []
diff --git a/...ata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_tag_format.yaml b/...ata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_tag_format.yaml
@@ -0,0 +1,14 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/source-alloydb-strict-encrypt
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 2.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  tags:
+    - notkv
diff --git a/...service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_connector_type.yaml b/...service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_connector_type.yaml
@@ -5,7 +5,9 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
+  tags:
+    - language:java
diff --git a/..._service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_definition_id.yaml b/..._service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_definition_id.yaml
@@ -5,11 +5,13 @@ data:
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerRepository: airbyte/image-exists-1
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
   allowedHosts:
     hosts:
       - "${host}"
       - "${tunnel_method.tunnel_host}"
+  tags:
+    - language:java
diff --git a/...a_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_image.yaml b/...a_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_image.yaml
@@ -5,11 +5,13 @@ data:
   connectorType: source
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerRepository: airbyte/image-exists-1
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
   allowedHosts:
     hosts:
       - "${host}"
       - "${tunnel_method.tunnel_host}"
+  tags:
+    - language:java
diff --git a/...service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_version.yaml b/...service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_version.yaml
@@ -5,11 +5,13 @@ data:
   connectorType: source
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerRepository: airbyte/image-exists-1
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
   allowedHosts:
     hosts:
       - "${host}"
       - "${tunnel_method.tunnel_host}"
+  tags:
+    - language:java
diff --git a/...tadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_version.yaml b/...tadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_version.yaml
@@ -5,11 +5,13 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
   allowedHosts:
     hosts:
       - "${host}"
       - "${tunnel_method.tunnel_host}"
+  tags:
+    - language:java
diff --git a/...s/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_lang_tag.yaml b/...s/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_lang_tag.yaml
@@ -0,0 +1,14 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/source-alloydb-strict-encrypt
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 2.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  tags:
+    - keyword:test
diff --git a/...ctors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_tags.yaml b/...ctors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_tags.yaml
@@ -0,0 +1,12 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/source-alloydb-strict-encrypt
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 2.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
diff --git a/...ervice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_id_override.yaml b/...ervice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_id_override.yaml
@@ -6,7 +6,7 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
@@ -18,3 +18,5 @@ data:
     oss:
       enabled: true
       definitionId: woohoo
+  tags:
+    - language:java
diff --git a/...vice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_type_override.yaml b/...vice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_type_override.yaml
@@ -6,7 +6,7 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
@@ -18,3 +18,5 @@ data:
     oss:
       enabled: true
       connectorType: destination
+  tags:
+    - language:java
diff --git a/...vice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_unknown_override.yaml b/...vice/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_unknown_override.yaml
@@ -6,7 +6,7 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
@@ -18,3 +18,5 @@ data:
     oss:
       enabled: true
       what: is this?
+  tags:
+    - language:java
diff --git a/...a_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_allowed_hosts.yaml b/...a_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_allowed_hosts.yaml
@@ -6,11 +6,13 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
   allowedHosts:
     hosts:
       - "${host}"
       - "${tunnel_method.tunnel_host}"
+  tags:
+    - language:java
diff --git a/...ervice/lib/tests/fixtures/metadata_validate/valid/metadata_registry_complex_override.yaml b/...ervice/lib/tests/fixtures/metadata_validate/valid/metadata_registry_complex_override.yaml
@@ -6,7 +6,7 @@ data:
   dockerRepository: airbyte/image-exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
   dockerImageTag: 0.0.1-exists
-  supportUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   connectorSubtype: database
   releaseStage: generally_available
   license: MIT
@@ -27,3 +27,5 @@ data:
             resourceRequirements:
               memory_request: 1Gi
               memory_limit: 1Gi
+  tags:
+    - language:java