Skip to content

Commit

Permalink
Add support refreshes to the metadata. (#37472)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benmoriceau
benmoriceau authored Apr 29, 2024
1 parent 3417fe8 commit bc7b4cb
Show file tree
Hide file tree
Showing 15 changed files with 131 additions and 339 deletions.
17 changes: 3 additions & 14 deletions ...data_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,8 @@ class Config:


class JobType(BaseModel):
__root__: Literal[
"get_spec",
"check_connection",
"discover_schema",
"sync",
"reset_connection",
"connection_updater",
"replicate",
] = Field(
...,
description="enum that describes the different types of jobs that the platform runs.",
title="JobType",
__root__: Literal["get_spec", "check_connection", "discover_schema", "sync", "reset_connection", "connection_updater", "replicate"] = (
Field(..., description="enum that describes the different types of jobs that the platform runs.", title="JobType")
)


Expand All @@ -48,7 +38,6 @@ class Config:
extra = Extra.forbid

default: Optional[ResourceRequirements] = Field(
None,
description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.",
None, description="if set, these are the requirements that should be set for ALL jobs run for this actor definition."
)
jobSpecific: Optional[List[JobTypeResourceLimit]] = None
83 changes: 18 additions & 65 deletions ...s/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,13 @@ class Config:

class ReleaseStage(BaseModel):
__root__: Literal["alpha", "beta", "generally_available", "custom"] = Field(
...,
description="enum that describes a connector's release stage",
title="ReleaseStage",
..., description="enum that describes a connector's release stage", title="ReleaseStage"
)


class SupportLevel(BaseModel):
__root__: Literal["community", "certified", "archived"] = Field(
...,
description="enum that describes a connector's release stage",
title="SupportLevel",
..., description="enum that describes a connector's release stage", title="SupportLevel"
)


Expand All @@ -52,13 +48,9 @@ class Config:
...,
description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.",
)
normalizationTag: str = Field(
...,
description="a field indicating the tag of the docker repository to be used for normalization.",
)
normalizationTag: str = Field(..., description="a field indicating the tag of the docker repository to be used for normalization.")
normalizationIntegrationType: str = Field(
...,
description="a field indicating the type of integration dialect to use for normalization.",
..., description="a field indicating the type of integration dialect to use for normalization."
)


Expand All @@ -83,18 +75,8 @@ class Config:


class JobType(BaseModel):
__root__: Literal[
"get_spec",
"check_connection",
"discover_schema",
"sync",
"reset_connection",
"connection_updater",
"replicate",
] = Field(
...,
description="enum that describes the different types of jobs that the platform runs.",
title="JobType",
__root__: Literal["get_spec", "check_connection", "discover_schema", "sync", "reset_connection", "connection_updater", "replicate"] = (
Field(..., description="enum that describes the different types of jobs that the platform runs.", title="JobType")
)


Expand All @@ -103,11 +85,7 @@ class Config:
extra = Extra.forbid

scopeType: Any = Field("stream", const=True)
impactedScopes: List[str] = Field(
...,
description="List of streams that are impacted by the breaking change.",
min_items=1,
)
impactedScopes: List[str] = Field(..., description="List of streams that are impacted by the breaking change.", min_items=1)


class AirbyteInternal(BaseModel):
Expand Down Expand Up @@ -135,10 +113,7 @@ class Config:


class BreakingChangeScope(BaseModel):
__root__: StreamBreakingChangeScope = Field(
...,
description="A scope that can be used to limit the impact of a breaking change.",
)
__root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")


class RemoteRegistries(BaseModel):
Expand All @@ -153,8 +128,7 @@ class Config:
extra = Extra.forbid

default: Optional[ResourceRequirements] = Field(
None,
description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.",
None, description="if set, these are the requirements that should be set for ALL jobs run for this actor definition."
)
jobSpecific: Optional[List[JobTypeResourceLimit]] = None

Expand All @@ -163,13 +137,8 @@ class VersionBreakingChange(BaseModel):
class Config:
extra = Extra.forbid

upgradeDeadline: date = Field(
...,
description="The deadline by which to upgrade before the breaking change takes effect.",
)
message: str = Field(
..., description="Descriptive message detailing the breaking change."
)
upgradeDeadline: date = Field(..., description="The deadline by which to upgrade before the breaking change takes effect.")
message: str = Field(..., description="Descriptive message detailing the breaking change.")
migrationDocumentationUrl: Optional[AnyUrl] = Field(
None,
description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
Expand Down Expand Up @@ -205,8 +174,7 @@ class Config:
extra = Extra.forbid

__root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field(
...,
description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.",
..., description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade."
)


Expand Down Expand Up @@ -246,31 +214,15 @@ class Config:
documentationUrl: AnyUrl
githubIssueLabel: str
maxSecondsBetweenMessages: Optional[int] = Field(
None,
description="Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached",
)
releaseDate: Optional[date] = Field(
None,
description="The date when this connector was first released, in yyyy-mm-dd format.",
)
protocolVersion: Optional[str] = Field(
None, description="the Airbyte Protocol version supported by the connector"
None, description="Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached"
)
connectorSubtype: Literal[
"api",
"database",
"datalake",
"file",
"custom",
"message_queue",
"unknown",
"vectorstore",
]
releaseDate: Optional[date] = Field(None, description="The date when this connector was first released, in yyyy-mm-dd format.")
protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
connectorSubtype: Literal["api", "database", "datalake", "file", "custom", "message_queue", "unknown", "vectorstore"]
releaseStage: ReleaseStage
supportLevel: Optional[SupportLevel] = None
tags: Optional[List[str]] = Field(
[],
description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
[], description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc."
)
registries: Optional[Registry] = None
allowedHosts: Optional[AllowedHosts] = None
Expand All @@ -280,6 +232,7 @@ class Config:
resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
ab_internal: Optional[AirbyteInternal] = None
remoteRegistries: Optional[RemoteRegistries] = None
supportsRefreshes: Optional[bool] = None


class ConnectorMetadataDefinitionV0(BaseModel):
Expand Down
80 changes: 18 additions & 62 deletions ...a_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,13 @@

class ReleaseStage(BaseModel):
__root__: Literal["alpha", "beta", "generally_available", "custom"] = Field(
...,
description="enum that describes a connector's release stage",
title="ReleaseStage",
..., description="enum that describes a connector's release stage", title="ReleaseStage"
)


class SupportLevel(BaseModel):
__root__: Literal["community", "certified", "archived"] = Field(
...,
description="enum that describes a connector's release stage",
title="SupportLevel",
..., description="enum that describes a connector's release stage", title="SupportLevel"
)


Expand All @@ -38,18 +34,8 @@ class Config:


class JobType(BaseModel):
__root__: Literal[
"get_spec",
"check_connection",
"discover_schema",
"sync",
"reset_connection",
"connection_updater",
"replicate",
] = Field(
...,
description="enum that describes the different types of jobs that the platform runs.",
title="JobType",
__root__: Literal["get_spec", "check_connection", "discover_schema", "sync", "reset_connection", "connection_updater", "replicate"] = (
Field(..., description="enum that describes the different types of jobs that the platform runs.", title="JobType")
)


Expand All @@ -61,13 +47,9 @@ class Config:
...,
description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.",
)
normalizationTag: str = Field(
...,
description="a field indicating the tag of the docker repository to be used for normalization.",
)
normalizationTag: str = Field(..., description="a field indicating the tag of the docker repository to be used for normalization.")
normalizationIntegrationType: str = Field(
...,
description="a field indicating the type of integration dialect to use for normalization.",
..., description="a field indicating the type of integration dialect to use for normalization."
)


Expand All @@ -86,11 +68,7 @@ class Config:
extra = Extra.forbid

scopeType: Any = Field("stream", const=True)
impactedScopes: List[str] = Field(
...,
description="List of streams that are impacted by the breaking change.",
min_items=1,
)
impactedScopes: List[str] = Field(..., description="List of streams that are impacted by the breaking change.", min_items=1)


class AirbyteInternal(BaseModel):
Expand All @@ -110,19 +88,15 @@ class Config:


class BreakingChangeScope(BaseModel):
__root__: StreamBreakingChangeScope = Field(
...,
description="A scope that can be used to limit the impact of a breaking change.",
)
__root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")


class ActorDefinitionResourceRequirements(BaseModel):
class Config:
extra = Extra.forbid

default: Optional[ResourceRequirements] = Field(
None,
description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.",
None, description="if set, these are the requirements that should be set for ALL jobs run for this actor definition."
)
jobSpecific: Optional[List[JobTypeResourceLimit]] = None

Expand All @@ -131,13 +105,8 @@ class VersionBreakingChange(BaseModel):
class Config:
extra = Extra.forbid

upgradeDeadline: date = Field(
...,
description="The deadline by which to upgrade before the breaking change takes effect.",
)
message: str = Field(
..., description="Descriptive message detailing the breaking change."
)
upgradeDeadline: date = Field(..., description="The deadline by which to upgrade before the breaking change takes effect.")
message: str = Field(..., description="Descriptive message detailing the breaking change.")
migrationDocumentationUrl: Optional[AnyUrl] = Field(
None,
description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
Expand All @@ -154,8 +123,7 @@ class Config:
extra = Extra.forbid

__root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field(
...,
description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.",
..., description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade."
)


Expand Down Expand Up @@ -183,30 +151,18 @@ class Config:
iconUrl: Optional[str] = None
spec: Dict[str, Any]
tombstone: Optional[bool] = Field(
False,
description="if false, the configuration is active. if true, then this configuration is permanently off.",
)
public: Optional[bool] = Field(
False,
description="true if this connector definition is available to all workspaces",
)
custom: Optional[bool] = Field(
False, description="whether this is a custom connector definition"
False, description="if false, the configuration is active. if true, then this configuration is permanently off."
)
public: Optional[bool] = Field(False, description="true if this connector definition is available to all workspaces")
custom: Optional[bool] = Field(False, description="whether this is a custom connector definition")
releaseStage: Optional[ReleaseStage] = None
supportLevel: Optional[SupportLevel] = None
releaseDate: Optional[date] = Field(
None,
description="The date when this connector was first released, in yyyy-mm-dd format.",
)
releaseDate: Optional[date] = Field(None, description="The date when this connector was first released, in yyyy-mm-dd format.")
tags: Optional[List[str]] = Field(
None,
description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
None, description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc."
)
resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
protocolVersion: Optional[str] = Field(
None, description="the Airbyte Protocol version supported by the connector"
)
protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None
supportsDbt: Optional[bool] = Field(
None,
Expand Down
Loading

0 comments on commit bc7b4cb

Please sign in to comment.