From df34893b63ea944731429cff294c4788c87e62a4 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:48:17 +0200 Subject: [PATCH] feat(airbyte-cdk): replace pydantic `BaseModel` with `dataclasses` + `serpyco-rs` in protocol (#44444) Signed-off-by: Artem Inzhyyants --- .../python/airbyte_cdk/config_observation.py | 12 +- airbyte-cdk/python/airbyte_cdk/connector.py | 4 +- .../airbyte_cdk/connector_builder/main.py | 15 +- .../connector_builder/message_grouper.py | 34 +- .../airbyte_cdk/destinations/destination.py | 12 +- airbyte-cdk/python/airbyte_cdk/entrypoint.py | 22 +- airbyte-cdk/python/airbyte_cdk/logger.py | 17 +- .../python/airbyte_cdk/models/__init__.py | 10 + .../airbyte_cdk/models/airbyte_protocol.py | 79 +- .../models/airbyte_protocol_serializers.py | 38 + .../airbyte_cdk/models/well_known_types.py | 2 +- .../concurrent_read_processor.py | 5 +- .../sources/connector_state_manager.py | 22 +- .../incremental/per_partition_cursor.py | 2 +- .../wait_time_from_header_backoff_strategy.py | 2 +- .../sources/declarative/spec/spec.py | 6 +- .../sources/embedded/base_integration.py | 4 +- .../file_based/file_types/avro_parser.py | 4 +- .../file_based/file_types/excel_parser.py | 6 +- .../python/airbyte_cdk/sources/source.py | 13 +- ...substream_resumable_full_refresh_cursor.py | 2 +- .../sources/streams/http/http_client.py | 18 +- .../sources/utils/catalog_helpers.py | 22 - .../sources/utils/schema_models.py | 84 -- .../airbyte_cdk/test/catalog_builder.py | 4 +- .../airbyte_cdk/test/entrypoint_wrapper.py | 30 +- .../python/airbyte_cdk/test/state_builder.py | 10 +- .../python/airbyte_cdk/test/utils/reading.py | 2 +- .../python/airbyte_cdk/utils/message_utils.py | 8 +- .../airbyte_cdk/utils/traced_exception.py | 21 +- airbyte-cdk/python/cdk-migrations.md | 31 + airbyte-cdk/python/poetry.lock | 776 +++++++++++------- airbyte-cdk/python/pyproject.toml | 4 +- airbyte-cdk/python/unit_tests/conftest.py | 4 +- .../test_connector_builder_handler.py | 40 +- .../connector_builder/test_message_grouper.py | 114 ++- .../unit_tests/connector_builder/utils.py | 4 +- .../destinations/test_destination.py | 11 +- .../document_processor_test.py | 10 +- .../vector_db_based/embedder_test.py | 2 +- .../vector_db_based/writer_test.py | 7 +- .../test_concurrent_source_adapter.py | 22 +- .../sources/declarative/auth/test_jwt.py | 37 +- .../declarative/checks/test_check_stream.py | 4 +- .../datetime/test_datetime_parser.py | 7 +- .../datetime/test_min_max_datetime.py | 7 +- .../declarative/decoders/test_json_decoder.py | 14 +- .../extractors/test_dpath_extractor.py | 7 +- .../incremental/test_datetime_based_cursor.py | 7 +- .../incremental/test_per_partition_cursor.py | 2 +- .../test_per_partition_cursor_integration.py | 58 +- .../test_resumable_full_refresh_cursor.py | 2 +- .../test_legacy_to_per_partition_migration.py | 250 +++--- .../test_model_to_component_factory.py | 78 +- ...test_cartesian_product_partition_router.py | 56 +- .../test_list_partition_router.py | 28 +- .../test_parent_state_stream.py | 72 +- .../test_substream_partition_router.py | 104 +-- .../test_wait_time_from_header.py | 10 +- .../test_composite_error_handler.py | 6 +- .../test_default_error_handler.py | 242 +++--- .../test_default_http_response_filter.py | 20 +- .../test_http_response_filter.py | 32 +- .../test_cursor_pagination_strategy.py | 2 +- .../paginators/test_default_paginator.py | 2 +- .../paginators/test_page_increment.py | 8 +- .../requesters/test_http_requester.py | 5 +- .../retrievers/test_simple_retriever.py | 55 +- .../sources/declarative/spec/test_spec.py | 16 +- .../declarative/test_declarative_stream.py | 2 +- .../test_manifest_declarative_source.py | 39 +- .../sources/declarative/test_types.py | 51 +- .../embedded/test_embedded_integration.py | 6 +- .../file_based/config/test_csv_format.py | 2 +- .../file_based/file_types/test_avro_parser.py | 2 +- .../file_types/test_excel_parser.py | 14 +- .../file_types/test_parquet_parser.py | 3 +- .../file_based/in_memory_files_source.py | 8 +- .../concurrent_incremental_scenarios.py | 12 +- .../file_based/scenarios/csv_scenarios.py | 98 ++- .../file_based/scenarios/excel_scenarios.py | 26 +- .../scenarios/file_based_source_builder.py | 4 +- .../file_based/scenarios/scenario_builder.py | 21 +- .../scenarios/unstructured_scenarios.py | 14 - .../scenarios/user_input_schema_scenarios.py | 2 +- .../test_file_based_concurrent_cursor.py | 2 +- .../sources/file_based/test_scenarios.py | 18 +- .../sources/message/test_repository.py | 9 - .../mock_server_tests/mock_source_fixture.py | 148 ++-- .../airbyte_message_assertions.py | 15 +- .../test_mock_server_abstract_source.py | 195 +++-- .../test_resumable_full_refresh.py | 95 ++- .../checkpoint/test_checkpoint_reader.py | 7 +- ...substream_resumable_full_refresh_cursor.py | 83 +- .../scenarios/stream_facade_builder.py | 22 +- ..._based_concurrent_stream_source_builder.py | 22 +- .../test_concurrent_read_processor.py | 4 +- .../sources/streams/concurrent/test_cursor.py | 21 +- .../concurrent/test_partition_enqueuer.py | 5 +- .../test_default_backoff_strategy.py | 5 +- .../test_http_status_error_handler.py | 20 +- .../test_json_error_message_parser.py | 34 +- .../error_handlers/test_response_models.py | 8 +- .../http/test_availability_strategy.py | 8 +- .../sources/streams/http/test_http.py | 291 +++---- .../sources/streams/http/test_http_client.py | 144 ++-- .../sources/streams/test_stream_read.py | 12 +- .../sources/streams/test_streams_core.py | 90 +- .../streams/utils/test_stream_helper.py | 5 +- .../sources/test_abstract_source.py | 45 +- .../sources/test_connector_state_manager.py | 102 +-- .../python/unit_tests/sources/test_source.py | 54 +- .../unit_tests/sources/test_source_read.py | 30 +- .../sources/utils/test_catalog_helpers.py | 30 - .../sources/utils/test_schema_helpers.py | 6 +- .../sources/utils/test_schema_models.py | 65 -- .../test/mock_http/test_response_builder.py | 18 +- .../test/test_entrypoint_wrapper.py | 54 +- .../python/unit_tests/test_connector.py | 5 +- .../python/unit_tests/test_entrypoint.py | 202 +++-- .../unit_tests/test_exception_handler.py | 24 +- .../utils/test_datetime_format_inferrer.py | 2 +- .../unit_tests/utils/test_message_utils.py | 14 +- .../unit_tests/utils/test_schema_inferrer.py | 98 +-- .../unit_tests/utils/test_traced_exception.py | 18 +- 125 files changed, 2730 insertions(+), 2270 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py delete mode 100644 airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py delete mode 100644 airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py diff --git a/airbyte-cdk/python/airbyte_cdk/config_observation.py b/airbyte-cdk/python/airbyte_cdk/config_observation.py index 55e03f335c86..94a3d64a511b 100644 --- a/airbyte-cdk/python/airbyte_cdk/config_observation.py +++ b/airbyte-cdk/python/airbyte_cdk/config_observation.py @@ -10,7 +10,15 @@ from copy import copy from typing import Any, List, MutableMapping -from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, OrchestratorType, Type +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteMessage, + AirbyteMessageSerializer, + OrchestratorType, + Type, +) +from orjson import orjson class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any] @@ -76,7 +84,7 @@ def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, An See the airbyte_cdk.sources.message package """ airbyte_message = create_connector_config_control_message(config) - print(airbyte_message.model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode()) def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage: diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index e40ace288e2b..658a0b167077 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -11,7 +11,7 @@ from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar import yaml -from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification +from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification, ConnectorSpecificationSerializer def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: @@ -84,7 +84,7 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: else: raise FileNotFoundError("Unable to find spec.yaml or spec.json in the package.") - return ConnectorSpecification.parse_obj(spec_obj) + return ConnectorSpecificationSerializer.load(spec_obj) @abstractmethod def check(self, logger: logging.Logger, config: TConfig) -> AirbyteConnectionStatus: diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py index 9f12b8aaeb11..1691b41b090d 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py @@ -9,10 +9,17 @@ from airbyte_cdk.connector import BaseConnector from airbyte_cdk.connector_builder.connector_builder_handler import TestReadLimits, create_source, get_limits, read_stream, resolve_manifest from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, +) from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.source import Source from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]: @@ -32,7 +39,7 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, command = config["__command"] if command == "test_read": - catalog = ConfiguredAirbyteCatalog.parse_obj(BaseConnector.read_config(catalog_path)) + catalog = ConfiguredAirbyteCatalogSerializer.load(BaseConnector.read_config(catalog_path)) state = Source.read_state(state_path) else: catalog = None @@ -67,7 +74,7 @@ def handle_request(args: List[str]) -> AirbyteMessage: command, config, catalog, state = get_config_and_catalog_from_args(args) limits = get_limits(config) source = create_source(config, limits) - return handle_connector_builder_request(source, command, config, catalog, state, limits).json(exclude_unset=True) + return AirbyteMessageSerializer.dump(handle_connector_builder_request(source, command, config, catalog, state, limits)) # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage if __name__ == "__main__": @@ -76,4 +83,4 @@ def handle_request(args: List[str]) -> AirbyteMessage: except Exception as exc: error = AirbyteTracedException.from_exception(exc, message=f"Error handling request: {str(exc)}") m = error.as_airbyte_message() - print(error.as_airbyte_message().model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(m)).decode()) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index 80cb8c36178e..4b00fc874cf5 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -18,13 +18,7 @@ StreamReadSlices, ) from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource -from airbyte_cdk.sources.utils.slice_logger import SliceLogger -from airbyte_cdk.sources.utils.types import JsonType -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer -from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException -from airbyte_protocol.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteControlMessage, AirbyteLogMessage, AirbyteMessage, @@ -34,7 +28,13 @@ OrchestratorType, TraceType, ) -from airbyte_protocol.models.airbyte_protocol import Type as MessageType +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_cdk.sources.utils.types import JsonType +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer +from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException class MessageGrouper: @@ -182,19 +182,19 @@ def _get_message_groups( if ( at_least_one_page_in_group and message.type == MessageType.LOG - and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) + and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message ): yield StreamReadSlices( pages=current_slice_pages, slice_descriptor=current_slice_descriptor, state=[latest_state_message] if latest_state_message else [], ) - current_slice_descriptor = self._parse_slice_description(message.log.message) + current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message current_slice_pages = [] at_least_one_page_in_group = False - elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX): + elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX): # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message # parsing the first slice - current_slice_descriptor = self._parse_slice_description(message.log.message) + current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message elif message.type == MessageType.LOG: if json_message is not None and self._is_http_log(json_message): if self._is_auxiliary_http_request(json_message): @@ -221,17 +221,17 @@ def _get_message_groups( else: yield message.log elif message.type == MessageType.TRACE: - if message.trace.type == TraceType.ERROR: + if message.trace.type == TraceType.ERROR: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has trace.type yield message.trace elif message.type == MessageType.RECORD: - current_page_records.append(message.record.data) + current_page_records.append(message.record.data) # type: ignore[union-attr] # AirbyteMessage with MessageType.RECORD has record.data records_count += 1 schema_inferrer.accumulate(message.record) datetime_format_inferrer.accumulate(message.record) - elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: + elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: # type: ignore[union-attr] # AirbyteMessage with MessageType.CONTROL has control.type yield message.control elif message.type == MessageType.STATE: - latest_state_message = message.state + latest_state_message = message.state # type: ignore[assignment] else: if current_page_request or current_page_response or current_page_records: self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) @@ -246,7 +246,7 @@ def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessag return ( at_least_one_page_in_group and message.type == MessageType.LOG - and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:")) + and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:")) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message ) @staticmethod diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/destination.py b/airbyte-cdk/python/airbyte_cdk/destinations/destination.py index f95e185aabfe..336a54a94e8f 100644 --- a/airbyte-cdk/python/airbyte_cdk/destinations/destination.py +++ b/airbyte-cdk/python/airbyte_cdk/destinations/destination.py @@ -11,10 +11,10 @@ from airbyte_cdk.connector import Connector from airbyte_cdk.exception_handler import init_uncaught_exception_handler -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type +from airbyte_cdk.models import AirbyteMessage, AirbyteMessageSerializer, ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer, Type from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from pydantic import ValidationError as V2ValidationError +from orjson import orjson logger = logging.getLogger("airbyte") @@ -36,14 +36,14 @@ def _parse_input_stream(self, input_stream: io.TextIOWrapper) -> Iterable[Airbyt """Reads from stdin, converting to Airbyte messages""" for line in input_stream: try: - yield AirbyteMessage.parse_raw(line) - except V2ValidationError: + yield AirbyteMessageSerializer.load(orjson.loads(line)) + except orjson.JSONDecodeError: logger.info(f"ignoring input which can't be deserialized as Airbyte Message: {line}") def _run_write( self, config: Mapping[str, Any], configured_catalog_path: str, input_stream: io.TextIOWrapper ) -> Iterable[AirbyteMessage]: - catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path) + catalog = ConfiguredAirbyteCatalogSerializer.load(orjson.loads(open(configured_catalog_path).read())) input_messages = self._parse_input_stream(input_stream) logger.info("Begin writing to the destination...") yield from self.write(config=config, configured_catalog=catalog, input_messages=input_messages) @@ -117,4 +117,4 @@ def run(self, args: List[str]) -> None: parsed_args = self.parse_args(args) output_messages = self.run_cmd(parsed_args) for message in output_messages: - print(message.model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode()) diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index cc9c3662ff67..57b604691bc1 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -19,8 +19,15 @@ from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.logger import init_logger -from airbyte_cdk.models import AirbyteMessage, FailureType, Status, Type -from airbyte_cdk.models.airbyte_protocol import AirbyteStateStats, ConnectorSpecification # type: ignore [attr-defined] +from airbyte_cdk.models import ( # type: ignore [attr-defined] + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteStateStats, + ConnectorSpecification, + FailureType, + Status, + Type, +) from airbyte_cdk.sources import Source from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config @@ -28,6 +35,7 @@ from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson from requests import PreparedRequest, Response, Session logger = init_logger("airbyte") @@ -170,13 +178,13 @@ def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: An def handle_record_counts(message: AirbyteMessage, stream_message_count: DefaultDict[HashableStreamDescriptor, float]) -> AirbyteMessage: match message.type: case Type.RECORD: - stream_message_count[HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace)] += 1.0 + stream_message_count[HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace)] += 1.0 # type: ignore[union-attr] # record has `stream` and `namespace` case Type.STATE: stream_descriptor = message_utils.get_stream_descriptor(message) # Set record count from the counter onto the state message - message.state.sourceStats = message.state.sourceStats or AirbyteStateStats() - message.state.sourceStats.recordCount = stream_message_count.get(stream_descriptor, 0.0) + message.state.sourceStats = message.state.sourceStats or AirbyteStateStats() # type: ignore[union-attr] # state has `sourceStats` + message.state.sourceStats.recordCount = stream_message_count.get(stream_descriptor, 0.0) # type: ignore[union-attr] # state has `sourceStats` # Reset the counter stream_message_count[stream_descriptor] = 0.0 @@ -197,8 +205,8 @@ def set_up_secret_filter(config: TConfig, connection_specification: Mapping[str, update_secrets(config_secrets) @staticmethod - def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> Any: - return airbyte_message.model_dump_json(exclude_unset=True) + def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str: + return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode() # type: ignore[no-any-return] # orjson.dumps(message).decode() always returns string @classmethod def extract_state(cls, args: List[str]) -> Optional[Any]: diff --git a/airbyte-cdk/python/airbyte_cdk/logger.py b/airbyte-cdk/python/airbyte_cdk/logger.py index 6f40e581df94..72673dcfa8f7 100644 --- a/airbyte-cdk/python/airbyte_cdk/logger.py +++ b/airbyte-cdk/python/airbyte_cdk/logger.py @@ -7,8 +7,9 @@ import logging.config from typing import Any, Mapping, Optional, Tuple -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteMessageSerializer, Level, Type from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets +from orjson import orjson LOGGING_CONFIG = { "version": 1, @@ -42,11 +43,11 @@ class AirbyteLogFormatter(logging.Formatter): # Transforming Python log levels to Airbyte protocol log levels level_mapping = { - logging.FATAL: "FATAL", - logging.ERROR: "ERROR", - logging.WARNING: "WARN", - logging.INFO: "INFO", - logging.DEBUG: "DEBUG", + logging.FATAL: Level.FATAL, + logging.ERROR: Level.ERROR, + logging.WARNING: Level.WARN, + logging.INFO: Level.INFO, + logging.DEBUG: Level.DEBUG, } def format(self, record: logging.LogRecord) -> str: @@ -59,8 +60,8 @@ def format(self, record: logging.LogRecord) -> str: else: message = super().format(record) message = filter_secrets(message) - log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=airbyte_level, message=message)) - return log_message.model_dump_json(exclude_unset=True) # type: ignore + log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)) + return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() # type: ignore[no-any-return] # orjson.dumps(message).decode() always returns string @staticmethod def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]: diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index b062a4468c22..c56df9adc43a 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -7,6 +7,7 @@ # of airbyte-cdk rather than a standalone package. from .airbyte_protocol import ( AdvancedAuth, + AirbyteStateStats, AirbyteAnalyticsTraceMessage, AirbyteCatalog, AirbyteConnectionStatus, @@ -58,3 +59,12 @@ TimeWithoutTimezone, TimeWithTimezone, ) + +from .airbyte_protocol_serializers import ( +AirbyteStreamStateSerializer, +AirbyteStateMessageSerializer, +AirbyteMessageSerializer, +ConfiguredAirbyteCatalogSerializer, +ConfiguredAirbyteStreamSerializer, +ConnectorSpecificationSerializer, +) \ No newline at end of file diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 74639c8bf3c1..477cfb8a66fd 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -2,4 +2,81 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_protocol.models import * +from dataclasses import InitVar, dataclass +from typing import Annotated, Any, Dict, List, Mapping, Optional + +from airbyte_protocol_dataclasses.models import * +from serpyco_rs.metadata import Alias + + +@dataclass +class AirbyteStateBlob: + """ + A dataclass that dynamically sets attributes based on provided keyword arguments and positional arguments. + Used to "mimic" pydantic Basemodel with ConfigDict(extra='allow') option. + + The `AirbyteStateBlob` class allows for flexible instantiation by accepting any number of keyword arguments + and positional arguments. These are used to dynamically update the instance's attributes. This class is useful + in scenarios where the attributes of an object are not known until runtime and need to be set dynamically. + + Attributes: + kwargs (InitVar[Mapping[str, Any]]): A dictionary of keyword arguments used to set attributes dynamically. + + Methods: + __init__(*args: Any, **kwargs: Any) -> None: + Initializes the `AirbyteStateBlob` by setting attributes from the provided arguments. + + __eq__(other: object) -> bool: + Checks equality between two `AirbyteStateBlob` instances based on their internal dictionaries. + Returns `False` if the other object is not an instance of `AirbyteStateBlob`. + """ + + kwargs: InitVar[Mapping[str, Any]] + + def __init__(self, *args: Any, **kwargs: Any) -> None: + # Set any attribute passed in through kwargs + for arg in args: + self.__dict__.update(arg) + for key, value in kwargs.items(): + setattr(self, key, value) + + def __eq__(self, other: object) -> bool: + return False if not isinstance(other, AirbyteStateBlob) else bool(self.__dict__ == other.__dict__) + + +# The following dataclasses have been redeclared to include the new version of AirbyteStateBlob +@dataclass +class AirbyteStreamState: + stream_descriptor: StreamDescriptor # type: ignore [name-defined] + stream_state: Optional[AirbyteStateBlob] = None + + +@dataclass +class AirbyteGlobalState: + stream_states: List[AirbyteStreamState] + shared_state: Optional[AirbyteStateBlob] = None + + +@dataclass +class AirbyteStateMessage: + type: Optional[AirbyteStateType] = None # type: ignore [name-defined] + stream: Optional[AirbyteStreamState] = None + global_: Annotated[ + AirbyteGlobalState | None, Alias("global") + ] = None # "global" is a reserved keyword in python ⇒ Alias is used for (de-)serialization + data: Optional[Dict[str, Any]] = None + sourceStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] + destinationStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] + + +@dataclass +class AirbyteMessage: + type: Type # type: ignore [name-defined] + log: Optional[AirbyteLogMessage] = None # type: ignore [name-defined] + spec: Optional[ConnectorSpecification] = None # type: ignore [name-defined] + connectionStatus: Optional[AirbyteConnectionStatus] = None # type: ignore [name-defined] + catalog: Optional[AirbyteCatalog] = None # type: ignore [name-defined] + record: Optional[AirbyteRecordMessage] = None # type: ignore [name-defined] + state: Optional[AirbyteStateMessage] = None + trace: Optional[AirbyteTraceMessage] = None # type: ignore [name-defined] + control: Optional[AirbyteControlMessage] = None # type: ignore [name-defined] diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py new file mode 100644 index 000000000000..aeac43f794ce --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from typing import Any, Dict + +from serpyco_rs import CustomType, Serializer + +from .airbyte_protocol import ( # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via * + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStreamState, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, +) + + +class AirbyteStateBlobType(CustomType[AirbyteStateBlob, Dict[str, Any]]): + def serialize(self, value: AirbyteStateBlob) -> Dict[str, Any]: + # cant use orjson.dumps() directly because private attributes are excluded, e.g. "__ab_full_refresh_sync_complete" + return {k: v for k, v in value.__dict__.items()} + + def deserialize(self, value: Dict[str, Any]) -> AirbyteStateBlob: + return AirbyteStateBlob(value) + + def get_json_schema(self) -> Dict[str, Any]: + return {"type": "object"} + + +def custom_type_resolver(t: type) -> CustomType[AirbyteStateBlob, Dict[str, Any]] | None: + return AirbyteStateBlobType() if t is AirbyteStateBlob else None + + +AirbyteStreamStateSerializer = Serializer(AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver) +AirbyteStateMessageSerializer = Serializer(AirbyteStateMessage, omit_none=True, custom_type_resolver=custom_type_resolver) +AirbyteMessageSerializer = Serializer(AirbyteMessage, omit_none=True, custom_type_resolver=custom_type_resolver) +ConfiguredAirbyteCatalogSerializer = Serializer(ConfiguredAirbyteCatalog, omit_none=True) +ConfiguredAirbyteStreamSerializer = Serializer(ConfiguredAirbyteStream, omit_none=True) +ConnectorSpecificationSerializer = Serializer(ConnectorSpecification, omit_none=True) diff --git a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py index 0cc409c7e070..a063ad7db03a 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py +++ b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py @@ -2,4 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_protocol.models.well_known_types import * +from airbyte_protocol_dataclasses.models.well_known_types import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py index f345c6b4bd75..40cc771ab780 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py @@ -5,7 +5,7 @@ from typing import Dict, Iterable, List, Optional, Set from airbyte_cdk.exception_handler import generate_failed_streams_error_message -from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus +from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, FailureType, StreamDescriptor from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel from airbyte_cdk.sources.concurrent_source.stream_thread_exception import StreamThreadException @@ -21,7 +21,6 @@ from airbyte_cdk.sources.utils.slice_logger import SliceLogger from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message -from airbyte_protocol.models import FailureType, StreamDescriptor class ConcurrentReadProcessor: @@ -76,7 +75,7 @@ def on_partition_generation_completed(self, sentinel: PartitionGenerationComplet if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0: yield from self._on_stream_is_done(stream_name) if self._stream_instances_to_start_partition_generation: - yield self.start_next_partition_generator() + yield self.start_next_partition_generator() # type:ignore # None may be yielded def on_partition(self, partition: Partition) -> None: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py index b550d81b7a27..547f4bb23dca 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py @@ -3,20 +3,22 @@ # import copy +from dataclasses import dataclass from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor from airbyte_cdk.models import Type as MessageType -from pydantic import ConfigDict as V2ConfigDict -class HashableStreamDescriptor(StreamDescriptor): +@dataclass(frozen=True) +class HashableStreamDescriptor: """ Helper class that overrides the existing StreamDescriptor class that is auto generated from the Airbyte Protocol and freezes its fields so that it be used as a hash key. This is only marked public because we use it outside for unit tests. """ - model_config = V2ConfigDict(extra="allow", frozen=True) + name: str + namespace: Optional[str] = None class ConnectorStateManager: @@ -47,9 +49,9 @@ def get_stream_state(self, stream_name: str, namespace: Optional[str]) -> Mutabl :param namespace: Namespace of the stream being fetched :return: The per-stream state for a stream """ - stream_state = self.per_stream_states.get(HashableStreamDescriptor(name=stream_name, namespace=namespace)) + stream_state: AirbyteStateBlob | None = self.per_stream_states.get(HashableStreamDescriptor(name=stream_name, namespace=namespace)) if stream_state: - return stream_state.dict() # type: ignore # mypy thinks dict() returns any, but it returns a dict + return copy.deepcopy({k: v for k, v in stream_state.__dict__.items()}) return {} def update_state_for_stream(self, stream_name: str, namespace: Optional[str], value: Mapping[str, Any]) -> None: @@ -60,7 +62,7 @@ def update_state_for_stream(self, stream_name: str, namespace: Optional[str], va :param value: A stream state mapping that is being updated for a stream """ stream_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace) - self.per_stream_states[stream_descriptor] = AirbyteStateBlob.parse_obj(value) + self.per_stream_states[stream_descriptor] = AirbyteStateBlob(value) def create_state_message(self, stream_name: str, namespace: Optional[str]) -> AirbyteMessage: """ @@ -100,19 +102,19 @@ def _extract_from_state_message( if is_global: global_state = state[0].global_ # type: ignore # We verified state is a list in _is_global_state - shared_state = copy.deepcopy(global_state.shared_state, {}) + shared_state = copy.deepcopy(global_state.shared_state, {}) # type: ignore[union-attr] # global_state has shared_state streams = { HashableStreamDescriptor( name=per_stream_state.stream_descriptor.name, namespace=per_stream_state.stream_descriptor.namespace ): per_stream_state.stream_state - for per_stream_state in global_state.stream_states + for per_stream_state in global_state.stream_states # type: ignore[union-attr] # global_state has shared_state } return shared_state, streams else: streams = { HashableStreamDescriptor( - name=per_stream_state.stream.stream_descriptor.name, namespace=per_stream_state.stream.stream_descriptor.namespace - ): per_stream_state.stream.stream_state + name=per_stream_state.stream.stream_descriptor.name, namespace=per_stream_state.stream.stream_descriptor.namespace # type: ignore[union-attr] # stream has stream_descriptor + ): per_stream_state.stream.stream_state # type: ignore[union-attr] # stream has stream_state for per_stream_state in state if per_stream_state.type == AirbyteStateType.STREAM and hasattr(per_stream_state, "stream") # type: ignore # state is always a list of AirbyteStateMessage if is_per_stream is True } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py index f0869b72fa29..28c2f0eb6b8b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py @@ -4,12 +4,12 @@ from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Union +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import PerPartitionKeySerializer from airbyte_cdk.sources.types import Record, StreamSlice, StreamState from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType class CursorFactory: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index d437a5c12ae5..79eb8a7fe23d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -7,12 +7,12 @@ from typing import Any, Mapping, Optional, Union import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy from airbyte_cdk.sources.types import Config from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType @dataclass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py index a0d499f5d13d..87c8911d6aa6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py @@ -5,7 +5,7 @@ from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional -from airbyte_cdk.models.airbyte_protocol import AdvancedAuth, ConnectorSpecification # type: ignore [attr-defined] +from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, ConnectorSpecificationSerializer # type: ignore [attr-defined] from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow @@ -36,7 +36,7 @@ def generate_spec(self) -> ConnectorSpecification: if self.advanced_auth: self.advanced_auth.auth_flow_type = self.advanced_auth.auth_flow_type.value # type: ignore # We know this is always assigned to an AuthFlow which has the auth_flow_type field # Map CDK AuthFlow model to protocol AdvancedAuth model - obj["advanced_auth"] = AdvancedAuth.parse_obj(self.advanced_auth.dict()) + obj["advanced_auth"] = self.advanced_auth.dict() # We remap these keys to camel case because that's the existing format expected by the rest of the platform - return ConnectorSpecification.parse_obj(obj) + return ConnectorSpecificationSerializer.load(obj) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py index 158dea4d135a..79c9bd850a3a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py @@ -6,11 +6,11 @@ from typing import Generic, Iterable, Optional, TypeVar from airbyte_cdk.connector import TConfig +from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type from airbyte_cdk.sources.embedded.catalog import create_configured_catalog, get_stream, get_stream_names from airbyte_cdk.sources.embedded.runner import SourceRunner from airbyte_cdk.sources.embedded.tools import get_defined_id from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit -from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type TOutput = TypeVar("TOutput") @@ -43,7 +43,7 @@ def _load_data(self, stream_name: str, state: Optional[AirbyteStateMessage] = No for message in self.source.read(self.config, configured_catalog, state): if message.type == Type.RECORD: - output = self._handle_record(message.record, get_defined_id(stream, message.record.data)) + output = self._handle_record(message.record, get_defined_id(stream, message.record.data)) # type: ignore[union-attr] # record has `data` if output: yield output elif message.type is Type.STATE and message.state: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py index bbae84b287bf..b033afa57fb3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py @@ -166,7 +166,9 @@ def file_read_mode(self) -> FileReadMode: @staticmethod def _to_output_value(avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any) -> Any: - if not isinstance(record_type, Mapping): + if isinstance(record_value, bytes): + return record_value.decode() + elif not isinstance(record_type, Mapping): if record_type == "double" and avro_format.double_as_string: return str(record_value) return record_value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py index 579a85390c66..93add4108dea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py @@ -17,6 +17,7 @@ from numpy import datetime64 from numpy import dtype as dtype_ from numpy import issubdtype +from orjson import orjson from pydantic.v1 import BaseModel @@ -97,7 +98,10 @@ def parse_records( with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: df = self.open_and_parse_file(fp) # Yield records as dictionaries - yield from df.to_dict(orient="records") + # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson + # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior + # see PR description: https://github.com/airbytehq/airbyte/pull/44444/ + yield from orjson.loads(df.to_json(orient="records", date_format="iso", date_unit="us")) except Exception as exc: # Raise a RecordParseError if any exception occurs during parsing diff --git a/airbyte-cdk/python/airbyte_cdk/sources/source.py b/airbyte-cdk/python/airbyte_cdk/sources/source.py index 77de81fbe7f1..975770c88949 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/source.py @@ -8,7 +8,14 @@ from typing import Any, Generic, Iterable, List, Mapping, Optional, TypeVar from airbyte_cdk.connector import BaseConnector, DefaultConnectorMixin, TConfig -from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteMessage, + AirbyteStateMessage, + AirbyteStateMessageSerializer, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, +) TState = TypeVar("TState") TCatalog = TypeVar("TCatalog") @@ -61,7 +68,7 @@ def read_state(cls, state_path: str) -> List[AirbyteStateMessage]: state_obj = BaseConnector._read_json_file(state_path) if state_obj: for state in state_obj: # type: ignore # `isinstance(state_obj, List)` ensures that this is a list - parsed_message = AirbyteStateMessage.parse_obj(state) + parsed_message = AirbyteStateMessageSerializer.load(state) if not parsed_message.stream and not parsed_message.data and not parsed_message.global_: raise ValueError("AirbyteStateMessage should contain either a stream, global, or state field") parsed_state_messages.append(parsed_message) @@ -70,7 +77,7 @@ def read_state(cls, state_path: str) -> List[AirbyteStateMessage]: # can be overridden to change an input catalog @classmethod def read_catalog(cls, catalog_path: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog.parse_obj(cls._read_json_file(catalog_path)) + return ConfiguredAirbyteCatalogSerializer.load(cls._read_json_file(catalog_path)) @property def name(self) -> str: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py index 761a37e1f180..0afc2974fa9a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from typing import Any, Mapping, MutableMapping, Optional +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.checkpoint import Cursor from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import PerPartitionKeySerializer from airbyte_cdk.sources.types import Record, StreamSlice, StreamState from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType FULL_REFRESH_COMPLETE_STATE: Mapping[str, Any] = {"__ab_full_refresh_sync_complete": True} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py index d52b92627577..b1f23aeb4e25 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py @@ -10,7 +10,14 @@ import requests import requests_cache -from airbyte_cdk.models import AirbyteStreamStatus, AirbyteStreamStatusReason, AirbyteStreamStatusReasonType, Level, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessageSerializer, + AirbyteStreamStatus, + AirbyteStreamStatusReason, + AirbyteStreamStatusReasonType, + Level, + StreamDescriptor, +) from airbyte_cdk.sources.http_config import MAX_CONNECTION_POOL_SIZE from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.call_rate import APIBudget, CachedLimiterSession, LimiterSession @@ -38,6 +45,7 @@ from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson from requests.auth import AuthBase BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH") @@ -281,9 +289,11 @@ def _send( if error_resolution.response_action == ResponseAction.RATE_LIMITED: # TODO: Update to handle with message repository when concurrent message repository is ready reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)] - message = stream_status_as_airbyte_message( - StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons - ).model_dump_json(exclude_unset=True) + message = orjson.dumps( + AirbyteMessageSerializer.dump( + stream_status_as_airbyte_message(StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons) + ) + ).decode() # Simply printing the stream status is a temporary solution and can cause future issues. Currently, the _send method is # wrapped with backoff decorators, and we can only emit messages by iterating record_iterator in the abstract source at the diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py deleted file mode 100644 index 415374a44bc1..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import AirbyteCatalog, SyncMode - - -class CatalogHelper: - @staticmethod - def coerce_catalog_as_full_refresh(catalog: AirbyteCatalog) -> AirbyteCatalog: - """ - Updates the sync mode on all streams in this catalog to be full refresh - """ - coerced_catalog = catalog.copy() - for stream in catalog.streams: - stream.source_defined_cursor = False - stream.supported_sync_modes = [SyncMode.full_refresh] - stream.default_cursor_field = None - - # remove nulls - return AirbyteCatalog.parse_raw(coerced_catalog.model_dump_json(exclude_unset=True, exclude_none=True)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py deleted file mode 100644 index de011bfb896b..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Dict, Optional, Type - -from airbyte_cdk.sources.utils.schema_helpers import expand_refs -from pydantic.v1 import BaseModel, Extra -from pydantic.v1.main import ModelMetaclass -from pydantic.v1.typing import resolve_annotations - - -class AllOptional(ModelMetaclass): - """ - Metaclass for marking all Pydantic model fields as Optional - Here is example of declaring model using this metaclass like: - ''' - class MyModel(BaseModel, metaclass=AllOptional): - a: str - b: str - ''' - it is an equivalent of: - ''' - class MyModel(BaseModel): - a: Optional[str] - b: Optional[str] - ''' - It would make code more clear and eliminate a lot of manual work. - """ - - def __new__(mcs, name, bases, namespaces, **kwargs): # type: ignore[no-untyped-def] # super().__new__ is also untyped - """ - Iterate through fields and wrap then with typing.Optional type. - """ - annotations = resolve_annotations(namespaces.get("__annotations__", {}), namespaces.get("__module__", None)) - for base in bases: - annotations = {**annotations, **getattr(base, "__annotations__", {})} - for field in annotations: - if not field.startswith("__"): - annotations[field] = Optional[annotations[field]] # type: ignore[assignment] - namespaces["__annotations__"] = annotations - return super().__new__(mcs, name, bases, namespaces, **kwargs) - - -class BaseSchemaModel(BaseModel): - """ - Base class for all schema models. It has some extra schema postprocessing. - Can be used in combination with AllOptional metaclass - """ - - class Config: - extra = Extra.allow - - @classmethod - def schema_extra(cls, schema: Dict[str, Any], model: Type[BaseModel]) -> None: - """Modify generated jsonschema, remove "title", "description" and "required" fields. - - Pydantic doesn't treat Union[None, Any] type correctly when generate jsonschema, - so we can't set field as nullable (i.e. field that can have either null and non-null values), - We generate this jsonschema value manually. - - :param schema: generated jsonschema - :param model: - """ - schema.pop("title", None) - schema.pop("description", None) - schema.pop("required", None) - for name, prop in schema.get("properties", {}).items(): - prop.pop("title", None) - prop.pop("description", None) - allow_none = model.__fields__[name].allow_none - if allow_none: - if "type" in prop: - prop["type"] = ["null", prop["type"]] - elif "$ref" in prop: - ref = prop.pop("$ref") - prop["oneOf"] = [{"type": "null"}, {"$ref": ref}] - - @classmethod - def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: - """We're overriding the schema classmethod to enable some post-processing""" - schema = super().schema(*args, **kwargs) - expand_refs(schema) - return schema # type: ignore[no-any-return] diff --git a/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py index c3e3578f3494..235be7c579b6 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Union, overload -from airbyte_protocol.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, ConfiguredAirbyteStreamSerializer, SyncMode class ConfiguredAirbyteStreamBuilder: @@ -37,7 +37,7 @@ def with_json_schema(self, json_schema: Dict[str, Any]) -> "ConfiguredAirbyteStr return self def build(self) -> ConfiguredAirbyteStream: - return ConfiguredAirbyteStream.parse_obj(self._stream) + return ConfiguredAirbyteStreamSerializer.load(self._stream) class CatalogBuilder: diff --git a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py index ef300bd864f0..9cc74ec2669b 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py +++ b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py @@ -26,18 +26,23 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.exception_handler import assemble_uncaught_exception from airbyte_cdk.logger import AirbyteLogFormatter -from airbyte_cdk.sources import Source -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteStateMessage, + AirbyteStateMessageSerializer, AirbyteStreamStatus, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, TraceType, Type, ) +from airbyte_cdk.sources import Source +from orjson import orjson from pydantic import ValidationError as V2ValidationError +from serpyco_rs import SchemaValidationError class EntrypointOutput: @@ -53,8 +58,8 @@ def __init__(self, messages: List[str], uncaught_exception: Optional[BaseExcepti @staticmethod def _parse_message(message: str) -> AirbyteMessage: try: - return AirbyteMessage.parse_obj(json.loads(message)) - except (json.JSONDecodeError, V2ValidationError): + return AirbyteMessageSerializer.load(orjson.loads(message)) # type: ignore[no-any-return] # Serializer.load() always returns AirbyteMessage + except (orjson.JSONDecodeError, SchemaValidationError): # The platform assumes that logs that are not of AirbyteMessage format are log messages return AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message=message)) @@ -75,7 +80,7 @@ def most_recent_state(self) -> Any: state_messages = self._get_message_by_types([Type.STATE]) if not state_messages: raise ValueError("Can't provide most recent state as there are no state messages") - return state_messages[-1].state.stream + return state_messages[-1].state.stream # type: ignore[union-attr] # state has `stream` @property def logs(self) -> List[AirbyteMessage]: @@ -102,9 +107,9 @@ def catalog(self) -> AirbyteMessage: def get_stream_statuses(self, stream_name: str) -> List[AirbyteStreamStatus]: status_messages = map( - lambda message: message.trace.stream_status.status, + lambda message: message.trace.stream_status.status, # type: ignore filter( - lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, + lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, # type: ignore # callable; trace has `stream_status` self._get_trace_message_by_trace_type(TraceType.STREAM_STATUS), ), ) @@ -114,11 +119,11 @@ def _get_message_by_types(self, message_types: List[Type]) -> List[AirbyteMessag return [message for message in self._messages if message.type in message_types] def _get_trace_message_by_trace_type(self, trace_type: TraceType) -> List[AirbyteMessage]: - return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == trace_type] + return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == trace_type] # type: ignore[union-attr] # trace has `type` def is_in_logs(self, pattern: str) -> bool: """Check if any log message case-insensitive matches the pattern.""" - return any(re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs) + return any(re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs) # type: ignore[union-attr] # log has `message` def is_not_in_logs(self, pattern: str) -> bool: """Check if no log message matches the case-insensitive pattern.""" @@ -188,7 +193,9 @@ def read( with tempfile.TemporaryDirectory() as tmp_directory: tmp_directory_path = Path(tmp_directory) config_file = make_file(tmp_directory_path / "config.json", config) - catalog_file = make_file(tmp_directory_path / "catalog.json", catalog.model_dump_json()) + catalog_file = make_file( + tmp_directory_path / "catalog.json", orjson.dumps(ConfiguredAirbyteCatalogSerializer.dump(catalog)).decode() + ) args = [ "read", "--config", @@ -201,7 +208,8 @@ def read( [ "--state", make_file( - tmp_directory_path / "state.json", f"[{','.join([stream_state.model_dump_json() for stream_state in state])}]" + tmp_directory_path / "state.json", + f"[{','.join([orjson.dumps(AirbyteStateMessageSerializer.dump(stream_state)).decode() for stream_state in state])}]", ), ] ) diff --git a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py index 0c43d4320428..50b5dbe5f793 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py @@ -2,7 +2,7 @@ from typing import Any, List -from airbyte_protocol.models import AirbyteStateMessage +from airbyte_cdk.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor class StateBuilder: @@ -11,7 +11,13 @@ def __init__(self) -> None: def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": self._state.append( - AirbyteStateMessage.parse_obj({"type": "STREAM", "stream": {"stream_state": state, "stream_descriptor": {"name": stream_name}}}) + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_state=state if isinstance(state, AirbyteStateBlob) else AirbyteStateBlob(state), + stream_descriptor=StreamDescriptor(**{"name": stream_name}), + ), + ) ) return self diff --git a/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py b/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py index f8100187d4fb..2d89cb870984 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py +++ b/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py @@ -3,9 +3,9 @@ from typing import Any, List, Mapping, Optional from airbyte_cdk import AbstractSource +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: diff --git a/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py index 37d9d1351afc..a862d4696495 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py @@ -1,18 +1,18 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from airbyte_cdk.models import AirbyteMessage, Type from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor -from airbyte_protocol.models import AirbyteMessage, Type def get_stream_descriptor(message: AirbyteMessage) -> HashableStreamDescriptor: match message.type: case Type.RECORD: - return HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace) + return HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace) # type: ignore[union-attr] # record has `stream` and `namespace` case Type.STATE: - if not message.state.stream or not message.state.stream.stream_descriptor: + if not message.state.stream or not message.state.stream.stream_descriptor: # type: ignore[union-attr] # state has `stream` raise ValueError("State message was not in per-stream state format, which is required for record counts.") return HashableStreamDescriptor( - name=message.state.stream.stream_descriptor.name, namespace=message.state.stream.stream_descriptor.namespace + name=message.state.stream.stream_descriptor.name, namespace=message.state.stream.stream_descriptor.namespace # type: ignore[union-attr] # state has `stream` ) case _: raise NotImplementedError(f"get_stream_descriptor is not implemented for message type '{message.type}'.") diff --git a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py index 9bec5ac095c0..bd96ea398146 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py @@ -1,15 +1,15 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import time import traceback -from datetime import datetime from typing import Optional from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteErrorTraceMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteTraceMessage, FailureType, Status, @@ -18,6 +18,7 @@ ) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets +from orjson import orjson class AirbyteTracedException(Exception): @@ -54,7 +55,7 @@ def as_airbyte_message(self, stream_descriptor: Optional[StreamDescriptor] = Non :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many stream_descriptors are defined, the one from `as_airbyte_message` will be discarded. """ - now_millis = datetime.now().timestamp() * 1000.0 + now_millis = time.time_ns() // 1_000_000 trace_exc = self._exception or self stack_trace_str = "".join(traceback.TracebackException.from_exception(trace_exc).format()) @@ -85,7 +86,7 @@ def emit_message(self) -> None: Prints the exception as an AirbyteTraceMessage. Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint. """ - message = self.as_airbyte_message().model_dump_json(exclude_unset=True) + message = orjson.dumps(AirbyteMessageSerializer.dump(self.as_airbyte_message())).decode() filtered_message = filter_secrets(message) print(filtered_message) @@ -106,10 +107,10 @@ def as_sanitized_airbyte_message(self, stream_descriptor: Optional[StreamDescrip stream_descriptors are defined, the one from `as_sanitized_airbyte_message` will be discarded. """ error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor) - if error_message.trace.error.message: - error_message.trace.error.message = filter_secrets(error_message.trace.error.message) - if error_message.trace.error.internal_message: - error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message) - if error_message.trace.error.stack_trace: - error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace) + if error_message.trace.error.message: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.message = filter_secrets(error_message.trace.error.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + if error_message.trace.error.internal_message: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + if error_message.trace.error.stack_trace: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage return error_message diff --git a/airbyte-cdk/python/cdk-migrations.md b/airbyte-cdk/python/cdk-migrations.md index 2f38fb8452fe..02ebf2e751c4 100644 --- a/airbyte-cdk/python/cdk-migrations.md +++ b/airbyte-cdk/python/cdk-migrations.md @@ -1,5 +1,36 @@ # CDK Migration Guide +## Upgrading to 5.0.0 + +Version 5.0.0 of the CDK updates the `airbyte_cdk.models` dependency to replace Pydantic v2 models with Python `dataclasses`. It also +updates the `airbyte-protocol-models` dependency to a version that uses dataclasses models. + +The changes to Airbyte CDK itself are backwards-compatible, but some changes are required if the connector: +- uses the `airbyte_protocol` models directly, or `airbyte_cdk.models`, which points to `airbyte_protocol` models +- uses third-party libraries, such as `pandas`, to read data from sources, which output non-native Python objects that cannot be serialized by the [orjson](https://github.com/ijl/orjson) library. + +### Updating direct usage of Pydantic based Airbyte Protocol Models + +If the connector uses Pydantic based Airbyte Protocol Models, the code will need to be updated to reflect the changes `pydantic`. +It is recommended to import protocol classes not directly by `import airbyte_protocol` statement, but from `airbyte_cdk.models` package. +It is also recommended to use `Serializers` from `airbyte_cdk.models` to manipulate the data or convert to/from JSON. + +### Updating third-party libraries + +For example, if `pandas` outputs data from the source, which has date-time `pandas.Timestamp` object in +it, [Orjson supported Types](https://github.com/ijl/orjson?tab=readme-ov-file#types), these fields should be transformed to native JSON +objects. + +```python3 +# Before +yield from df.to_dict(orient="records") + +# After - Option 1 +yield orjson.loads(df.to_json(orient="records", date_format="iso", date_unit="us")) + +``` + + ## Upgrading to 4.5.0 In this release, we are no longer supporting the legacy state format in favor of the current per-stream state diff --git a/airbyte-cdk/python/poetry.lock b/airbyte-cdk/python/poetry.lock index f327a8d2bf00..4564f7891812 100644 --- a/airbyte-cdk/python/poetry.lock +++ b/airbyte-cdk/python/poetry.lock @@ -2,98 +2,113 @@ [[package]] name = "aiohappyeyeballs" -version = "2.3.5" +version = "2.4.0" description = "Happy Eyeballs for asyncio" optional = true python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, - {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, + {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"}, + {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"}, ] [[package]] name = "aiohttp" -version = "3.10.3" +version = "3.10.5" description = "Async http client/server framework (asyncio)" optional = true python-versions = ">=3.8" files = [ - {file = "aiohttp-3.10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc36cbdedf6f259371dbbbcaae5bb0e95b879bc501668ab6306af867577eb5db"}, - {file = "aiohttp-3.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85466b5a695c2a7db13eb2c200af552d13e6a9313d7fa92e4ffe04a2c0ea74c1"}, - {file = "aiohttp-3.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71bb1d97bfe7e6726267cea169fdf5df7658831bb68ec02c9c6b9f3511e108bb"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baec1eb274f78b2de54471fc4c69ecbea4275965eab4b556ef7a7698dee18bf2"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13031e7ec1188274bad243255c328cc3019e36a5a907978501256000d57a7201"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2bbc55a964b8eecb341e492ae91c3bd0848324d313e1e71a27e3d96e6ee7e8e8"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8cc0564b286b625e673a2615ede60a1704d0cbbf1b24604e28c31ed37dc62aa"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f817a54059a4cfbc385a7f51696359c642088710e731e8df80d0607193ed2b73"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8542c9e5bcb2bd3115acdf5adc41cda394e7360916197805e7e32b93d821ef93"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:671efce3a4a0281060edf9a07a2f7e6230dca3a1cbc61d110eee7753d28405f7"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0974f3b5b0132edcec92c3306f858ad4356a63d26b18021d859c9927616ebf27"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:44bb159b55926b57812dca1b21c34528e800963ffe130d08b049b2d6b994ada7"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6ae9ae382d1c9617a91647575255ad55a48bfdde34cc2185dd558ce476bf16e9"}, - {file = "aiohttp-3.10.3-cp310-cp310-win32.whl", hash = "sha256:aed12a54d4e1ee647376fa541e1b7621505001f9f939debf51397b9329fd88b9"}, - {file = "aiohttp-3.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:b51aef59370baf7444de1572f7830f59ddbabd04e5292fa4218d02f085f8d299"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e021c4c778644e8cdc09487d65564265e6b149896a17d7c0f52e9a088cc44e1b"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24fade6dae446b183e2410a8628b80df9b7a42205c6bfc2eff783cbeedc224a2"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bc8e9f15939dacb0e1f2d15f9c41b786051c10472c7a926f5771e99b49a5957f"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5a9ec959b5381271c8ec9310aae1713b2aec29efa32e232e5ef7dcca0df0279"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a5d0ea8a6467b15d53b00c4e8ea8811e47c3cc1bdbc62b1aceb3076403d551f"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9ed607dbbdd0d4d39b597e5bf6b0d40d844dfb0ac6a123ed79042ef08c1f87e"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3e66d5b506832e56add66af88c288c1d5ba0c38b535a1a59e436b300b57b23e"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fda91ad797e4914cca0afa8b6cccd5d2b3569ccc88731be202f6adce39503189"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:61ccb867b2f2f53df6598eb2a93329b5eee0b00646ee79ea67d68844747a418e"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d881353264e6156f215b3cb778c9ac3184f5465c2ece5e6fce82e68946868ef"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b031ce229114825f49cec4434fa844ccb5225e266c3e146cb4bdd025a6da52f1"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5337cc742a03f9e3213b097abff8781f79de7190bbfaa987bd2b7ceb5bb0bdec"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ab3361159fd3dcd0e48bbe804006d5cfb074b382666e6c064112056eb234f1a9"}, - {file = "aiohttp-3.10.3-cp311-cp311-win32.whl", hash = "sha256:05d66203a530209cbe40f102ebaac0b2214aba2a33c075d0bf825987c36f1f0b"}, - {file = "aiohttp-3.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:70b4a4984a70a2322b70e088d654528129783ac1ebbf7dd76627b3bd22db2f17"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:166de65e2e4e63357cfa8417cf952a519ac42f1654cb2d43ed76899e2319b1ee"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7084876352ba3833d5d214e02b32d794e3fd9cf21fdba99cff5acabeb90d9806"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d98c604c93403288591d7d6d7d6cc8a63459168f8846aeffd5b3a7f3b3e5e09"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d73b073a25a0bb8bf014345374fe2d0f63681ab5da4c22f9d2025ca3e3ea54fc"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8da6b48c20ce78f5721068f383e0e113dde034e868f1b2f5ee7cb1e95f91db57"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a9dcdccf50284b1b0dc72bc57e5bbd3cc9bf019060dfa0668f63241ccc16aa7"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56fb94bae2be58f68d000d046172d8b8e6b1b571eb02ceee5535e9633dcd559c"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf75716377aad2c718cdf66451c5cf02042085d84522aec1f9246d3e4b8641a6"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6c51ed03e19c885c8e91f574e4bbe7381793f56f93229731597e4a499ffef2a5"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b84857b66fa6510a163bb083c1199d1ee091a40163cfcbbd0642495fed096204"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c124b9206b1befe0491f48185fd30a0dd51b0f4e0e7e43ac1236066215aff272"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3461d9294941937f07bbbaa6227ba799bc71cc3b22c40222568dc1cca5118f68"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:08bd0754d257b2db27d6bab208c74601df6f21bfe4cb2ec7b258ba691aac64b3"}, - {file = "aiohttp-3.10.3-cp312-cp312-win32.whl", hash = "sha256:7f9159ae530297f61a00116771e57516f89a3de6ba33f314402e41560872b50a"}, - {file = "aiohttp-3.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:e1128c5d3a466279cb23c4aa32a0f6cb0e7d2961e74e9e421f90e74f75ec1edf"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d1100e68e70eb72eadba2b932b185ebf0f28fd2f0dbfe576cfa9d9894ef49752"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a541414578ff47c0a9b0b8b77381ea86b0c8531ab37fc587572cb662ccd80b88"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d5548444ef60bf4c7b19ace21f032fa42d822e516a6940d36579f7bfa8513f9c"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba2e838b5e6a8755ac8297275c9460e729dc1522b6454aee1766c6de6d56e5e"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48665433bb59144aaf502c324694bec25867eb6630fcd831f7a893ca473fcde4"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bac352fceed158620ce2d701ad39d4c1c76d114255a7c530e057e2b9f55bdf9f"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b0f670502100cdc567188c49415bebba947eb3edaa2028e1a50dd81bd13363f"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43b09f38a67679e32d380fe512189ccb0b25e15afc79b23fbd5b5e48e4fc8fd9"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:cd788602e239ace64f257d1c9d39898ca65525583f0fbf0988bcba19418fe93f"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:214277dcb07ab3875f17ee1c777d446dcce75bea85846849cc9d139ab8f5081f"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:32007fdcaab789689c2ecaaf4b71f8e37bf012a15cd02c0a9db8c4d0e7989fa8"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:123e5819bfe1b87204575515cf448ab3bf1489cdeb3b61012bde716cda5853e7"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:812121a201f0c02491a5db335a737b4113151926a79ae9ed1a9f41ea225c0e3f"}, - {file = "aiohttp-3.10.3-cp38-cp38-win32.whl", hash = "sha256:b97dc9a17a59f350c0caa453a3cb35671a2ffa3a29a6ef3568b523b9113d84e5"}, - {file = "aiohttp-3.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:3731a73ddc26969d65f90471c635abd4e1546a25299b687e654ea6d2fc052394"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38d91b98b4320ffe66efa56cb0f614a05af53b675ce1b8607cdb2ac826a8d58e"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9743fa34a10a36ddd448bba8a3adc2a66a1c575c3c2940301bacd6cc896c6bf1"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7c126f532caf238031c19d169cfae3c6a59129452c990a6e84d6e7b198a001dc"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:926e68438f05703e500b06fe7148ef3013dd6f276de65c68558fa9974eeb59ad"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:434b3ab75833accd0b931d11874e206e816f6e6626fd69f643d6a8269cd9166a"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d35235a44ec38109b811c3600d15d8383297a8fab8e3dec6147477ec8636712a"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59c489661edbd863edb30a8bd69ecb044bd381d1818022bc698ba1b6f80e5dd1"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50544fe498c81cb98912afabfc4e4d9d85e89f86238348e3712f7ca6a2f01dab"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:09bc79275737d4dc066e0ae2951866bb36d9c6b460cb7564f111cc0427f14844"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:af4dbec58e37f5afff4f91cdf235e8e4b0bd0127a2a4fd1040e2cad3369d2f06"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b22cae3c9dd55a6b4c48c63081d31c00fc11fa9db1a20c8a50ee38c1a29539d2"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ba562736d3fbfe9241dad46c1a8994478d4a0e50796d80e29d50cabe8fbfcc3f"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f25d6c4e82d7489be84f2b1c8212fafc021b3731abdb61a563c90e37cced3a21"}, - {file = "aiohttp-3.10.3-cp39-cp39-win32.whl", hash = "sha256:b69d832e5f5fa15b1b6b2c8eb6a9fd2c0ec1fd7729cb4322ed27771afc9fc2ac"}, - {file = "aiohttp-3.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:673bb6e3249dc8825df1105f6ef74e2eab779b7ff78e96c15cadb78b04a83752"}, - {file = "aiohttp-3.10.3.tar.gz", hash = "sha256:21650e7032cc2d31fc23d353d7123e771354f2a3d5b05a5647fc30fea214e696"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683"}, + {file = "aiohttp-3.10.5-cp310-cp310-win32.whl", hash = "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef"}, + {file = "aiohttp-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"}, + {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"}, + {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6"}, + {file = "aiohttp-3.10.5-cp312-cp312-win32.whl", hash = "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12"}, + {file = "aiohttp-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987"}, + {file = "aiohttp-3.10.5-cp313-cp313-win32.whl", hash = "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04"}, + {file = "aiohttp-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511"}, + {file = "aiohttp-3.10.5-cp38-cp38-win32.whl", hash = "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a"}, + {file = "aiohttp-3.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11"}, + {file = "aiohttp-3.10.5-cp39-cp39-win32.whl", hash = "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1"}, + {file = "aiohttp-3.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862"}, + {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"}, ] [package.dependencies] @@ -123,19 +138,16 @@ files = [ frozenlist = ">=1.1.0" [[package]] -name = "airbyte-protocol-models-pdv2" -version = "0.12.2" -description = "Declares the Airbyte Protocol." +name = "airbyte-protocol-models-dataclasses" +version = "0.13.0" +description = "Declares the Airbyte Protocol using Python Dataclasses. Dataclasses in Python have less performance overhead compared to Pydantic models, making them a more efficient choice for scenarios where speed and memory usage are critical" optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, - {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0-py3-none-any.whl", hash = "sha256:0aedb99ffc4f9aab0ce91bba2c292fa17cd8fd4b42eeba196d6a16c20bbbd7a5"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0.tar.gz", hash = "sha256:72e67850d661e2808406aec5839b3158ebb94d3553b798dbdae1b4a278548d2f"}, ] -[package.dependencies] -pydantic = ">=2.7.2,<3.0.0" - [[package]] name = "alabaster" version = "0.7.16" @@ -169,6 +181,28 @@ files = [ {file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"}, ] +[[package]] +name = "anyio" +version = "4.4.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = true +python-versions = ">=3.8" +files = [ + {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, + {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + [[package]] name = "async-timeout" version = "4.0.3" @@ -193,6 +227,17 @@ files = [ {file = "asyncio-3.4.3.tar.gz", hash = "sha256:83360ff8bc97980e4ff25c964c7bd3923d333d177aa4f7fb736b019f26c7cb41"}, ] +[[package]] +name = "attributes-doc" +version = "0.4.0" +description = "PEP 224 implementation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attributes-doc-0.4.0.tar.gz", hash = "sha256:b1576c94a714e9fc2c65c47cf10d0c8e1a5f7c4f5ae7f69006be108d95cbfbfb"}, + {file = "attributes_doc-0.4.0-py2.py3-none-any.whl", hash = "sha256:4c3007d9e58f3a6cb4b9c614c4d4ce2d92161581f28e594ddd8241cc3a113bdd"}, +] + [[package]] name = "attrs" version = "24.2.0" @@ -347,24 +392,24 @@ files = [ [[package]] name = "cachetools" -version = "5.4.0" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, - {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] name = "cattrs" -version = "23.2.3" +version = "24.1.0" description = "Composable complex class support for attrs and dataclasses." optional = false python-versions = ">=3.8" files = [ - {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, - {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, + {file = "cattrs-24.1.0-py3-none-any.whl", hash = "sha256:043bb8af72596432a7df63abcff0055ac0f198a4d2e95af8db5a936a7074a761"}, + {file = "cattrs-24.1.0.tar.gz", hash = "sha256:8274f18b253bf7674a43da851e3096370d67088165d23138b04a1c04c8eaf48e"}, ] [package.dependencies] @@ -376,6 +421,7 @@ typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_ver bson = ["pymongo (>=4.4.0)"] cbor2 = ["cbor2 (>=5.4.6)"] msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5)"] orjson = ["orjson (>=3.9.2)"] pyyaml = ["pyyaml (>=6.0)"] tomlkit = ["tomlkit (>=0.11.8)"] @@ -383,13 +429,13 @@ ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2024.7.4" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, - {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] @@ -597,13 +643,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "codeflash" -version = "0.6.17" +version = "0.6.19" description = "Client for codeflash.ai - automatic code performance optimization, powered by AI" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "codeflash-0.6.17-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2d767d1bf035fbfd95d4f8d62a8800ceee833cfff919ea65e6ec406e618228b3"}, - {file = "codeflash-0.6.17.tar.gz", hash = "sha256:96afca1263230c0dd0c6fc3a4601e2680fb25ffa43657310bde4cbaeb83b9000"}, + {file = "codeflash-0.6.19-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f4e19b78ef4dfbbbaeab6d953422027a6e9120545de0657e70adfe23a3fdb51a"}, + {file = "codeflash-0.6.19.tar.gz", hash = "sha256:0cb61aec565b286b1d5fcaa64e55f3e91e03bd6154e03c718c086186c30040da"}, ] [package.dependencies] @@ -662,66 +708,87 @@ files = [ [[package]] name = "contourpy" -version = "1.2.1" +version = "1.3.0" description = "Python library for calculating contours of 2D quadrilateral grids" optional = true python-versions = ">=3.9" files = [ - {file = "contourpy-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bd7c23df857d488f418439686d3b10ae2fbf9bc256cd045b37a8c16575ea1040"}, - {file = "contourpy-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b9eb0ca724a241683c9685a484da9d35c872fd42756574a7cfbf58af26677fd"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c75507d0a55378240f781599c30e7776674dbaf883a46d1c90f37e563453480"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11959f0ce4a6f7b76ec578576a0b61a28bdc0696194b6347ba3f1c53827178b9"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb3315a8a236ee19b6df481fc5f997436e8ade24a9f03dfdc6bd490fea20c6da"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39f3ecaf76cd98e802f094e0d4fbc6dc9c45a8d0c4d185f0f6c2234e14e5f75b"}, - {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:94b34f32646ca0414237168d68a9157cb3889f06b096612afdd296003fdd32fd"}, - {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:457499c79fa84593f22454bbd27670227874cd2ff5d6c84e60575c8b50a69619"}, - {file = "contourpy-1.2.1-cp310-cp310-win32.whl", hash = "sha256:ac58bdee53cbeba2ecad824fa8159493f0bf3b8ea4e93feb06c9a465d6c87da8"}, - {file = "contourpy-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9cffe0f850e89d7c0012a1fb8730f75edd4320a0a731ed0c183904fe6ecfc3a9"}, - {file = "contourpy-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6022cecf8f44e36af10bd9118ca71f371078b4c168b6e0fab43d4a889985dbb5"}, - {file = "contourpy-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef5adb9a3b1d0c645ff694f9bca7702ec2c70f4d734f9922ea34de02294fdf72"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6150ffa5c767bc6332df27157d95442c379b7dce3a38dff89c0f39b63275696f"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c863140fafc615c14a4bf4efd0f4425c02230eb8ef02784c9a156461e62c965"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:00e5388f71c1a0610e6fe56b5c44ab7ba14165cdd6d695429c5cd94021e390b2"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4492d82b3bc7fbb7e3610747b159869468079fe149ec5c4d771fa1f614a14df"}, - {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49e70d111fee47284d9dd867c9bb9a7058a3c617274900780c43e38d90fe1205"}, - {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b59c0ffceff8d4d3996a45f2bb6f4c207f94684a96bf3d9728dbb77428dd8cb8"}, - {file = "contourpy-1.2.1-cp311-cp311-win32.whl", hash = "sha256:7b4182299f251060996af5249c286bae9361fa8c6a9cda5efc29fe8bfd6062ec"}, - {file = "contourpy-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2855c8b0b55958265e8b5888d6a615ba02883b225f2227461aa9127c578a4922"}, - {file = "contourpy-1.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:62828cada4a2b850dbef89c81f5a33741898b305db244904de418cc957ff05dc"}, - {file = "contourpy-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:309be79c0a354afff9ff7da4aaed7c3257e77edf6c1b448a779329431ee79d7e"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e785e0f2ef0d567099b9ff92cbfb958d71c2d5b9259981cd9bee81bd194c9a4"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cac0a8f71a041aa587410424ad46dfa6a11f6149ceb219ce7dd48f6b02b87a7"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af3f4485884750dddd9c25cb7e3915d83c2db92488b38ccb77dd594eac84c4a0"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ce6889abac9a42afd07a562c2d6d4b2b7134f83f18571d859b25624a331c90b"}, - {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a1eea9aecf761c661d096d39ed9026574de8adb2ae1c5bd7b33558af884fb2ce"}, - {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:187fa1d4c6acc06adb0fae5544c59898ad781409e61a926ac7e84b8f276dcef4"}, - {file = "contourpy-1.2.1-cp312-cp312-win32.whl", hash = "sha256:c2528d60e398c7c4c799d56f907664673a807635b857df18f7ae64d3e6ce2d9f"}, - {file = "contourpy-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a07fc092a4088ee952ddae19a2b2a85757b923217b7eed584fdf25f53a6e7ce"}, - {file = "contourpy-1.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb6834cbd983b19f06908b45bfc2dad6ac9479ae04abe923a275b5f48f1a186b"}, - {file = "contourpy-1.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1d59e739ab0e3520e62a26c60707cc3ab0365d2f8fecea74bfe4de72dc56388f"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd3db01f59fdcbce5b22afad19e390260d6d0222f35a1023d9adc5690a889364"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a12a813949e5066148712a0626895c26b2578874e4cc63160bb007e6df3436fe"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe0ccca550bb8e5abc22f530ec0466136379c01321fd94f30a22231e8a48d985"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1d59258c3c67c865435d8fbeb35f8c59b8bef3d6f46c1f29f6123556af28445"}, - {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f32c38afb74bd98ce26de7cc74a67b40afb7b05aae7b42924ea990d51e4dac02"}, - {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d31a63bc6e6d87f77d71e1abbd7387ab817a66733734883d1fc0021ed9bfa083"}, - {file = "contourpy-1.2.1-cp39-cp39-win32.whl", hash = "sha256:ddcb8581510311e13421b1f544403c16e901c4e8f09083c881fab2be80ee31ba"}, - {file = "contourpy-1.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:10a37ae557aabf2509c79715cd20b62e4c7c28b8cd62dd7d99e5ed3ce28c3fd9"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a31f94983fecbac95e58388210427d68cd30fe8a36927980fab9c20062645609"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef2b055471c0eb466033760a521efb9d8a32b99ab907fc8358481a1dd29e3bd3"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b33d2bc4f69caedcd0a275329eb2198f560b325605810895627be5d4b876bf7f"}, - {file = "contourpy-1.2.1.tar.gz", hash = "sha256:4d8908b3bee1c889e547867ca4cdc54e5ab6be6d3e078556814a22457f49423c"}, + {file = "contourpy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:880ea32e5c774634f9fcd46504bf9f080a41ad855f4fef54f5380f5133d343c7"}, + {file = "contourpy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:76c905ef940a4474a6289c71d53122a4f77766eef23c03cd57016ce19d0f7b42"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92f8557cbb07415a4d6fa191f20fd9d2d9eb9c0b61d1b2f52a8926e43c6e9af7"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36f965570cff02b874773c49bfe85562b47030805d7d8360748f3eca570f4cab"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cacd81e2d4b6f89c9f8a5b69b86490152ff39afc58a95af002a398273e5ce589"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69375194457ad0fad3a839b9e29aa0b0ed53bb54db1bfb6c3ae43d111c31ce41"}, + {file = "contourpy-1.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a52040312b1a858b5e31ef28c2e865376a386c60c0e248370bbea2d3f3b760d"}, + {file = "contourpy-1.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3faeb2998e4fcb256542e8a926d08da08977f7f5e62cf733f3c211c2a5586223"}, + {file = "contourpy-1.3.0-cp310-cp310-win32.whl", hash = "sha256:36e0cff201bcb17a0a8ecc7f454fe078437fa6bda730e695a92f2d9932bd507f"}, + {file = "contourpy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:87ddffef1dbe5e669b5c2440b643d3fdd8622a348fe1983fad7a0f0ccb1cd67b"}, + {file = "contourpy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fa4c02abe6c446ba70d96ece336e621efa4aecae43eaa9b030ae5fb92b309ad"}, + {file = "contourpy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:834e0cfe17ba12f79963861e0f908556b2cedd52e1f75e6578801febcc6a9f49"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dbc4c3217eee163fa3984fd1567632b48d6dfd29216da3ded3d7b844a8014a66"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4865cd1d419e0c7a7bf6de1777b185eebdc51470800a9f42b9e9decf17762081"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:303c252947ab4b14c08afeb52375b26781ccd6a5ccd81abcdfc1fafd14cf93c1"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637f674226be46f6ba372fd29d9523dd977a291f66ab2a74fbeb5530bb3f445d"}, + {file = "contourpy-1.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:76a896b2f195b57db25d6b44e7e03f221d32fe318d03ede41f8b4d9ba1bff53c"}, + {file = "contourpy-1.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e1fd23e9d01591bab45546c089ae89d926917a66dceb3abcf01f6105d927e2cb"}, + {file = "contourpy-1.3.0-cp311-cp311-win32.whl", hash = "sha256:d402880b84df3bec6eab53cd0cf802cae6a2ef9537e70cf75e91618a3801c20c"}, + {file = "contourpy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:6cb6cc968059db9c62cb35fbf70248f40994dfcd7aa10444bbf8b3faeb7c2d67"}, + {file = "contourpy-1.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:570ef7cf892f0afbe5b2ee410c507ce12e15a5fa91017a0009f79f7d93a1268f"}, + {file = "contourpy-1.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:da84c537cb8b97d153e9fb208c221c45605f73147bd4cadd23bdae915042aad6"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0be4d8425bfa755e0fd76ee1e019636ccc7c29f77a7c86b4328a9eb6a26d0639"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c0da700bf58f6e0b65312d0a5e695179a71d0163957fa381bb3c1f72972537c"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb8b141bb00fa977d9122636b16aa67d37fd40a3d8b52dd837e536d64b9a4d06"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3634b5385c6716c258d0419c46d05c8aa7dc8cb70326c9a4fb66b69ad2b52e09"}, + {file = "contourpy-1.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0dce35502151b6bd35027ac39ba6e5a44be13a68f55735c3612c568cac3805fd"}, + {file = "contourpy-1.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea348f053c645100612b333adc5983d87be69acdc6d77d3169c090d3b01dc35"}, + {file = "contourpy-1.3.0-cp312-cp312-win32.whl", hash = "sha256:90f73a5116ad1ba7174341ef3ea5c3150ddf20b024b98fb0c3b29034752c8aeb"}, + {file = "contourpy-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b11b39aea6be6764f84360fce6c82211a9db32a7c7de8fa6dd5397cf1d079c3b"}, + {file = "contourpy-1.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3e1c7fa44aaae40a2247e2e8e0627f4bea3dd257014764aa644f319a5f8600e3"}, + {file = "contourpy-1.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:364174c2a76057feef647c802652f00953b575723062560498dc7930fc9b1cb7"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32b238b3b3b649e09ce9aaf51f0c261d38644bdfa35cbaf7b263457850957a84"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d51fca85f9f7ad0b65b4b9fe800406d0d77017d7270d31ec3fb1cc07358fdea0"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:732896af21716b29ab3e988d4ce14bc5133733b85956316fb0c56355f398099b"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d73f659398a0904e125280836ae6f88ba9b178b2fed6884f3b1f95b989d2c8da"}, + {file = "contourpy-1.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c6c7c2408b7048082932cf4e641fa3b8ca848259212f51c8c59c45aa7ac18f14"}, + {file = "contourpy-1.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f317576606de89da6b7e0861cf6061f6146ead3528acabff9236458a6ba467f8"}, + {file = "contourpy-1.3.0-cp313-cp313-win32.whl", hash = "sha256:31cd3a85dbdf1fc002280c65caa7e2b5f65e4a973fcdf70dd2fdcb9868069294"}, + {file = "contourpy-1.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:4553c421929ec95fb07b3aaca0fae668b2eb5a5203d1217ca7c34c063c53d087"}, + {file = "contourpy-1.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:345af746d7766821d05d72cb8f3845dfd08dd137101a2cb9b24de277d716def8"}, + {file = "contourpy-1.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3bb3808858a9dc68f6f03d319acd5f1b8a337e6cdda197f02f4b8ff67ad2057b"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:420d39daa61aab1221567b42eecb01112908b2cab7f1b4106a52caaec8d36973"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d63ee447261e963af02642ffcb864e5a2ee4cbfd78080657a9880b8b1868e18"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:167d6c890815e1dac9536dca00828b445d5d0df4d6a8c6adb4a7ec3166812fa8"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:710a26b3dc80c0e4febf04555de66f5fd17e9cf7170a7b08000601a10570bda6"}, + {file = "contourpy-1.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:75ee7cb1a14c617f34a51d11fa7524173e56551646828353c4af859c56b766e2"}, + {file = "contourpy-1.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:33c92cdae89ec5135d036e7218e69b0bb2851206077251f04a6c4e0e21f03927"}, + {file = "contourpy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a11077e395f67ffc2c44ec2418cfebed032cd6da3022a94fc227b6faf8e2acb8"}, + {file = "contourpy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e8134301d7e204c88ed7ab50028ba06c683000040ede1d617298611f9dc6240c"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e12968fdfd5bb45ffdf6192a590bd8ddd3ba9e58360b29683c6bb71a7b41edca"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fd2a0fc506eccaaa7595b7e1418951f213cf8255be2600f1ea1b61e46a60c55f"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4cfb5c62ce023dfc410d6059c936dcf96442ba40814aefbfa575425a3a7f19dc"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68a32389b06b82c2fdd68276148d7b9275b5f5cf13e5417e4252f6d1a34f72a2"}, + {file = "contourpy-1.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:94e848a6b83da10898cbf1311a815f770acc9b6a3f2d646f330d57eb4e87592e"}, + {file = "contourpy-1.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d78ab28a03c854a873787a0a42254a0ccb3cb133c672f645c9f9c8f3ae9d0800"}, + {file = "contourpy-1.3.0-cp39-cp39-win32.whl", hash = "sha256:81cb5ed4952aae6014bc9d0421dec7c5835c9c8c31cdf51910b708f548cf58e5"}, + {file = "contourpy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:14e262f67bd7e6eb6880bc564dcda30b15e351a594657e55b7eec94b6ef72843"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fe41b41505a5a33aeaed2a613dccaeaa74e0e3ead6dd6fd3a118fb471644fd6c"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca7e17a65f72a5133bdbec9ecf22401c62bcf4821361ef7811faee695799779"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1ec4dc6bf570f5b22ed0d7efba0dfa9c5b9e0431aeea7581aa217542d9e809a4"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:00ccd0dbaad6d804ab259820fa7cb0b8036bda0686ef844d24125d8287178ce0"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca947601224119117f7c19c9cdf6b3ab54c5726ef1d906aa4a69dfb6dd58102"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6ec93afeb848a0845a18989da3beca3eec2c0f852322efe21af1931147d12cb"}, + {file = "contourpy-1.3.0.tar.gz", hash = "sha256:7ffa0db17717a8ffb127efd0c95a4362d996b892c2904db72428d5b52e1938a4"}, ] [package.dependencies] -numpy = ">=1.20" +numpy = ">=1.23" [package.extras] bokeh = ["bokeh", "selenium"] docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] -mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.8.0)", "types-Pillow"] +mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.11.1)", "types-Pillow"] test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] -test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] [[package]] name = "coverage" @@ -1350,6 +1417,63 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = true +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.5" +description = "A minimal low-level HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.26.0)"] + +[[package]] +name = "httpx" +version = "0.27.2" +description = "The next generation HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "humanize" version = "4.10.0" @@ -1366,13 +1490,13 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] @@ -1824,16 +1948,17 @@ six = "*" [[package]] name = "langsmith" -version = "0.1.99" +version = "0.1.107" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = true python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.99-py3-none-any.whl", hash = "sha256:ef8d1d74a2674c514aa429b0171a9fbb661207dc3835142cca0e8f1bf97b26b0"}, - {file = "langsmith-0.1.99.tar.gz", hash = "sha256:b5c6a1f158abda61600a4a445081ee848b4a28b758d91f2793dc02aeffafcaf1"}, + {file = "langsmith-0.1.107-py3-none-any.whl", hash = "sha256:ddd0c846980474e271a553e9c220122e32d1f2ce877cc87d39ecd86726b9e78c"}, + {file = "langsmith-0.1.107.tar.gz", hash = "sha256:f44de0a5f199381d0b518ecbe295d541c44ff33d13f18098ecc54a4547eccb3f"}, ] [package.dependencies] +httpx = ">=0.23.0,<1" orjson = ">=3.9.14,<4.0.0" pydantic = [ {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""}, @@ -2057,13 +2182,13 @@ source = ["Cython (>=3.0.11)"] [[package]] name = "markdown" -version = "3.6" +version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = true python-versions = ">=3.8" files = [ - {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, - {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, + {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, + {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] [package.extras] @@ -2167,13 +2292,13 @@ files = [ [[package]] name = "marshmallow" -version = "3.21.3" +version = "3.22.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = true python-versions = ">=3.8" files = [ - {file = "marshmallow-3.21.3-py3-none-any.whl", hash = "sha256:86ce7fb914aa865001a4b2092c4c2872d13bc347f3d42673272cabfdbad386f1"}, - {file = "marshmallow-3.21.3.tar.gz", hash = "sha256:4f57c5e050a54d66361e826f94fba213eb10b67b2fdb02c3e0343ce207ba1662"}, + {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, + {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, ] [package.dependencies] @@ -2181,45 +2306,56 @@ packaging = ">=17.0" [package.extras] dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==0.7.16)", "autodocsumm (==0.2.12)", "sphinx (==7.3.7)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] +docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] tests = ["pytest", "pytz", "simplejson"] [[package]] name = "matplotlib" -version = "3.9.1.post1" +version = "3.9.2" description = "Python plotting package" optional = true python-versions = ">=3.9" files = [ - {file = "matplotlib-3.9.1.post1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3779ad3e8b72df22b8a622c5796bbcfabfa0069b835412e3c1dec8ee3de92d0c"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec400340f8628e8e2260d679078d4e9b478699f386e5cc8094e80a1cb0039c7c"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82c18791b8862ea095081f745b81f896b011c5a5091678fb33204fef641476af"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:621a628389c09a6b9f609a238af8e66acecece1cfa12febc5fe4195114ba7446"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9a54734ca761ebb27cd4f0b6c2ede696ab6861052d7d7e7b8f7a6782665115f5"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-win_amd64.whl", hash = "sha256:0721f93db92311bb514e446842e2b21c004541dcca0281afa495053e017c5458"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b08b46058fe2a31ecb81ef6aa3611f41d871f6a8280e9057cb4016cb3d8e894a"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:22b344e84fcc574f561b5731f89a7625db8ef80cdbb0026a8ea855a33e3429d1"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b49fee26d64aefa9f061b575f0f7b5fc4663e51f87375c7239efa3d30d908fa"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89eb7e89e2b57856533c5c98f018aa3254fa3789fcd86d5f80077b9034a54c9a"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c06e742bade41fda6176d4c9c78c9ea016e176cd338e62a1686384cb1eb8de41"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-win_amd64.whl", hash = "sha256:c44edab5b849e0fc1f1c9d6e13eaa35ef65925f7be45be891d9784709ad95561"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bf28b09986aee06393e808e661c3466be9c21eff443c9bc881bce04bfbb0c500"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:92aeb8c439d4831510d8b9d5e39f31c16c7f37873879767c26b147cef61e54cd"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f15798b0691b45c80d3320358a88ce5a9d6f518b28575b3ea3ed31b4bd95d009"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d59fc6096da7b9c1df275f9afc3fef5cbf634c21df9e5f844cba3dd8deb1847d"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ab986817a32a70ce22302438691e7df4c6ee4a844d47289db9d583d873491e0b"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-win_amd64.whl", hash = "sha256:0d78e7d2d86c4472da105d39aba9b754ed3dfeaeaa4ac7206b82706e0a5362fa"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bd07eba6431b4dc9253cce6374a28c415e1d3a7dc9f8aba028ea7592f06fe172"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca230cc4482010d646827bd2c6d140c98c361e769ae7d954ebf6fff2a226f5b1"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ace27c0fdeded399cbc43f22ffa76e0f0752358f5b33106ec7197534df08725a"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a4f3aeb7ba14c497dc6f021a076c48c2e5fbdf3da1e7264a5d649683e284a2f"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:23f96fbd4ff4cfa9b8a6b685a65e7eb3c2ced724a8d965995ec5c9c2b1f7daf5"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-win_amd64.whl", hash = "sha256:2808b95452b4ffa14bfb7c7edffc5350743c31bda495f0d63d10fdd9bc69e895"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ffc91239f73b4179dec256b01299d46d0ffa9d27d98494bc1476a651b7821cbe"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f965ebca9fd4feaaca45937c4849d92b70653057497181100fcd1e18161e5f29"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:801ee9323fd7b2da0d405aebbf98d1da77ea430bbbbbec6834c0b3af15e5db44"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50113e9b43ceb285739f35d43db36aa752fb8154325b35d134ff6e177452f9ec"}, - {file = "matplotlib-3.9.1.post1.tar.gz", hash = "sha256:c91e585c65092c975a44dc9d4239ba8c594ba3c193d7c478b6d178c4ef61f406"}, + {file = "matplotlib-3.9.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9d78bbc0cbc891ad55b4f39a48c22182e9bdaea7fc0e5dbd364f49f729ca1bbb"}, + {file = "matplotlib-3.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c375cc72229614632c87355366bdf2570c2dac01ac66b8ad048d2dabadf2d0d4"}, + {file = "matplotlib-3.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d94ff717eb2bd0b58fe66380bd8b14ac35f48a98e7c6765117fe67fb7684e64"}, + {file = "matplotlib-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab68d50c06938ef28681073327795c5db99bb4666214d2d5f880ed11aeaded66"}, + {file = "matplotlib-3.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:65aacf95b62272d568044531e41de26285d54aec8cb859031f511f84bd8b495a"}, + {file = "matplotlib-3.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:3fd595f34aa8a55b7fc8bf9ebea8aa665a84c82d275190a61118d33fbc82ccae"}, + {file = "matplotlib-3.9.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d8dd059447824eec055e829258ab092b56bb0579fc3164fa09c64f3acd478772"}, + {file = "matplotlib-3.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c797dac8bb9c7a3fd3382b16fe8f215b4cf0f22adccea36f1545a6d7be310b41"}, + {file = "matplotlib-3.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d719465db13267bcef19ea8954a971db03b9f48b4647e3860e4bc8e6ed86610f"}, + {file = "matplotlib-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8912ef7c2362f7193b5819d17dae8629b34a95c58603d781329712ada83f9447"}, + {file = "matplotlib-3.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7741f26a58a240f43bee74965c4882b6c93df3e7eb3de160126d8c8f53a6ae6e"}, + {file = "matplotlib-3.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:ae82a14dab96fbfad7965403c643cafe6515e386de723e498cf3eeb1e0b70cc7"}, + {file = "matplotlib-3.9.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ac43031375a65c3196bee99f6001e7fa5bdfb00ddf43379d3c0609bdca042df9"}, + {file = "matplotlib-3.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be0fc24a5e4531ae4d8e858a1a548c1fe33b176bb13eff7f9d0d38ce5112a27d"}, + {file = "matplotlib-3.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf81de2926c2db243c9b2cbc3917619a0fc85796c6ba4e58f541df814bbf83c7"}, + {file = "matplotlib-3.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6ee45bc4245533111ced13f1f2cace1e7f89d1c793390392a80c139d6cf0e6c"}, + {file = "matplotlib-3.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:306c8dfc73239f0e72ac50e5a9cf19cc4e8e331dd0c54f5e69ca8758550f1e1e"}, + {file = "matplotlib-3.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:5413401594cfaff0052f9d8b1aafc6d305b4bd7c4331dccd18f561ff7e1d3bd3"}, + {file = "matplotlib-3.9.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:18128cc08f0d3cfff10b76baa2f296fc28c4607368a8402de61bb3f2eb33c7d9"}, + {file = "matplotlib-3.9.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4876d7d40219e8ae8bb70f9263bcbe5714415acfdf781086601211335e24f8aa"}, + {file = "matplotlib-3.9.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d9f07a80deab4bb0b82858a9e9ad53d1382fd122be8cde11080f4e7dfedb38b"}, + {file = "matplotlib-3.9.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7c0410f181a531ec4e93bbc27692f2c71a15c2da16766f5ba9761e7ae518413"}, + {file = "matplotlib-3.9.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:909645cce2dc28b735674ce0931a4ac94e12f5b13f6bb0b5a5e65e7cea2c192b"}, + {file = "matplotlib-3.9.2-cp313-cp313-win_amd64.whl", hash = "sha256:f32c7410c7f246838a77d6d1eff0c0f87f3cb0e7c4247aebea71a6d5a68cab49"}, + {file = "matplotlib-3.9.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:37e51dd1c2db16ede9cfd7b5cabdfc818b2c6397c83f8b10e0e797501c963a03"}, + {file = "matplotlib-3.9.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b82c5045cebcecd8496a4d694d43f9cc84aeeb49fe2133e036b207abe73f4d30"}, + {file = "matplotlib-3.9.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f053c40f94bc51bc03832a41b4f153d83f2062d88c72b5e79997072594e97e51"}, + {file = "matplotlib-3.9.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbe196377a8248972f5cede786d4c5508ed5f5ca4a1e09b44bda889958b33f8c"}, + {file = "matplotlib-3.9.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5816b1e1fe8c192cbc013f8f3e3368ac56fbecf02fb41b8f8559303f24c5015e"}, + {file = "matplotlib-3.9.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:cef2a73d06601437be399908cf13aee74e86932a5ccc6ccdf173408ebc5f6bb2"}, + {file = "matplotlib-3.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0830e188029c14e891fadd99702fd90d317df294c3298aad682739c5533721a"}, + {file = "matplotlib-3.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ba9c1299c920964e8d3857ba27173b4dbb51ca4bab47ffc2c2ba0eb5e2cbc5"}, + {file = "matplotlib-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cd93b91ab47a3616b4d3c42b52f8363b88ca021e340804c6ab2536344fad9ca"}, + {file = "matplotlib-3.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6d1ce5ed2aefcdce11904fc5bbea7d9c21fff3d5f543841edf3dea84451a09ea"}, + {file = "matplotlib-3.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:b2696efdc08648536efd4e1601b5fd491fd47f4db97a5fbfd175549a7365c1b2"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d52a3b618cb1cbb769ce2ee1dcdb333c3ab6e823944e9a2d36e37253815f9556"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:039082812cacd6c6bec8e17a9c1e6baca230d4116d522e81e1f63a74d01d2e21"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6758baae2ed64f2331d4fd19be38b7b4eae3ecec210049a26b6a4f3ae1c85dcc"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:050598c2b29e0b9832cde72bcf97627bf00262adbc4a54e2b856426bb2ef0697"}, + {file = "matplotlib-3.9.2.tar.gz", hash = "sha256:96ab43906269ca64a6366934106fa01534454a69e471b7bf3d79083981aaab92"}, ] [package.dependencies] @@ -2447,38 +2583,38 @@ files = [ [[package]] name = "mypy" -version = "1.11.1" +version = "1.11.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a32fc80b63de4b5b3e65f4be82b4cfa362a46702672aa6a0f443b4689af7008c"}, - {file = "mypy-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1952f5ea8a5a959b05ed5f16452fddadbaae48b5d39235ab4c3fc444d5fd411"}, - {file = "mypy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1e30dc3bfa4e157e53c1d17a0dad20f89dc433393e7702b813c10e200843b03"}, - {file = "mypy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c63350af88f43a66d3dfeeeb8d77af34a4f07d760b9eb3a8697f0386c7590b4"}, - {file = "mypy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:a831671bad47186603872a3abc19634f3011d7f83b083762c942442d51c58d58"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, - {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, - {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, - {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, - {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, - {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, - {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:749fd3213916f1751fff995fccf20c6195cae941dc968f3aaadf9bb4e430e5a2"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b639dce63a0b19085213ec5fdd8cffd1d81988f47a2dec7100e93564f3e8fb3b"}, - {file = "mypy-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c956b49c5d865394d62941b109728c5c596a415e9c5b2be663dd26a1ff07bc0"}, - {file = "mypy-1.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45df906e8b6804ef4b666af29a87ad9f5921aad091c79cc38e12198e220beabd"}, - {file = "mypy-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:d44be7551689d9d47b7abc27c71257adfdb53f03880841a5db15ddb22dc63edb"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2684d3f693073ab89d76da8e3921883019ea8a3ec20fa5d8ecca6a2db4c54bbe"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79c07eb282cb457473add5052b63925e5cc97dfab9812ee65a7c7ab5e3cb551c"}, - {file = "mypy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11965c2f571ded6239977b14deebd3f4c3abd9a92398712d6da3a772974fad69"}, - {file = "mypy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a2b43895a0f8154df6519706d9bca8280cda52d3d9d1514b2d9c3e26792a0b74"}, - {file = "mypy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a81cf05975fd61aec5ae16501a091cfb9f605dc3e3c878c0da32f250b74760b"}, - {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, - {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, + {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, + {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, + {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, + {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, + {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, + {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, + {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, + {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, + {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, ] [package.dependencies] @@ -3016,13 +3152,13 @@ type = ["mypy (>=1.8)"] [[package]] name = "plotly" -version = "5.23.0" +version = "5.24.0" description = "An open-source, interactive data visualization library for Python" optional = true python-versions = ">=3.8" files = [ - {file = "plotly-5.23.0-py3-none-any.whl", hash = "sha256:76cbe78f75eddc10c56f5a4ee3e7ccaade7c0a57465546f02098c0caed6c2d1a"}, - {file = "plotly-5.23.0.tar.gz", hash = "sha256:89e57d003a116303a34de6700862391367dd564222ab71f8531df70279fc0193"}, + {file = "plotly-5.24.0-py3-none-any.whl", hash = "sha256:0e54efe52c8cef899f7daa41be9ed97dfb6be622613a2a8f56a86a0634b2b67e"}, + {file = "plotly-5.24.0.tar.gz", hash = "sha256:eae9f4f54448682442c92c1e97148e3ad0c52f0cf86306e1b76daba24add554a"}, ] [package.dependencies] @@ -3064,13 +3200,13 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] [[package]] name = "posthog" -version = "3.5.0" +version = "3.6.0" description = "Integrate PostHog into any python application." optional = false python-versions = "*" files = [ - {file = "posthog-3.5.0-py2.py3-none-any.whl", hash = "sha256:3c672be7ba6f95d555ea207d4486c171d06657eb34b3ce25eb043bfe7b6b5b76"}, - {file = "posthog-3.5.0.tar.gz", hash = "sha256:8f7e3b2c6e8714d0c0c542a2109b83a7549f63b7113a133ab2763a89245ef2ef"}, + {file = "posthog-3.6.0-py2.py3-none-any.whl", hash = "sha256:6f8dacc6d14d80734b1d15bd4ab08b049629c5f0fc420cafcf1ce0667c76c83c"}, + {file = "posthog-3.6.0.tar.gz", hash = "sha256:27dbf537241a69fb5f6a3e9561caa2d555d5891d95fa65c27ffa6b52d1fb63b6"}, ] [package.dependencies] @@ -3322,13 +3458,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = true python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -4094,13 +4230,13 @@ compatible-mypy = ["mypy (>=1.10,<1.11)"] [[package]] name = "rich" -version = "13.7.1" +version = "13.8.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "rich-13.8.0-py3-none-any.whl", hash = "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc"}, + {file = "rich-13.8.0.tar.gz", hash = "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"}, ] [package.dependencies] @@ -4171,36 +4307,44 @@ tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc ( [[package]] name = "scipy" -version = "1.14.0" +version = "1.14.1" description = "Fundamental algorithms for scientific computing in Python" optional = true python-versions = ">=3.10" files = [ - {file = "scipy-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e911933d54ead4d557c02402710c2396529540b81dd554fc1ba270eb7308484"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:687af0a35462402dd851726295c1a5ae5f987bd6e9026f52e9505994e2f84ef6"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:07e179dc0205a50721022344fb85074f772eadbda1e1b3eecdc483f8033709b7"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:6a9c9a9b226d9a21e0a208bdb024c3982932e43811b62d202aaf1bb59af264b1"}, - {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:076c27284c768b84a45dcf2e914d4000aac537da74236a0d45d82c6fa4b7b3c0"}, - {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42470ea0195336df319741e230626b6225a740fd9dce9642ca13e98f667047c0"}, - {file = "scipy-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:176c6f0d0470a32f1b2efaf40c3d37a24876cebf447498a4cefb947a79c21e9d"}, - {file = "scipy-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:ad36af9626d27a4326c8e884917b7ec321d8a1841cd6dacc67d2a9e90c2f0359"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6d056a8709ccda6cf36cdd2eac597d13bc03dba38360f418560a93050c76a16e"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f0a50da861a7ec4573b7c716b2ebdcdf142b66b756a0d392c236ae568b3a93fb"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:94c164a9e2498e68308e6e148646e486d979f7fcdb8b4cf34b5441894bdb9caf"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a7d46c3e0aea5c064e734c3eac5cf9eb1f8c4ceee756262f2c7327c4c2691c86"}, - {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eee2989868e274aae26125345584254d97c56194c072ed96cb433f32f692ed8"}, - {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3154691b9f7ed73778d746da2df67a19d046a6c8087c8b385bc4cdb2cfca74"}, - {file = "scipy-1.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c40003d880f39c11c1edbae8144e3813904b10514cd3d3d00c277ae996488cdb"}, - {file = "scipy-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b083c8940028bb7e0b4172acafda6df762da1927b9091f9611b0bcd8676f2bc"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:64b2ff514a98cf2bb734a9f90d32dc89dc6ad4a4a36a312cd0d6327170339eb0"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:7d3da42fbbbb860211a811782504f38ae7aaec9de8764a9bef6b262de7a2b50f"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a01cc03bcdc777c9da3cfdcc74b5a75caffb48a6c39c8450a9a05f82c4250a14"}, - {file = "scipy-1.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:65df4da3c12a2bb9ad52b86b4dcf46813e869afb006e58be0f516bc370165159"}, - {file = "scipy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:4c4161597c75043f7154238ef419c29a64ac4a7c889d588ea77690ac4d0d9b20"}, - {file = "scipy-1.14.0.tar.gz", hash = "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, + {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, + {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, + {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, + {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1729560c906963fc8389f6aac023739ff3983e727b1a4d87696b7bf108316a79"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:4079b90df244709e675cdc8b93bfd8a395d59af40b72e339c2287c91860deb8e"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e0cf28db0f24a38b2a0ca33a85a54852586e43cf6fd876365c86e0657cfe7d73"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0c2f95de3b04e26f5f3ad5bb05e74ba7f68b837133a4492414b3afd79dfe540e"}, + {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99722ea48b7ea25e8e015e8341ae74624f72e5f21fc2abd45f3a93266de4c5d"}, + {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5149e3fd2d686e42144a093b206aef01932a0059c2a33ddfa67f5f035bdfe13e"}, + {file = "scipy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4f5a7c49323533f9103d4dacf4e4f07078f360743dec7f7596949149efeec06"}, + {file = "scipy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:baff393942b550823bfce952bb62270ee17504d02a1801d7fd0719534dfb9c84"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, ] [package.dependencies] @@ -4208,8 +4352,8 @@ numpy = ">=1.23.5,<2.3" [package.extras] dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] -doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<=7.3.7)", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentry-sdk" @@ -4262,21 +4406,71 @@ starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] tornado = ["tornado (>=6)"] +[[package]] +name = "serpyco-rs" +version = "1.10.2" +description = "" +optional = false +python-versions = ">=3.9" +files = [ + {file = "serpyco_rs-1.10.2-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e01d824fdebb9bded57ec40b9ac0ca3b312ad617fd5deba61113a3b23bcb915d"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ef9a31f8d62c17b1ccfffb3e91c5aed2d6fd2187c7611ee3ca1b572046150cd"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aab2241b2d87bca5f15d5d34a3948b1c9ad1724cc55d1332e0c5325aff02635f"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:87d8118e9ba6e37aee1b0f7c14b19fe494f1589dc81ae0cc5168812779e1bfab"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d18a77d23aeb49904b2462410e57b4027511158845291bf6251e5857a881d60"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8da7ff487ada75f6b724d6ef9e40cde5cf703a2b89e6a3f466a8db0049e153a"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5212fa00ff8874ecabca0cf5f11eb7c1291b55ec9ee6aa7ee3ae2ec344abcf7f"}, + {file = "serpyco_rs-1.10.2-cp310-none-win_amd64.whl", hash = "sha256:ff83f5296f0ab08e77d09a4888020e636d4385a642fec52eacd2ab480d0ec22c"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d0e6d6546145ba30d6032381b27261e338f7c1b96b9fb0773a481970a809827"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf4d5c69d1fcd7007b7792cb5ea62a0702822f6f8982349f44b795677ab7414c"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fc4c1241c0707bfdd93991c0a2cea3f51a17acad343d9b5c296fc0a9f044d78"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:413fe29db4cab826269371a89ff9ccbd897ee7ff0eaaf1090362fdb86d5b8beb"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54ce4d5ac0ac4d62911998bfba1ac149a61c43f5dbfa23f831f0d87290c1861a"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd9c8a31440a3158c601fdcd523e77cd5fefa2ae5be061a4151c38a7a6060624"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8e323f5420c3e6f99627291a2d47d7fcd7f5c4433aaa6cc35e15d5b22ba19d6"}, + {file = "serpyco_rs-1.10.2-cp311-none-win_amd64.whl", hash = "sha256:743c1e1943f51883cb498c2c16c5f49bab2adb991c842077fcd0fa5a1658da25"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6379d789daff44e5f535d7e1c0131b30cee86988e9561cc9d98e87021188220d"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805162d7b67fd08b04b1e2ef1deeaedc37c7ee24a200f24778fb98b9fe7f5cdd"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1366df15ae2548a8a063eca84b9a8c2af92ac55df73ce60a7c4f2dfe71e2526b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:35d0a1a1a69ae074b123f6ad1487dc67717727d9dce4f95a393298743d60aafb"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a79517070e0b021803cafdf11d326e1149eac4a226443040e9fa1492c74337b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdd2b8d3b9160ddcab0400ca5e258c16e870ae49c6586ed5405c18e8910c957b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:045965a32c651797a73c7b7165165ed0d78efc233af4bf24c47acd41d222fae8"}, + {file = "serpyco_rs-1.10.2-cp312-none-win_amd64.whl", hash = "sha256:c6c95f6c9e04af94c33e4e514291df7380c3960a155e9fe264ccaaa46d4d0de8"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f19a82836699d102b288b17ae370dd4d37af60ccd2254f5bfdbd053d168cecee"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c3830bb3f6a342825e27592e86baa46774bfb1f08c82dbf561b5f1380a18b48"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f726392e6380b1e7d642d7633ac27929c8616a59db0a54632f5a9ab80987e071"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9ce029f8f29f4f335d0f3c9e005b71d7e8a934735d9654e3f03ccc54d50c107a"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e1f011370259602b55141ce866bf31dcdc9d8b68105c32f18ee442bc651ee880"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14d9e22016e2860c1f524aa123cfadd4a4eea25af10d1be76cc3d97d9c85c2e2"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:441b8045f91f30120c00a1f617a0ad6f22c1753c6b98899e8476d6e7775a3667"}, + {file = "serpyco_rs-1.10.2-cp39-none-win_amd64.whl", hash = "sha256:a124608cc998e3854fc743dea5dd7d948edbeaa70c1c1777b6dbb4b64ce465b0"}, + {file = "serpyco_rs-1.10.2.tar.gz", hash = "sha256:9cf06956eb14b326e522c9665aa5136f8fd7ece2df8a393c2e84bee8204362d0"}, +] + +[package.dependencies] +attributes-doc = "*" +typing-extensions = "*" + [[package]] name = "setuptools" -version = "72.1.0" +version = "74.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-74.0.0-py3-none-any.whl", hash = "sha256:0274581a0037b638b9fc1c6883cc71c0210865aaa76073f7882376b641b84e8f"}, + {file = "setuptools-74.0.0.tar.gz", hash = "sha256:a85e96b8be2b906f3e3e789adec6a9323abf79758ecfa3065bd740d81158b11e"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] [[package]] name = "six" @@ -4300,6 +4494,17 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -4313,13 +4518,13 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." optional = true python-versions = ">=3.8" files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] @@ -4587,17 +4792,18 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "textual" -version = "0.76.0" +version = "0.79.0" description = "Modern Text User Interface framework" optional = false python-versions = "<4.0.0,>=3.8.1" files = [ - {file = "textual-0.76.0-py3-none-any.whl", hash = "sha256:e2035609c889dba507d34a5d7b333f1c8c53a29fb170962cb92101507663517a"}, - {file = "textual-0.76.0.tar.gz", hash = "sha256:b12e8879d591090c0901b5cb8121d086e28e677353b368292d3865ec99b83b70"}, + {file = "textual-0.79.0-py3-none-any.whl", hash = "sha256:59785f20e13b0e530e3d21c0fca5eb09bd1ff329f47abce29a8e50a59646228d"}, + {file = "textual-0.79.0.tar.gz", hash = "sha256:b5ae63ae11227c158da90e486e99a6db7ef198470219edaf8c200a999d27577a"}, ] [package.dependencies] markdown-it-py = {version = ">=2.1.0", extras = ["linkify", "plugins"]} +platformdirs = ">=4.2.2,<5.0.0" rich = ">=13.3.3" typing-extensions = ">=4.4.0,<5.0.0" @@ -4881,13 +5087,13 @@ xlsx = ["networkx", "openpyxl", "pandas", "xlrd"] [[package]] name = "unstructured-pytesseract" -version = "0.3.12" +version = "0.3.13" description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" optional = true python-versions = ">=3.8" files = [ - {file = "unstructured.pytesseract-0.3.12-py3-none-any.whl", hash = "sha256:6ed42530fc697bb08d1ae4884cc517ee808620c1c1414efe8d5d90334da068d3"}, - {file = "unstructured.pytesseract-0.3.12.tar.gz", hash = "sha256:751a21d67b1f109036bf4daf796d3e04631697a355efd650f3373412b249de2e"}, + {file = "unstructured.pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:8001bc860470d56185176eb3ceb4623e888eba058ca3b30af79003784bc40e19"}, + {file = "unstructured.pytesseract-0.3.13.tar.gz", hash = "sha256:ff2e6391496e457dbf4b4e327f4a4577cce18921ea6570dc74bd64381b10e963"}, ] [package.dependencies] @@ -4952,13 +5158,13 @@ files = [ [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, ] [package.dependencies] @@ -5173,18 +5379,22 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.20.0" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.8" files = [ - {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, - {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [extras] file-based = ["avro", "fastavro", "markdown", "pandas", "pdf2image", "pdfminer.six", "pyarrow", "pytesseract", "python-calamine", "unstructured", "unstructured.pytesseract"] @@ -5194,4 +5404,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a61b0b329edd46e9efd6ff722f9518d63210de55c4f770a29785e630c372bb0e" +content-hash = "1759d8574c392cf39fccff997263873168087159c5f741314ceff6db4e5a32af" diff --git a/airbyte-cdk/python/pyproject.toml b/airbyte-cdk/python/pyproject.toml index 0eda309b44a8..4b07bed0626f 100644 --- a/airbyte-cdk/python/pyproject.toml +++ b/airbyte-cdk/python/pyproject.toml @@ -22,9 +22,10 @@ classifiers = [ ] keywords = ["airbyte", "connector-development-kit", "cdk"] + [tool.poetry.dependencies] python = "^3.10" -airbyte-protocol-models-pdv2 = "^0.12.2" +airbyte-protocol-models-dataclasses = "^0.13" backoff = "*" cachetools = "*" Deprecated = "~1.2" @@ -66,6 +67,7 @@ pyjwt = "^2.8.0" cryptography = "^42.0.5" pytz = "2024.1" orjson = "^3.10.7" +serpyco-rs = "^1.10.2" [tool.poetry.group.dev.dependencies] freezegun = "*" diff --git a/airbyte-cdk/python/unit_tests/conftest.py b/airbyte-cdk/python/unit_tests/conftest.py index a5883fe095a5..5d1e1f03f342 100644 --- a/airbyte-cdk/python/unit_tests/conftest.py +++ b/airbyte-cdk/python/unit_tests/conftest.py @@ -10,6 +10,6 @@ @pytest.fixture() def mock_sleep(monkeypatch): - with freezegun.freeze_time(datetime.datetime.now(), ignore=['_pytest.runner', '_pytest.terminal']) as frozen_datetime: - monkeypatch.setattr('time.sleep', lambda x: frozen_datetime.tick(x)) + with freezegun.freeze_time(datetime.datetime.now(), ignore=["_pytest.runner", "_pytest.terminal"]) as frozen_datetime: + monkeypatch.setattr("time.sleep", lambda x: frozen_datetime.tick(x)) yield diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index a967087f0d0e..ca6b8e47ea68 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -27,11 +27,13 @@ from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream, AirbyteStreamState, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, @@ -46,6 +48,7 @@ from airbyte_cdk.sources.declarative.retrievers import SimpleRetrieverTestReadDecorator from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets, update_secrets +from orjson import orjson from unit_tests.connector_builder.utils import create_configured_catalog _stream_name = "stream_with_custom_requester" @@ -73,8 +76,8 @@ }, ], "parent_state": {}, - } - ) + }, + ), ) ] @@ -277,13 +280,13 @@ def _mocked_send(self, request, **kwargs) -> requests.Response: def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request", return_value=AirbyteMessage(type=MessageType.RECORD)) as patched_handle: handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)]) assert patched_handle.call_count == 1 def test_handle_test_read(valid_read_config_file, configured_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request", return_value=AirbyteMessage(type=MessageType.RECORD)) as patch: handle_request(["read", "--config", str(valid_read_config_file), "--catalog", str(configured_catalog)]) assert patch.call_count == 1 @@ -487,11 +490,14 @@ def test_read(): limits = TestReadLimits() with patch("airbyte_cdk.connector_builder.message_grouper.MessageGrouper.get_message_groups", return_value=stream_read) as mock: output_record = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits + source, "test_read", config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits ) - mock.assert_called_with(source, config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits.max_records) + mock.assert_called_with(source, config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits.max_records) output_record.record.emitted_at = 1 - assert output_record == expected_airbyte_message + assert ( + orjson.dumps(AirbyteMessageSerializer.dump(output_record)).decode() + == orjson.dumps(AirbyteMessageSerializer.dump(expected_airbyte_message)).decode() + ) def test_config_update(): @@ -523,7 +529,12 @@ def test_config_update(): return_value=refresh_request_response, ): output = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, TestReadLimits() + source, + "test_read", + config, + ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), + _A_PER_PARTITION_STATE, + TestReadLimits(), ) assert output.record.data["latest_config_update"] @@ -560,7 +571,7 @@ def check_config_against_spec(self): source = MockManifestDeclarativeSource() limits = TestReadLimits() - response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits) + response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits) expected_stream_read = StreamRead( logs=[LogMessage("error_message - a stack trace", "ERROR")], @@ -584,13 +595,8 @@ def test_handle_429_response(): response = _create_429_page_response({"result": [{"error": "too many requests"}], "_metadata": {"next": "next"}}) # Add backoff strategy to avoid default endless backoff loop - TEST_READ_CONFIG["__injected_declarative_manifest"]['definitions']['retriever']['requester']['error_handler'] = { - "backoff_strategies": [ - { - "type": "ConstantBackoffStrategy", - "backoff_time_in_seconds": 5 - } - ] + TEST_READ_CONFIG["__injected_declarative_manifest"]["definitions"]["retriever"]["requester"]["error_handler"] = { + "backoff_strategies": [{"type": "ConstantBackoffStrategy", "backoff_time_in_seconds": 5}] } config = TEST_READ_CONFIG @@ -599,7 +605,7 @@ def test_handle_429_response(): with patch("requests.Session.send", return_value=response) as mock_send: response = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, limits + source, "test_read", config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, limits ) mock_send.assert_called_once() diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index b865c719b211..41ce94513560 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -23,6 +23,7 @@ StreamDescriptor, ) from airbyte_cdk.models import Type as MessageType +from orjson import orjson from unit_tests.connector_builder.utils import create_configured_catalog _NO_PK = [[]] @@ -147,7 +148,10 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert actual_response.inferred_schema == expected_schema @@ -212,7 +216,10 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) single_slice = actual_response.slices[0] for i, actual_page in enumerate(single_slice.pages): @@ -230,7 +237,9 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: ], ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") -def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int, should_fail: bool) -> None: +def test_get_grouped_messages_record_limit( + mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int, should_fail: bool +) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { "headers": {"Content-Type": "application/json"}, @@ -258,11 +267,19 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r if should_fail: with pytest.raises(ValueError): api.get_message_groups( - mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=request_record_limit + mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, + record_limit=request_record_limit, ) else: actual_response: StreamRead = api.get_message_groups( - mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=request_record_limit + mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, + record_limit=request_record_limit, ) single_slice = actual_response.slices[0] total_records = 0 @@ -338,7 +355,9 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) with pytest.raises(ValueError): - api.get_message_groups(source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=0) + api.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=0 + ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") @@ -386,7 +405,10 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = message_grouper.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) single_slice = actual_response.slices[0] @@ -484,7 +506,10 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert not stream_read.test_read_limit_reached @@ -501,7 +526,10 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No assert len(stream_read.slices[1].pages[1].records) == 1 assert len(stream_read.slices[1].pages[2].records) == 0 - assert stream_read.slices[1].state[0].stream.stream_state == AirbyteStateBlob(a_timestamp=123) + assert ( + orjson.dumps(stream_read.slices[1].state[0].stream.stream_state).decode() + == orjson.dumps(AirbyteStateBlob(a_timestamp=123)).decode() + ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") @@ -516,7 +544,10 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.test_read_limit_reached @@ -535,7 +566,10 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.test_read_limit_reached @@ -550,7 +584,10 @@ def test_read_stream_returns_error_if_stream_does_not_exist() -> None: message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response = message_grouper.get_message_groups( - source=mock_source, config=full_config, configured_catalog=create_configured_catalog("not_in_manifest"), state=_NO_STATE, + source=mock_source, + config=full_config, + configured_catalog=create_configured_catalog("not_in_manifest"), + state=_NO_STATE, ) assert len(actual_response.logs) == 1 @@ -566,7 +603,10 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == updated_config @@ -591,7 +631,10 @@ def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_em ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == latest_config @@ -616,7 +659,10 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == latest_config @@ -646,11 +692,16 @@ def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_read: Mock) -> None: - mock_source = make_mock_source(mock_entrypoint_read, iter([ - request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), - record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), - record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), - ])) + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + [ + request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), + record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), + ] + ), + ) mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = [["id"]] mock_source.streams.return_value[0].cursor_field = _NO_CURSOR_FIELD @@ -665,11 +716,16 @@ def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_re @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_cursor_field_then_ensure_cursor_field_is_pass_to_schema_inferrence(mock_entrypoint_read: Mock) -> None: - mock_source = make_mock_source(mock_entrypoint_read, iter([ - request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), - record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), - record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), - ])) + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + [ + request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), + record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), + ] + ), + ) mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = _NO_PK mock_source.streams.return_value[0].cursor_field = ["date"] @@ -709,10 +765,10 @@ def record_message(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: def state_message(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=MessageType.STATE, state=AirbyteStateMessage(stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream), - stream_state=data - ))) + return AirbyteMessage( + type=MessageType.STATE, + state=AirbyteStateMessage(stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream), stream_state=data)), + ) def slice_message(slice_descriptor: str = '{"key": "value"}') -> AirbyteMessage: diff --git a/airbyte-cdk/python/unit_tests/connector_builder/utils.py b/airbyte-cdk/python/unit_tests/connector_builder/utils.py index 15abdd30b9d9..a94a0416437c 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/utils.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/utils.py @@ -4,7 +4,7 @@ from typing import Any, Mapping -from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer def create_configured_catalog_dict(stream_name: str) -> Mapping[str, Any]: @@ -24,4 +24,4 @@ def create_configured_catalog_dict(stream_name: str) -> Mapping[str, Any]: def create_configured_catalog(stream_name: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog.parse_obj(create_configured_catalog_dict(stream_name)) + return ConfiguredAirbyteCatalogSerializer.load(create_configured_catalog_dict(stream_name)) diff --git a/airbyte-cdk/python/unit_tests/destinations/test_destination.py b/airbyte-cdk/python/unit_tests/destinations/test_destination.py index 89d16453d530..a03d7ffcc6b0 100644 --- a/airbyte-cdk/python/unit_tests/destinations/test_destination.py +++ b/airbyte-cdk/python/unit_tests/destinations/test_destination.py @@ -16,10 +16,12 @@ AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, @@ -27,6 +29,7 @@ SyncMode, Type, ) +from orjson import orjson @pytest.fixture(name="destination") @@ -194,7 +197,7 @@ def test_run_check_with_invalid_config(self, mocker, destination: Destination, t parsed_args = argparse.Namespace(**args) destination.run_cmd(parsed_args) - spec = {'type': 'integer'} + spec = {"type": "integer"} spec_msg = ConnectorSpecification(connectionSpecification=spec) mocker.patch.object(destination, "spec", return_value=spec_msg) @@ -214,7 +217,7 @@ def test_run_check_with_invalid_config(self, mocker, destination: Destination, t assert returned_check_result.type == Type.CONNECTION_STATUS assert returned_check_result.connectionStatus.status == Status.FAILED # the specific phrasing is not relevant, so only check for the keywords - assert 'validation error' in returned_check_result.connectionStatus.message + assert "validation error" in returned_check_result.connectionStatus.message def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch): config_path, dummy_config = tmp_path / "config.json", {"user": "sherif"} @@ -230,7 +233,7 @@ def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch ] ) catalog_path = tmp_path / "catalog.json" - write_file(catalog_path, dummy_catalog.json(exclude_unset=True)) + write_file(catalog_path, ConfiguredAirbyteCatalogSerializer.dump(dummy_catalog)) args = {"command": "write", "config": config_path, "catalog": catalog_path} parsed_args = argparse.Namespace(**args) @@ -244,7 +247,7 @@ def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch validate_mock = mocker.patch("airbyte_cdk.destinations.destination.check_config_against_spec_or_exit") # mock input is a record followed by some state messages mocked_input: List[AirbyteMessage] = [_wrapped(_record("s1", {"k1": "v1"})), *expected_write_result] - mocked_stdin_string = "\n".join([record.json(exclude_unset=True) for record in mocked_input]) + mocked_stdin_string = "\n".join([orjson.dumps(AirbyteMessageSerializer.dump(record)).decode() for record in mocked_input]) mocked_stdin_string += "\n add this non-serializable string to verify the destination does not break on malformed input" mocked_stdin = io.TextIOWrapper(io.BytesIO(bytes(mocked_stdin_string, "utf-8"))) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py index 41da64916368..db3ce730c89e 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py @@ -14,8 +14,14 @@ SeparatorSplitterConfigModel, ) from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ( + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py index a5f22b752ed2..600a4c0890d3 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py @@ -24,7 +24,7 @@ OpenAICompatibleEmbedder, OpenAIEmbedder, ) -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py index c906d0f3e9b5..ac831694c726 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py @@ -7,12 +7,13 @@ import pytest from airbyte_cdk.destinations.vector_db_based import ProcessingConfigModel, Writer -from airbyte_cdk.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, Type, ) @@ -61,7 +62,7 @@ def test_write(omit_raw_text: bool): """ config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) - configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj({"streams": [generate_stream()]}) + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalogSerializer.load({"streams": [generate_stream()]}) # messages are flushed after 32 records or after a state message, so this will trigger two batches to be processed input_messages = [_generate_record_message(i) for i in range(BATCH_SIZE + 5)] state_message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage()) @@ -126,7 +127,7 @@ def test_write_stream_namespace_split(): """ config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) - configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj( + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalogSerializer.load( { "streams": [ generate_stream(), diff --git a/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py index f6ff8684fa94..1c7315cb6969 100644 --- a/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +++ b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py @@ -43,7 +43,15 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ - StreamFacade.create_from_stream(s, self, self._logger, None, FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository())) if is_concurrent else s + StreamFacade.create_from_stream( + s, + self, + self._logger, + None, + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), + ) + if is_concurrent + else s for s, is_concurrent in self._streams_to_is_concurrent.items() ] @@ -96,7 +104,13 @@ def test_concurrent_source_adapter(as_stream_status, remove_stack_trace): assert records == expected_records - unavailable_stream_trace_messages = [m for m in messages if m.type == MessageType.TRACE and m.trace.type == TraceType.STREAM_STATUS and m.trace.stream_status.status == AirbyteStreamStatus.INCOMPLETE] + unavailable_stream_trace_messages = [ + m + for m in messages + if m.type == MessageType.TRACE + and m.trace.type == TraceType.STREAM_STATUS + and m.trace.stream_status.status == AirbyteStreamStatus.INCOMPLETE + ] expected_status = [as_stream_status("s3", AirbyteStreamStatus.INCOMPLETE)] assert len(unavailable_stream_trace_messages) == 1 @@ -133,7 +147,9 @@ def _configured_catalog(streams: List[Stream]): @pytest.mark.parametrize("raise_exception_on_missing_stream", [True, False]) -def test_read_nonexistent_concurrent_stream_emit_incomplete_stream_status(mocker, remove_stack_trace, as_stream_status, raise_exception_on_missing_stream): +def test_read_nonexistent_concurrent_stream_emit_incomplete_stream_status( + mocker, remove_stack_trace, as_stream_status, raise_exception_on_missing_stream +): """ Tests that attempting to sync a stream which the source does not return from the `streams` method emits incomplete stream status. """ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py index b625ddd5b357..51bef48230c9 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py @@ -19,11 +19,18 @@ class TestJwtAuthenticator: """ @pytest.mark.parametrize( - "algorithm, kid, typ, cty, additional_jwt_headers, expected", - [ - ("ALGORITHM", "test_kid", "test_typ", "test_cty", {"test": "test"}, {"kid": "test_kid", "typ": "test_typ", "cty": "test_cty", "test": "test", "alg": "ALGORITHM"}), - ("ALGORITHM", None, None, None, None, {"alg": "ALGORITHM"}) - ] + "algorithm, kid, typ, cty, additional_jwt_headers, expected", + [ + ( + "ALGORITHM", + "test_kid", + "test_typ", + "test_cty", + {"test": "test"}, + {"kid": "test_kid", "typ": "test_typ", "cty": "test_cty", "test": "test", "alg": "ALGORITHM"}, + ), + ("ALGORITHM", None, None, None, None, {"alg": "ALGORITHM"}), + ], ) def test_get_jwt_headers(self, algorithm, kid, typ, cty, additional_jwt_headers, expected): authenticator = JwtAuthenticator( @@ -61,14 +68,8 @@ def test_given_overriden_reserverd_properties_get_jwt_headers_throws_error(self) {"test": "test"}, {"iss": "test_iss", "sub": "test_sub", "aud": "test_aud", "test": "test"}, ), - ( - None, - None, - None, - None, - {} - ), - ] + (None, None, None, None, {}), + ], ) def test_get_jwt_payload(self, iss, sub, aud, additional_jwt_payload, expected): authenticator = JwtAuthenticator( @@ -105,7 +106,7 @@ def test_given_overriden_reserverd_properties_get_jwt_payload_throws_error(self) [ (True, "test", base64.b64encode("test".encode()).decode()), (False, "test", "test"), - ] + ], ) def test_get_secret_key(self, base64_encode_secret_key, secret_key, expected): authenticator = JwtAuthenticator( @@ -152,13 +153,7 @@ def test_given_invalid_algorithm_get_signed_token_throws_error(self): with pytest.raises(ValueError): authenticator._get_signed_token() - @pytest.mark.parametrize( - "header_prefix, expected", - [ - ("test", "test"), - (None, None) - ] - ) + @pytest.mark.parametrize("header_prefix, expected", [("test", "test"), (None, None)]) def test_get_header_prefix(self, header_prefix, expected): authenticator = JwtAuthenticator( config={}, diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 8ccf70b4e7a9..4ebe449dcd69 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -92,9 +92,7 @@ def test_check_stream_with_no_stream_slices_aborts(): "test_stream_unavailable_handled_error", 403, False, - [ - "Forbidden. You don't have permission to access this resource." - ], + ["Forbidden. You don't have permission to access this resource."], ), ("test_stream_available", 200, True, []), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py index 0a25be1129c1..1a7d45f7a78f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py @@ -55,7 +55,12 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), ("test_format_timestamp_ms", datetime.datetime(2021, 1, 1, 0, 0, 0, 1000, tzinfo=datetime.timezone.utc), "%ms", "1609459200001"), - ("test_format_timestamp_as_float", datetime.datetime(2023, 1, 30, 15, 28, 28, 873709, tzinfo=datetime.timezone.utc), "%s_as_float", "1675092508.873709"), + ( + "test_format_timestamp_as_float", + datetime.datetime(2023, 1, 30, 15, 28, 28, 873709, tzinfo=datetime.timezone.utc), + "%s_as_float", + "1675092508.873709", + ), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py index 84a63969cec6..ff9aedf0752a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py @@ -114,11 +114,12 @@ def test_min_max_datetime_lazy_eval(): @pytest.mark.parametrize( - "input_datetime", [ + "input_datetime", + [ pytest.param("2022-01-01T00:00:00", id="test_create_min_max_datetime_from_string"), pytest.param(InterpolatedString.create("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_string"), - pytest.param(MinMaxDatetime("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_minmaxdatetime") - ] + pytest.param(MinMaxDatetime("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_minmaxdatetime"), + ], ) def test_create_min_max_datetime(input_datetime): minMaxDatetime = MinMaxDatetime.create(input_datetime, parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py b/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py index 52bc55201bbe..65ed78698ca6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py @@ -7,19 +7,15 @@ import pytest import requests from airbyte_cdk import YamlDeclarativeSource +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory -from airbyte_protocol.models import SyncMode @pytest.mark.parametrize( "response_body, first_element", - [ - ("", {}), - ("[]", {}), - ('{"healthcheck": {"status": "ok"}}', {"healthcheck": {"status": "ok"}}) - ], + [("", {}), ("[]", {}), ('{"healthcheck": {"status": "ok"}}', {"healthcheck": {"status": "ok"}})], ) def test_json_decoder(requests_mock, response_body, first_element): requests_mock.register_uri("GET", "https://airbyte.io/", text=response_body) @@ -45,13 +41,13 @@ def test_jsonl_decoder(requests_mock, response_body, expected_json): @pytest.fixture(name="large_events_response") def large_event_response_fixture(): data = {"email": "email1@example.com"} - json_string = json.dumps(data) - lines_in_response = 5_000_000 + jsonl_string = f"{json.dumps(data)}\n" + lines_in_response = 2_000_000 # ≈ 58 MB of response dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = f"{dir_path}/test_response.txt" with open(file_path, "w") as file: for _ in range(lines_in_response): - file.write(json_string + "\n") + file.write(jsonl_string) yield (lines_in_response, file_path) os.remove(file_path) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py index 24fb662d726a..92b4ffbb4804 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -56,12 +56,15 @@ def create_response(body: Union[Dict, bytes]): ["data"], decoder_jsonl, b'{"data": [{"id": 1, "text_field": "This is a text\\n. New paragraph start here."}]}\n{"data": [{"id": 2, "text_field": "This is another text\\n. New paragraph start here."}]}', - [{"id": 1, "text_field": "This is a text\n. New paragraph start here."}, {"id": 2, "text_field": "This is another text\n. New paragraph start here."}], + [ + {"id": 1, "text_field": "This is a text\n. New paragraph start here."}, + {"id": 2, "text_field": "This is another text\n. New paragraph start here."}, + ], ), ( [], decoder_iterable, - b'user1@example.com\nuser2@example.com', + b"user1@example.com\nuser2@example.com", [{"record": "user1@example.com"}, {"record": "user2@example.com"}], ), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py index 8132e4b60349..33bd6786c152 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py @@ -333,7 +333,7 @@ def mock_datetime_now(monkeypatch): [ {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-31T23:59:59.999999+0000"}, ], - ) + ), ], ) def test_stream_slices( @@ -580,10 +580,11 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, @pytest.mark.parametrize( - "stream_slice", [ + "stream_slice", + [ pytest.param(None, id="test_none_stream_slice"), pytest.param({}, id="test_none_stream_slice"), - ] + ], ) def test_request_option_with_empty_stream_slice(stream_slice): start_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="starttime") diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py index 96eb3c86e52a..b2c8d5faf46d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py @@ -6,12 +6,12 @@ from unittest.mock import Mock import pytest +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import PerPartitionCursor, PerPartitionKeySerializer, StreamSlice from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter from airbyte_cdk.sources.types import Record from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType PARTITION = { "partition_key string": "partition value", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py index 1b3550a99861..4d2141b42373 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py @@ -37,9 +37,8 @@ def with_substream_partition_router(self, stream_name): "stream": "#/definitions/Rates", "parent_key": "id", "partition_field": "parent_id", - } - ] + ], } return self @@ -100,10 +99,7 @@ def build(self): }, }, }, - "streams": [ - {"$ref": "#/definitions/Rates"}, - {"$ref": "#/definitions/AnotherStream"} - ], + "streams": [{"$ref": "#/definitions/Rates"}, {"$ref": "#/definitions/AnotherStream"}], "spec": { "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", @@ -180,11 +176,9 @@ def test_given_record_for_partition_when_read_then_update_state(): stream_instance = source.streams({})[0] list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) - stream_slice = StreamSlice(partition={"partition_field": "1"}, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}) + stream_slice = StreamSlice(partition={"partition_field": "1"}, cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}) with patch.object( - SimpleRetriever, "_read_pages", - side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]] + SimpleRetriever, "_read_pages", side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]] ): list( stream_instance.read_records( @@ -236,17 +230,41 @@ def test_substream_without_input_state(): # This mocks the resulting records of the Rates stream which acts as the parent stream of the SubstreamPartitionRouter being tested with patch.object( - SimpleRetriever, "_read_pages", side_effect=[[Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], - [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)]] + SimpleRetriever, + "_read_pages", + side_effect=[ + [Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], + [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], + ], ): slices = list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) assert list(slices) == [ - StreamSlice(partition={"parent_id": "1", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}), - StreamSlice(partition={"parent_id": "1", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}), - StreamSlice(partition={"parent_id": "2", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}), - StreamSlice(partition={"parent_id": "2", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}), + StreamSlice( + partition={ + "parent_id": "1", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}, + ), + StreamSlice( + partition={ + "parent_id": "1", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}, + ), + StreamSlice( + partition={ + "parent_id": "2", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}, + ), + StreamSlice( + partition={ + "parent_id": "2", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}, + ), ] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py index bb15e465e8fa..b45973283aad 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py @@ -32,7 +32,7 @@ StreamSlice(cursor_slice={}, partition={}), id="test_empty_substream_resumable_full_refresh_stream_state", ), - ] + ], ) def test_stream_slices(stream_state, cursor, expected_slice): cursor = cursor(parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py b/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py index 7fce15031ee1..97e5efd69f97 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py @@ -23,12 +23,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): input_state = { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00" - }, + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00"}, } migrator = _migrator() @@ -37,14 +33,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): expected_state = { "states": [ - { - "partition": {"parent_id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"parent_id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, + {"partition": {"parent_id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"parent_id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] } @@ -52,115 +42,88 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): @pytest.mark.parametrize( - "input_state", [ - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - }, - ] - }, id="test_should_not_migrate_state_without_a_cursor_component"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"updated_at": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"updated_at": "2022-12-27T08:35:39+00:00"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state_with_wrong_cursor_field"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00", "updated_at": "2021-01-01"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state_with_multiple_cursor_fields"), + "input_state", + [ pytest.param( { "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] - }, id="test_should_not_migrate_state_without_a_partition_component" + }, + id="test_should_not_migrate_a_per_partition_state", ), pytest.param( { "states": [ + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, { - "partition": {"id": "13506132", "another_id": "A"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "13506134"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} + "partition": {"id": "14351124"}, }, ] - }, id="test_should_not_migrate_state_if_multiple_partition_keys" + }, + id="test_should_not_migrate_state_without_a_cursor_component", + ), + pytest.param( + { + "states": [ + {"partition": {"id": "13506132"}, "cursor": {"updated_at": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"updated_at": "2022-12-27T08:35:39+00:00"}}, + ] + }, + id="test_should_not_migrate_a_per_partition_state_with_wrong_cursor_field", ), pytest.param( { "states": [ - { - "partition": {"identifier": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "13506134"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00", "updated_at": "2021-01-01"}}, + ] + }, + id="test_should_not_migrate_a_per_partition_state_with_multiple_cursor_fields", + ), + pytest.param( + { + "states": [ + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, ] - }, id="test_should_not_migrate_state_if_invalid_partition_key" + }, + id="test_should_not_migrate_state_without_a_partition_component", ), pytest.param( { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00", - "another_key": "2022-12-27T08:35:39+00:00" - }, - }, id="test_should_not_migrate_if_the_partitioned_state_has_more_than_one_key" + "states": [ + {"partition": {"id": "13506132", "another_id": "A"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "13506134"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + ] + }, + id="test_should_not_migrate_state_if_multiple_partition_keys", + ), + pytest.param( + { + "states": [ + {"partition": {"identifier": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "13506134"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + ] + }, + id="test_should_not_migrate_state_if_invalid_partition_key", ), - pytest.param({ - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" + pytest.param( + { + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00", "another_key": "2022-12-27T08:35:39+00:00"}, }, - "14351124": { - "another_key": "2022-12-27T08:35:39+00:00" + id="test_should_not_migrate_if_the_partitioned_state_has_more_than_one_key", + ), + pytest.param( + { + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"another_key": "2022-12-27T08:35:39+00:00"}, }, - }, id="test_should_not_migrate_if_the_partitioned_state_key_is_not_the_cursor_field"), - ] + id="test_should_not_migrate_if_the_partitioned_state_key_is_not_the_cursor_field", + ), + ], ) def test_should_not_migrate(input_state): migrator = _migrator() @@ -169,12 +132,8 @@ def test_should_not_migrate(input_state): def test_should_not_migrate_stream_with_multiple_parent_streams(): input_state = { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00" - }, + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00"}, } migrator = _migrator_with_multiple_parent_streams() @@ -191,14 +150,10 @@ def _migrator(): parent_key="{{ parameters['parent_key_id'] }}", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ) - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", @@ -220,26 +175,18 @@ def _migrator_with_multiple_parent_streams(): parent_key="id", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ), ParentStreamConfig( type="ParentStreamConfig", parent_key="id", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ), - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", @@ -256,10 +203,28 @@ def _migrator_with_multiple_parent_streams(): "retriever_type, partition_router_class, is_parent_stream_config, expected_exception, expected_error_message", [ (SimpleRetriever, CustomPartitionRouter, True, None, None), - (None, CustomPartitionRouter, True, ValueError, "LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got "), - (SimpleRetriever, None, False, ValueError, "LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got "), - (SimpleRetriever, CustomPartitionRouter, False, ValueError, "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration."), - ] + ( + None, + CustomPartitionRouter, + True, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got ", + ), + ( + SimpleRetriever, + None, + False, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got ", + ), + ( + SimpleRetriever, + CustomPartitionRouter, + False, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration.", + ), + ], ) def test_create_legacy_to_per_partition_state_migration( retriever_type, @@ -283,13 +248,30 @@ def test_create_legacy_to_per_partition_state_migration( state_migrations_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["state_migrations"][0], {}) if is_parent_stream_config: - parent_stream_config = ParentStreamConfig(type="ParentStreamConfig", parent_key="id", partition_field="parent_id", stream=DeclarativeStream(type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name"))) + parent_stream_config = ParentStreamConfig( + type="ParentStreamConfig", + parent_key="id", + partition_field="parent_id", + stream=DeclarativeStream( + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), + ) partition_router.parent_stream_configs = [parent_stream_config] if expected_exception: with pytest.raises(expected_exception) as excinfo: - factory.create_component(model_type=LegacyToPerPartitionStateMigrationModel, component_definition=state_migrations_manifest, config={}, declarative_stream=stream) + factory.create_component( + model_type=LegacyToPerPartitionStateMigrationModel, + component_definition=state_migrations_manifest, + config={}, + declarative_stream=stream, + ) assert str(excinfo.value) == expected_error_message else: - migration_instance = factory.create_component(model_type=LegacyToPerPartitionStateMigrationModel, component_definition=state_migrations_manifest, config={}, declarative_stream=stream) + migration_instance = factory.create_component( + model_type=LegacyToPerPartitionStateMigrationModel, + component_definition=state_migrations_manifest, + config={}, + declarative_stream=stream, + ) assert migration_instance is not None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index d73527ad6f0f..d574ed8724e8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -9,7 +9,7 @@ import freezegun import pytest from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import Level +from airbyte_cdk.models import FailureType, Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, @@ -82,7 +82,6 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator -from airbyte_protocol.models import FailureType from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent factory = ModelToComponentFactory() @@ -1043,10 +1042,7 @@ def test_create_record_selector(test_name, record_selector, expected_runtime_sel selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {}) selector = factory.create_component( - model_type=RecordSelectorModel, component_definition=selector_manifest, - decoder=None, - transformations=[], - config=input_config + model_type=RecordSelectorModel, component_definition=selector_manifest, decoder=None, transformations=[], config=input_config ) assert isinstance(selector, RecordSelector) @@ -1127,7 +1123,8 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty selector = factory.create_component( model_type=HttpRequesterModel, - component_definition=requester_manifest, config=input_config, + component_definition=requester_manifest, + config=input_config, name=name, decoder=None, ) @@ -1179,8 +1176,7 @@ def test_create_request_with_legacy_session_authenticator(): requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {}) selector = factory.create_component( - model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name, - decoder=None + model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name, decoder=None ) assert isinstance(selector, HttpRequester) @@ -1265,11 +1261,13 @@ def test_given_composite_error_handler_does_not_match_response_then_fallback_on_ resolved_manifest = resolver.preprocess_manifest(parsed_manifest) requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {}) http_requester = factory.create_component( - model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name="any name", decoder=JsonDecoder(parameters={}) - ) - requests_mock.get( - "https://api.sendgrid.com/v3/marketing/lists", status_code=401 + model_type=HttpRequesterModel, + component_definition=requester_manifest, + config=input_config, + name="any name", + decoder=JsonDecoder(parameters={}), ) + requests_mock.get("https://api.sendgrid.com/v3/marketing/lists", status_code=401) with pytest.raises(AirbyteTracedException) as exception: http_requester.send_request() @@ -1453,8 +1451,11 @@ def test_create_default_paginator(): paginator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["paginator"], {}) paginator = factory.create_component( - model_type=DefaultPaginatorModel, component_definition=paginator_manifest, config=input_config, url_base="https://airbyte.io", - decoder=JsonDecoder(parameters={}) + model_type=DefaultPaginatorModel, + component_definition=paginator_manifest, + config=input_config, + url_base="https://airbyte.io", + decoder=JsonDecoder(parameters={}), ) assert isinstance(paginator, DefaultPaginator) @@ -1481,7 +1482,12 @@ def test_create_default_paginator(): "subcomponent_field_with_hint": {"type": "DpathExtractor", "field_path": [], "decoder": {"type": "JsonDecoder"}}, }, "subcomponent_field_with_hint", - DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, decoder=JsonDecoder(parameters={}), parameters={}), + DpathExtractor( + field_path=[], + config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, + decoder=JsonDecoder(parameters={}), + parameters={}, + ), None, id="test_create_custom_component_with_subcomponent_that_must_be_parsed", ), @@ -2118,10 +2124,7 @@ def test_create_page_increment_with_interpolated_page_size(): start_from_page=1, inject_on_first_request=True, ) - config = { - **input_config, - "page_size": 5 - } + config = {**input_config, "page_size": 5} expected_strategy = PageIncrement(page_size=5, start_from_page=1, inject_on_first_request=True, parameters={}, config=config) strategy = factory.create_page_increment(model, config) @@ -2156,7 +2159,7 @@ def test_create_custom_schema_loader(): definition = { "type": "CustomSchemaLoader", - "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader" + "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader", } component = factory.create_component(CustomSchemaLoaderModel, definition, {}) assert isinstance(component, MyCustomSchemaLoader) @@ -2181,12 +2184,9 @@ def test_create_custom_schema_loader(): "algorithm": "HS256", "base64_encode_secret_key": False, "token_duration": 1200, - "jwt_headers": { - "typ": "JWT", - "alg": "HS256" - }, - "jwt_payload": {} - } + "jwt_headers": {"typ": "JWT", "alg": "HS256"}, + "jwt_payload": {}, + }, ), ( { @@ -2228,7 +2228,6 @@ def test_create_custom_schema_loader(): "alg": "RS256", "cty": "JWT", "test": "test custom header", - }, "jwt_payload": { "iss": "test iss", @@ -2236,7 +2235,7 @@ def test_create_custom_schema_loader(): "aud": "test aud", "test": "test custom payload", }, - } + }, ), ( { @@ -2261,12 +2260,11 @@ def test_create_custom_schema_loader(): "typ": "JWT", "alg": "HS256", "custom_header": "custom header value", - }, "jwt_payload": { "custom_payload": "custom payload value", }, - } + }, ), ( { @@ -2280,7 +2278,7 @@ def test_create_custom_schema_loader(): """, { "expect_error": True, - } + }, ), ], ) @@ -2297,9 +2295,7 @@ def test_create_jwt_authenticator(config, manifest, expected): ) return - authenticator = factory.create_component( - model_type=JwtAuthenticatorModel, component_definition=authenticator_manifest, config=config - ) + authenticator = factory.create_component(model_type=JwtAuthenticatorModel, component_definition=authenticator_manifest, config=config) assert isinstance(authenticator, JwtAuthenticator) assert authenticator._secret_key.eval(config) == expected["secret_key"] @@ -2310,9 +2306,11 @@ def test_create_jwt_authenticator(config, manifest, expected): assert authenticator._header_prefix.eval(config) == expected["header_prefix"] assert authenticator._get_jwt_headers() == expected["jwt_headers"] jwt_payload = expected["jwt_payload"] - jwt_payload.update({ - "iat": int(datetime.datetime.now().timestamp()), - "nbf": int(datetime.datetime.now().timestamp()), - "exp": int(datetime.datetime.now().timestamp()) + expected["token_duration"] - }) + jwt_payload.update( + { + "iat": int(datetime.datetime.now().timestamp()), + "nbf": int(datetime.datetime.now().timestamp()), + "exp": int(datetime.datetime.now().timestamp()) + expected["token_duration"], + } + ) assert authenticator._get_jwt_payload() == jwt_payload diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py index 3ec2537e0072..2b9313b3ebd7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py @@ -17,9 +17,11 @@ ( "test_single_stream_slicer", [ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={})], - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( "test_two_stream_slicers", @@ -37,24 +39,24 @@ ], ), ( - "test_singledatetime", - [ - DatetimeBasedCursor( - start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), - end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), - step="P1D", - cursor_field=InterpolatedString.create("", parameters={}), - datetime_format="%Y-%m-%d", - cursor_granularity="P1D", - config={}, - parameters={}, - ), - ], - [ - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), - ], + "test_singledatetime", + [ + DatetimeBasedCursor( + start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), + end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), + step="P1D", + cursor_field=InterpolatedString.create("", parameters={}), + datetime_format="%Y-%m-%d", + cursor_granularity="P1D", + config={}, + parameters={}, + ), + ], + [ + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), + ], ), ( "test_list_and_datetime", @@ -78,9 +80,15 @@ StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"} + ), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"} + ), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"} + ), ], ), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py index 387579783e4d..87aa18f5a0b4 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py @@ -18,23 +18,29 @@ ( ["customer", "store", "subscription"], "owner_resource", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( '["customer", "store", "subscription"]', "owner_resource", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( '["customer", "store", "subscription"]', "{{ parameters['cursor_field'] }}", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ], ids=[ @@ -106,8 +112,8 @@ def test_request_option(request_option, expected_req_params, expected_headers, e [ pytest.param({}, id="test_request_option_is_empty_if_empty_stream_slice"), pytest.param({"not the cursor": "value"}, id="test_request_option_is_empty_if_the_stream_slice_does_not_have_cursor_field"), - pytest.param(None, id="test_request_option_is_empty_if_no_stream_slice") - ] + pytest.param(None, id="test_request_option_is_empty_if_no_stream_slice"), + ], ) def test_request_option_is_empty_if_no_stream_slice(stream_slice): request_option = RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner_resource") diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py index 773ed96571d0..9ced561742f6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py @@ -19,6 +19,7 @@ SyncMode, ) from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource +from orjson import orjson SUBSTREAM_MANIFEST: MutableMapping[str, Any] = { "version": "0.51.42", @@ -349,7 +350,7 @@ def _run_read( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="post_comment_votes", namespace=None), - stream_state=AirbyteStateBlob.parse_obj( + stream_state=AirbyteStateBlob( { "parent_state": { "post_comments": { @@ -424,7 +425,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r output_data = [message.record.data for message in output if message.record] assert output_data == expected_records - final_state = [message.state.stream.stream_state.dict() for message in output if message.state] + final_state = [orjson.loads(orjson.dumps(message.state.stream.stream_state)) for message in output if message.state] assert final_state[-1] == expected_state @@ -467,56 +468,56 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r ), # Fetch the first page of votes for comment 10 of post 1 ( - "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time=2024-01-02T00:00:00Z", - { - "votes": [{"id": 100, "comment_id": 10, "created_at": "2024-01-15T00:00:00Z"}], - "next_page": "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", - }, + "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time=2024-01-02T00:00:00Z", + { + "votes": [{"id": 100, "comment_id": 10, "created_at": "2024-01-15T00:00:00Z"}], + "next_page": "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", + }, ), # Fetch the second page of votes for comment 10 of post 1 ( - "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 101, "comment_id": 10, "created_at": "2024-01-14T00:00:00Z"}]}, + "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 101, "comment_id": 10, "created_at": "2024-01-14T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 11 of post 1 ( - "https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time=2024-01-03T00:00:00Z", - {"votes": [{"id": 102, "comment_id": 11, "created_at": "2024-01-13T00:00:00Z"}]}, + "https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time=2024-01-03T00:00:00Z", + {"votes": [{"id": 102, "comment_id": 11, "created_at": "2024-01-13T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 12 of post 1 ("https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time=2024-01-01T00:00:01Z", {"votes": []}), # Fetch the first page of comments for post 2 ( - "https://api.example.com/community/posts/2/comments?per_page=100", - { - "comments": [{"id": 20, "post_id": 2, "updated_at": "2024-01-22T00:00:00Z"}], - "next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2", - }, + "https://api.example.com/community/posts/2/comments?per_page=100", + { + "comments": [{"id": 20, "post_id": 2, "updated_at": "2024-01-22T00:00:00Z"}], + "next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2", + }, ), # Fetch the second page of comments for post 2 ( - "https://api.example.com/community/posts/2/comments?per_page=100&page=2", - {"comments": [{"id": 21, "post_id": 2, "updated_at": "2024-01-21T00:00:00Z"}]}, + "https://api.example.com/community/posts/2/comments?per_page=100&page=2", + {"comments": [{"id": 21, "post_id": 2, "updated_at": "2024-01-21T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 20 of post 2 ( - "https://api.example.com/community/posts/2/comments/20/votes?per_page=100&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 200, "comment_id": 20, "created_at": "2024-01-12T00:00:00Z"}]}, + "https://api.example.com/community/posts/2/comments/20/votes?per_page=100&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 200, "comment_id": 20, "created_at": "2024-01-12T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 21 of post 2 ( - "https://api.example.com/community/posts/2/comments/21/votes?per_page=100&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 201, "comment_id": 21, "created_at": "2024-01-12T00:00:15Z"}]}, + "https://api.example.com/community/posts/2/comments/21/votes?per_page=100&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 201, "comment_id": 21, "created_at": "2024-01-12T00:00:15Z"}]}, ), # Fetch the first page of comments for post 3 ( - "https://api.example.com/community/posts/3/comments?per_page=100", - {"comments": [{"id": 30, "post_id": 3, "updated_at": "2024-01-09T00:00:00Z"}]}, + "https://api.example.com/community/posts/3/comments?per_page=100", + {"comments": [{"id": 30, "post_id": 3, "updated_at": "2024-01-09T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 30 of post 3 ( - "https://api.example.com/community/posts/3/comments/30/votes?per_page=100", - {"votes": [{"id": 300, "comment_id": 30, "created_at": "2024-01-10T00:00:00Z"}]}, + "https://api.example.com/community/posts/3/comments/30/votes?per_page=100", + {"votes": [{"id": 300, "comment_id": 30, "created_at": "2024-01-10T00:00:00Z"}]}, ), ], # Expected records @@ -534,7 +535,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="post_comment_votes", namespace=None), - stream_state=AirbyteStateBlob.parse_obj( + stream_state=AirbyteStateBlob( { # This should not happen since parent state is disabled, but I've added this to validate that and # incoming parent_state is ignored when the parent stream's incremental_dependency is disabled @@ -590,12 +591,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r ], ) def test_incremental_parent_state_no_incremental_dependency( - test_name, - manifest, - mock_requests, - expected_records, - initial_state, - expected_state + test_name, manifest, mock_requests, expected_records, initial_state, expected_state ): """ This is a pretty complicated test that syncs a low-code connector stream with three levels of substreams @@ -614,8 +610,12 @@ def test_incremental_parent_state_no_incremental_dependency( config = {"start_date": "2024-01-01T00:00:01Z", "credentials": {"email": "email", "api_token": "api_token"}} # Disable incremental_dependency - manifest["definitions"]["post_comments_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0]["incremental_dependency"] = False - manifest["definitions"]["post_comment_votes_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0]["incremental_dependency"] = False + manifest["definitions"]["post_comments_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0][ + "incremental_dependency" + ] = False + manifest["definitions"]["post_comment_votes_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0][ + "incremental_dependency" + ] = False with requests_mock.Mocker() as m: for url, response in mock_requests: @@ -625,5 +625,5 @@ def test_incremental_parent_state_no_incremental_dependency( output_data = [message.record.data for message in output if message.record] assert output_data == expected_records - final_state = [message.state.stream.stream_state.dict() for message in output if message.state] + final_state = [orjson.loads(orjson.dumps(message.state.stream.stream_state)) for message in output if message.state] assert final_state[-1] == expected_state diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index 5201dbc8f241..3a80407cea96 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -41,9 +41,7 @@ def __init__(self, slices, records, name, cursor_field="", cursor=None): self._slices = slices self._records = records self._stream_cursor_field = ( - InterpolatedString.create(cursor_field, parameters={}) - if isinstance(cursor_field, str) - else cursor_field + InterpolatedString.create(cursor_field, parameters={}) if isinstance(cursor_field, str) else cursor_field ) self._name = name self._state = {"states": []} @@ -311,15 +309,17 @@ def test_substream_partition_router(parent_stream_configs, expected_slices): def test_substream_partition_router_invalid_parent_record_type(): partition_router = SubstreamPartitionRouter( - parent_stream_configs=[ParentStreamConfig( - stream=MockStream([{}], [list()], "first_stream"), - parent_key="id", - partition_field="first_stream_id", - parameters={}, - config={}, - )], + parent_stream_configs=[ + ParentStreamConfig( + stream=MockStream([{}], [list()], "first_stream"), + parent_key="id", + partition_field="first_stream_id", + parameters={}, + config={}, + ) + ], parameters={}, - config={} + config={}, ) with pytest.raises(AirbyteTracedException): @@ -664,7 +664,7 @@ def test_substream_checkpoints_after_each_parent_partition(): [ pytest.param(False, id="test_resumable_full_refresh_stream_without_parent_checkpoint"), pytest.param(True, id="test_resumable_full_refresh_stream_with_use_incremental_dependency_for_parent_checkpoint"), - ] + ], ) def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_dependency): mock_slices = [ @@ -687,8 +687,8 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de {"next_page_token": 2}, {"next_page_token": 3}, {"next_page_token": 3}, - {'__ab_full_refresh_sync_complete': True}, - {'__ab_full_refresh_sync_complete': True}, + {"__ab_full_refresh_sync_complete": True}, + {"__ab_full_refresh_sync_complete": True}, ] partition_router = SubstreamPartitionRouter( @@ -737,7 +737,7 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de [ pytest.param(False, id="test_substream_resumable_full_refresh_stream_without_parent_checkpoint"), pytest.param(True, id="test_substream_resumable_full_refresh_stream_with_use_incremental_dependency_for_parent_checkpoint"), - ] + ], ) def test_substream_using_resumable_full_refresh_parent_stream_slices(use_incremental_dependency): mock_parent_slices = [ @@ -760,72 +760,20 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme {"next_page_token": 2}, {"next_page_token": 3}, {"next_page_token": 3}, - {'__ab_full_refresh_sync_complete': True}, - {'__ab_full_refresh_sync_complete': True}, + {"__ab_full_refresh_sync_complete": True}, + {"__ab_full_refresh_sync_complete": True}, ] expected_substream_state = { "states": [ - { - "partition": { - "parent_slice": {}, - "partition_field": "makoto_yuki" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "yukari_takeba" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "mitsuru_kirijo" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "akihiko_sanada" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "junpei_iori" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "fuuka_yamagishi" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"parent_slice": {}, "partition_field": "makoto_yuki"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "yukari_takeba"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "mitsuru_kirijo"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "akihiko_sanada"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "junpei_iori"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "fuuka_yamagishi"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ], - "parent_state": { - "persona_3_characters": { - "__ab_full_refresh_sync_complete": True - } - } + "parent_state": {"persona_3_characters": {"__ab_full_refresh_sync_complete": True}}, } partition_router = SubstreamPartitionRouter( @@ -874,7 +822,9 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme assert actual_slice == expected_parent_slices[expected_counter] # check for parent state if use_incremental_dependency: - assert substream_cursor_slicer._partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] + assert ( + substream_cursor_slicer._partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] + ) expected_counter += 1 # validate final state for closed substream slices diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py index b57fd714b735..59dbb6b419a7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py @@ -6,10 +6,10 @@ import pytest from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_time_from_header_backoff_strategy import ( WaitTimeFromHeaderBackoffStrategy, ) -from airbyte_protocol.models import FailureType from requests import Response SOME_BACKOFF_TIME = 60 @@ -44,7 +44,9 @@ def test_given_retry_after_smaller_than_max_time_then_raise_transient_error(): response_mock = MagicMock(spec=Response) retry_after = _A_MAX_TIME - 1 response_mock.headers = {_A_RETRY_HEADER: str(retry_after)} - backoff_strategy = WaitTimeFromHeaderBackoffStrategy(header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={}) + backoff_strategy = WaitTimeFromHeaderBackoffStrategy( + header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={} + ) assert backoff_strategy.backoff_time(response_mock, 1) == retry_after @@ -52,7 +54,9 @@ def test_given_retry_after_smaller_than_max_time_then_raise_transient_error(): def test_given_retry_after_greater_than_max_time_then_raise_transient_error(): response_mock = MagicMock(spec=Response) response_mock.headers = {_A_RETRY_HEADER: str(_A_MAX_TIME + 1)} - backoff_strategy = WaitTimeFromHeaderBackoffStrategy(header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={}) + backoff_strategy = WaitTimeFromHeaderBackoffStrategy( + header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={} + ) with pytest.raises(AirbyteTracedException) as exception: backoff_strategy.backoff_time(response_mock, 1) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 5f34bb28c969..574f3eec0e75 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -6,11 +6,11 @@ import pytest import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction -from airbyte_protocol.models import FailureType SOME_BACKOFF_TIME = 60 @@ -34,7 +34,7 @@ response_action=ResponseAction.SUCCESS, failure_type=None, error_message=None, - ) + ), ), ( "test_chain_retrier_ignore_fail", @@ -83,7 +83,7 @@ ErrorResolution( response_action=ResponseAction.IGNORE, ), - ) + ), ], ) def test_composite_error_handler(test_name, first_handler_behavior, second_handler_behavior, expected_behavior): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py index f80aef233ebe..6fc99159afed 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py @@ -18,42 +18,42 @@ @pytest.mark.parametrize( - "test_name, http_status_code, expected_error_resolution", - [ - ( - "_with_http_response_status_200", - 200, - ErrorResolution( - response_action=ResponseAction.SUCCESS, - failure_type=None, - error_message=None, - ), - ), - ( - "_with_http_response_status_400", - 400, - DEFAULT_ERROR_MAPPING[400], - ), - ( - "_with_http_response_status_404", - 404, - DEFAULT_ERROR_MAPPING[404], + "test_name, http_status_code, expected_error_resolution", + [ + ( + "_with_http_response_status_200", + 200, + ErrorResolution( + response_action=ResponseAction.SUCCESS, + failure_type=None, + error_message=None, ), - ( - "_with_http_response_status_408", - 408, - DEFAULT_ERROR_MAPPING[408], + ), + ( + "_with_http_response_status_400", + 400, + DEFAULT_ERROR_MAPPING[400], + ), + ( + "_with_http_response_status_404", + 404, + DEFAULT_ERROR_MAPPING[404], + ), + ( + "_with_http_response_status_408", + 408, + DEFAULT_ERROR_MAPPING[408], + ), + ( + "_with_unmapped_http_status_418", + 418, + ErrorResolution( + response_action=ResponseAction.RETRY, + failure_type=FailureType.system_error, + error_message="Unexpected response with HTTP status 418", ), - ( - "_with_unmapped_http_status_418", - 418, - ErrorResolution( - response_action=ResponseAction.RETRY, - failure_type=FailureType.system_error, - error_message="Unexpected response with HTTP status 418", - ), - ) - ], + ), + ], ) def test_default_error_handler_with_default_response_filter(test_name, http_status_code: int, expected_error_resolution: ErrorResolution): response_mock = create_response(http_status_code) @@ -65,76 +65,78 @@ def test_default_error_handler_with_default_response_filter(test_name, http_stat @pytest.mark.parametrize( - "test_name, http_status_code, test_response_filter, response_action, failure_type, error_message", - [ - ( - "_with_http_response_status_400_fail_with_default_failure_type", - 400, - HttpResponseFilter( - http_codes=[400], - action=ResponseAction.RETRY, - config={}, - parameters={}, - ), - ResponseAction.RETRY, - FailureType.system_error, - "Bad request. Please check your request parameters.", + "test_name, http_status_code, test_response_filter, response_action, failure_type, error_message", + [ + ( + "_with_http_response_status_400_fail_with_default_failure_type", + 400, + HttpResponseFilter( + http_codes=[400], + action=ResponseAction.RETRY, + config={}, + parameters={}, ), - ( - "_with_http_response_status_402_fail_with_default_failure_type", - 402, - HttpResponseFilter( - http_codes=[402], - action=ResponseAction.FAIL, - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.RETRY, + FailureType.system_error, + "Bad request. Please check your request parameters.", + ), + ( + "_with_http_response_status_402_fail_with_default_failure_type", + 402, + HttpResponseFilter( + http_codes=[402], + action=ResponseAction.FAIL, + config={}, + parameters={}, ), - ( - "_with_http_response_status_403_fail_with_default_failure_type", - 403, - HttpResponseFilter( - http_codes=[403], - action="FAIL", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.config_error, - "Forbidden. You don't have permission to access this resource.", + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ( + "_with_http_response_status_403_fail_with_default_failure_type", + 403, + HttpResponseFilter( + http_codes=[403], + action="FAIL", + config={}, + parameters={}, ), - ( - "_with_http_response_status_200_fail_with_contained_error_message", - 418, - HttpResponseFilter( - action=ResponseAction.FAIL, - error_message_contains="test", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.FAIL, + FailureType.config_error, + "Forbidden. You don't have permission to access this resource.", + ), + ( + "_with_http_response_status_200_fail_with_contained_error_message", + 418, + HttpResponseFilter( + action=ResponseAction.FAIL, + error_message_contains="test", + config={}, + parameters={}, ), - ( - "_fail_with_predicate", - 418, - HttpResponseFilter( - action=ResponseAction.FAIL, - predicate="{{ 'error' in response }}", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ( + "_fail_with_predicate", + 418, + HttpResponseFilter( + action=ResponseAction.FAIL, + predicate="{{ 'error' in response }}", + config={}, + parameters={}, ), - ], + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ], ) -def test_default_error_handler_with_custom_response_filter(test_name, http_status_code, test_response_filter, response_action, failure_type, error_message): +def test_default_error_handler_with_custom_response_filter( + test_name, http_status_code, test_response_filter, response_action, failure_type, error_message +): response_mock = create_response(http_status_code) if http_status_code == 418: response_mock.json.return_value = {"error": "test"} @@ -148,11 +150,11 @@ def test_default_error_handler_with_custom_response_filter(test_name, http_statu @pytest.mark.parametrize( - "http_status_code, expected_response_action", - [ - (400, ResponseAction.RETRY), - (402, ResponseAction.FAIL), - ], + "http_status_code, expected_response_action", + [ + (400, ResponseAction.RETRY), + (402, ResponseAction.FAIL), + ], ) def test_default_error_handler_with_multiple_response_filters(http_status_code, expected_response_action): response_filter_one = HttpResponseFilter( @@ -175,15 +177,17 @@ def test_default_error_handler_with_multiple_response_filters(http_status_code, @pytest.mark.parametrize( - "first_response_filter_action, second_response_filter_action, expected_response_action", - [ - (ResponseAction.RETRY, ResponseAction.FAIL, ResponseAction.RETRY), - (ResponseAction.FAIL, ResponseAction.RETRY, ResponseAction.FAIL), - (ResponseAction.IGNORE, ResponseAction.IGNORE, ResponseAction.IGNORE), - (ResponseAction.SUCCESS, ResponseAction.IGNORE, ResponseAction.SUCCESS), - ] + "first_response_filter_action, second_response_filter_action, expected_response_action", + [ + (ResponseAction.RETRY, ResponseAction.FAIL, ResponseAction.RETRY), + (ResponseAction.FAIL, ResponseAction.RETRY, ResponseAction.FAIL), + (ResponseAction.IGNORE, ResponseAction.IGNORE, ResponseAction.IGNORE), + (ResponseAction.SUCCESS, ResponseAction.IGNORE, ResponseAction.SUCCESS), + ], ) -def test_default_error_handler_with_conflicting_response_filters(first_response_filter_action, second_response_filter_action, expected_response_action): +def test_default_error_handler_with_conflicting_response_filters( + first_response_filter_action, second_response_filter_action, expected_response_action +): response_filter_one = HttpResponseFilter( http_codes=[400], action=first_response_filter_action, @@ -205,19 +209,29 @@ def test_default_error_handler_with_conflicting_response_filters(first_response_ def test_default_error_handler_with_constant_backoff_strategy(): response_mock = create_response(429) - error_handler = DefaultErrorHandler(config={}, parameters={}, backoff_strategies=[ConstantBackoffStrategy(SOME_BACKOFF_TIME, config={}, parameters={})]) + error_handler = DefaultErrorHandler( + config={}, parameters={}, backoff_strategies=[ConstantBackoffStrategy(SOME_BACKOFF_TIME, config={}, parameters={})] + ) assert error_handler.backoff_time(response_or_exception=response_mock, attempt_count=0) == SOME_BACKOFF_TIME @pytest.mark.parametrize( "attempt_count", [ - 0, 1, 2, 3, 4, 5, 6, + 0, + 1, + 2, + 3, + 4, + 5, + 6, ], ) def test_default_error_handler_with_exponential_backoff_strategy(attempt_count): response_mock = create_response(429) - error_handler = DefaultErrorHandler(config={}, parameters={}, backoff_strategies=[ExponentialBackoffStrategy(factor=1, config={}, parameters={})]) + error_handler = DefaultErrorHandler( + config={}, parameters={}, backoff_strategies=[ExponentialBackoffStrategy(factor=1, config={}, parameters={})] + ) assert error_handler.backoff_time(response_or_exception=response_mock, attempt_count=attempt_count) == (1 * 2**attempt_count) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py index 6da87a183ff2..b3e4c517da26 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py @@ -5,31 +5,19 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import DefaultHttpResponseFilter from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction -from airbyte_protocol.models import FailureType from requests import RequestException, Response @pytest.mark.parametrize( "http_code, expected_error_resolution", [ - pytest.param( - 403, - DEFAULT_ERROR_MAPPING[403], - id="403 mapping" - ), - pytest.param( - 404, - DEFAULT_ERROR_MAPPING[404], - id="404 mapping" - ), - pytest.param( - 408, - DEFAULT_ERROR_MAPPING[408], - id="408 mapping" - ), + pytest.param(403, DEFAULT_ERROR_MAPPING[403], id="403 mapping"), + pytest.param(404, DEFAULT_ERROR_MAPPING[404], id="404 mapping"), + pytest.param(408, DEFAULT_ERROR_MAPPING[408], id="408 mapping"), ], ) def test_matches_mapped_http_status_code(http_code, expected_error_resolution): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index 5fd5990e898b..9c6817c268c4 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -22,7 +22,9 @@ "", "custom error message", {"status_code": 503}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.transient_error, error_message="custom error message"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.transient_error, error_message="custom error message" + ), id="test_http_code_matches", ), pytest.param( @@ -33,7 +35,11 @@ "", "", {"status_code": 403}, - ErrorResolution(response_action=ResponseAction.IGNORE, failure_type=FailureType.config_error, error_message="Forbidden. You don't have permission to access this resource."), + ErrorResolution( + response_action=ResponseAction.IGNORE, + failure_type=FailureType.config_error, + error_message="Forbidden. You don't have permission to access this resource.", + ), id="test_http_code_matches_ignore_action", ), pytest.param( @@ -44,7 +50,9 @@ "", "", {"status_code": 429}, - ErrorResolution(response_action=ResponseAction.RETRY, failure_type=FailureType.transient_error, error_message="Too many requests."), + ErrorResolution( + response_action=ResponseAction.RETRY, failure_type=FailureType.transient_error, error_message="Too many requests." + ), id="test_http_code_matches_retry_action", ), pytest.param( @@ -55,7 +63,9 @@ "", "error message was: {{ response.failure }}", {"status_code": 404, "json": {"the_body": "do_i_match", "failure": "i failed you"}}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error message was: i failed you"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error message was: i failed you" + ), id="test_predicate_matches_json", ), pytest.param( @@ -66,7 +76,9 @@ "", "error from header: {{ headers.warning }}", {"status_code": 404, "headers": {"the_key": "header_match", "warning": "this failed"}}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error from header: this failed"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error from header: this failed" + ), id="test_predicate_matches_headers", ), pytest.param( @@ -80,7 +92,7 @@ ErrorResolution( response_action=ResponseAction.FAIL, failure_type=FailureType.config_error, - error_message="Forbidden. You don't have permission to access this resource." + error_message="Forbidden. You don't have permission to access this resource.", ), id="test_predicate_matches_headers", ), @@ -147,12 +159,16 @@ "", "rate limits", {"status_code": 500}, - ErrorResolution(response_action=ResponseAction.RATE_LIMITED, failure_type=FailureType.transient_error, error_message="rate limits"), + ErrorResolution( + response_action=ResponseAction.RATE_LIMITED, failure_type=FailureType.transient_error, error_message="rate limits" + ), id="test_http_code_matches_response_action_rate_limited", ), ], ) -def test_matches(requests_mock, action, failure_type, http_codes, predicate, error_contains, error_message, response, expected_error_resolution): +def test_matches( + requests_mock, action, failure_type, http_codes, predicate, error_contains, error_message, response, expected_error_resolution +): requests_mock.register_uri( "GET", "https://airbyte.io/", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 6f010323c8f0..31d9ae5e05f5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -50,7 +50,7 @@ "test_static_token_with_string_stop_condition", "test_token_from_header", "test_token_from_response_header_links", - ] + ], ) def test_cursor_pagination_strategy(template_string, stop_condition, expected_token, page_size): decoder = JsonDecoder(parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 109f153cbcc8..54fcb2883ab2 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -247,7 +247,7 @@ def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): ids=[ "test_reset_inject_on_first_request", "test_reset_no_inject_on_first_request", - ] + ], ) def test_reset(inject_on_first_request): page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 1ca14cc60481..da2bf6d9450e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -40,13 +40,7 @@ def test_page_increment_paginator_strategy(page_size, start_from, last_page_size assert start_from == paginator_strategy._page -@pytest.mark.parametrize( - "page_size", - [ - pytest.param("{{ config['value'] }}"), - pytest.param("not-an-integer") - ] -) +@pytest.mark.parametrize("page_size", [pytest.param("{{ config['value'] }}"), pytest.param("not-an-integer")]) def test_page_increment_paginator_strategy_malformed_page_size(page_size): with pytest.raises(Exception, match=".* is of type . Expected "): PageIncrement(page_size=page_size, parameters={}, start_from_page=0, config={"value": "not-an-integer"}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index 77e94778ea2f..404bf9f50e15 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -415,7 +415,7 @@ def test_send_request_params(provider_params, param_params, authenticator_params {"k": [1, 2]}, "%5B%22a%22%2C+%22b%22%5D=1&%5B%22a%22%2C+%22b%22%5D=2", id="test-key-with-list-to-be-interpolated", - ) + ), ], ) def test_request_param_interpolation(request_parameters, config, expected_query_params): @@ -464,8 +464,7 @@ def test_request_param_interpolation_with_incorrect_values(request_parameters, c requester.send_request() assert ( - error.value.args[0] - == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an object." + error.value.args[0] == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an object." ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 9c36c65b8553..fd3db0452f04 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -147,7 +147,7 @@ def test_simple_retriever_with_request_response_logs(mock_http_stream): [ pytest.param(None, None, 1, id="test_initial_sync_no_state"), pytest.param({"next_page_token": 10}, 10, 11, id="test_reset_with_next_page_token"), - ] + ], ) def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_state, expected_reset_value, expected_next_page): expected_records = [ @@ -184,7 +184,7 @@ def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_s expected_records[5], expected_records[6], expected_records[7], - ] + ], ] page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) @@ -230,11 +230,13 @@ def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_s {"next_page_token": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens"}, "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens", "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens", - id="test_reset_with_next_page_token" + id="test_reset_with_next_page_token", ), - ] + ], ) -def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination(initial_state, expected_reset_value, expected_next_page, requests_mock): +def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination( + initial_state, expected_reset_value, expected_next_page, requests_mock +): expected_records = [ Record(data={"name": "ed_baldwin"}, associated_slice=None), Record(data={"name": "danielle_poole"}, associated_slice=None), @@ -288,7 +290,7 @@ def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination( stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={}) response_body = { "data": [r.data for r in expected_records[:5]], - "next_page": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens" + "next_page": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens", } requests_mock.get("https://for-all-mankind.nasa.com/api/v1/astronauts", json=response_body) requests_mock.get("https://for-all-mankind.nasa.com/astronauts?next_page=tracy_stevens", json=response_body) @@ -334,7 +336,10 @@ def test_simple_retriever_resumable_full_refresh_cursor_reset_skip_completed_str ] record_selector = MagicMock() - record_selector.select_records.return_value = [expected_records[0],expected_records[1],] + record_selector.select_records.return_value = [ + expected_records[0], + expected_records[1], + ] page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) paginator = DefaultPaginator(config={}, pagination_strategy=page_increment_strategy, url_base="https://airbyte.io", parameters={}) @@ -463,13 +468,39 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): @pytest.mark.parametrize( "test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping", [ - ("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}), - ("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}), - ("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}), - ("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}), + ( + "test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", + {"key_from_pagination": "1000"}, + True, + None, + {"key_from_pagination": "1000"}, + ), + ( + "test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", + {"key_from_pagination": "1000"}, + False, + None, + {"key_from_pagination": "1000", "key_from_slicer": "value"}, + ), + ( + "test_ignore_stream_slicer_params_on_paginated_request", + {"key_from_pagination": "1000"}, + True, + {"page": 2}, + {"key_from_pagination": "1000"}, + ), + ( + "test_do_not_ignore_stream_slicer_params_on_paginated_request", + {"key_from_pagination": "1000"}, + False, + {"page": 2}, + {"key_from_pagination": "1000", "key_from_slicer": "value"}, + ), ], ) -def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping): +def test_ignore_stream_slicer_parameters_on_paginated_requests( + test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping +): # This test is separate from the other request options because request headers must be strings paginator = MagicMock() paginator.get_request_headers.return_value = paginator_mapping diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py index 46b892256a25..1e1ef498082f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py @@ -3,32 +3,34 @@ # import pytest -from airbyte_cdk.models.airbyte_protocol import AdvancedAuth, ConnectorSpecification +from airbyte_cdk.models import AdvancedAuth, AuthFlowType, ConnectorSpecification from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow from airbyte_cdk.sources.declarative.spec.spec import Spec @pytest.mark.parametrize( - "test_name, spec, expected_connection_specification", + "spec, expected_connection_specification", [ ( - "test_only_connection_specification", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}), ), ( - "test_with_doc_url", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, documentation_url="https://airbyte.io"), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}, documentationUrl="https://airbyte.io"), ), ( - "test_auth_flow", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, advanced_auth=AuthFlow(auth_flow_type="oauth2.0")), ConnectorSpecification( - connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type="oauth2.0") + connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type=AuthFlowType.oauth2_0) ), ), ], + ids=[ + "test_only_connection_specification", + "test_with_doc_url", + "test_auth_flow", + ], ) -def test_spec(test_name, spec, expected_connection_specification): +def test_spec(spec, expected_connection_specification): assert spec.generate_spec() == expected_connection_specification diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py index bd8281b80b46..8906b625fb8f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py @@ -189,7 +189,7 @@ def test_no_state_migration_is_applied_if_the_state_should_not_be_migrated(): [ pytest.param(True, True, id="test_retriever_has_cursor"), pytest.param(False, False, id="test_retriever_has_cursor"), - ] + ], ) def test_is_resumable(use_cursor, expected_supports_checkpointing): schema_loader = _schema_loader() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index d7017eb52dd5..2d350fa12b4b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -29,7 +29,6 @@ from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from jsonschema.exceptions import ValidationError -from pydantic import AnyUrl logger = logging.getLogger("airbyte") @@ -210,7 +209,7 @@ def test_manifest_with_spec(self): source = ManifestDeclarativeSource(source_config=manifest) connector_specification = source.spec(logger) assert connector_specification is not None - assert connector_specification.documentationUrl == AnyUrl("https://airbyte.com/#yaml-from-manifest") + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-manifest" assert connector_specification.connectionSpecification["title"] == "Test Spec" assert connector_specification.connectionSpecification["required"][0] == "api_key" assert connector_specification.connectionSpecification["additionalProperties"] is False @@ -277,7 +276,7 @@ def test_manifest_with_external_spec(self, use_external_yaml_spec): connector_specification = source.spec(logger) - assert connector_specification.documentationUrl == AnyUrl("https://airbyte.com/#yaml-from-external") + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-external" assert connector_specification.connectionSpecification == EXTERNAL_CONNECTION_SPECIFICATION def test_source_is_not_created_if_toplevel_fields_are_unknown(self): @@ -1045,8 +1044,12 @@ def _create_page(response_body): ), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"ABC": 2, "partition": 1}], [ - call({'states': []}, {"partition": "0"}, None), - call({'states': [{'partition': {'partition': '0'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}]}, {"partition": "1"}, None), + call({"states": []}, {"partition": "0"}, None), + call( + {"states": [{"partition": {"partition": "0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}]}, + {"partition": "1"}, + None, + ), ], ), ( @@ -1119,9 +1122,13 @@ def _create_page(response_body): ), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"USD": 3, "partition": 0}, {"ABC": 2, "partition": 1}], [ - call({'states': []}, {"partition": "0"}, None), - call({'states': []}, {"partition": "0"}, {"next_page_token": "next"}), - call({'states': [{'partition': {'partition': '0'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}]}, {'partition': '1'}, None), + call({"states": []}, {"partition": "0"}, None), + call({"states": []}, {"partition": "0"}, {"next_page_token": "next"}), + call( + {"states": [{"partition": {"partition": "0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}]}, + {"partition": "1"}, + None, + ), ], ), ], @@ -1269,14 +1276,14 @@ def _run_read(manifest: Mapping[str, Any], stream_name: str) -> List[AirbyteMess def test_declarative_component_schema_valid_ref_links(): def load_yaml(file_path) -> Mapping[str, Any]: - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return yaml.safe_load(file) - def extract_refs(data, base_path='#') -> List[str]: + def extract_refs(data, base_path="#") -> List[str]: refs = [] if isinstance(data, dict): for key, value in data.items(): - if key == '$ref' and isinstance(value, str) and value.startswith('#'): + if key == "$ref" and isinstance(value, str) and value.startswith("#"): ref_path = value refs.append(ref_path) else: @@ -1287,11 +1294,11 @@ def extract_refs(data, base_path='#') -> List[str]: return refs def resolve_pointer(data: Mapping[str, Any], pointer: str) -> bool: - parts = pointer.split('/')[1:] # Skip the first empty part due to leading '#/' + parts = pointer.split("/")[1:] # Skip the first empty part due to leading '#/' current = data try: for part in parts: - part = part.replace('~1', '/').replace('~0', '~') # Unescape JSON Pointer + part = part.replace("~1", "/").replace("~0", "~") # Unescape JSON Pointer current = current[part] return True except (KeyError, TypeError): @@ -1300,8 +1307,10 @@ def resolve_pointer(data: Mapping[str, Any], pointer: str) -> bool: def validate_refs(yaml_file: str) -> List[str]: data = load_yaml(yaml_file) refs = extract_refs(data) - invalid_refs = [ref for ref in refs if not resolve_pointer(data, ref.replace('#', ''))] + invalid_refs = [ref for ref in refs if not resolve_pointer(data, ref.replace("#", ""))] return invalid_refs - yaml_file_path = Path(__file__).resolve().parent.parent.parent.parent / 'airbyte_cdk/sources/declarative/declarative_component_schema.yaml' + yaml_file_path = ( + Path(__file__).resolve().parent.parent.parent.parent / "airbyte_cdk/sources/declarative/declarative_component_schema.yaml" + ) assert not validate_refs(yaml_file_path) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py index dd3f8e5b4ab2..b6eb42f940b6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py @@ -7,13 +7,25 @@ @pytest.mark.parametrize( "stream_slice, expected_partition", [ - pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_partition_with_empty_partition"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={}), {}, id="test_partition_nested_empty"), - pytest.param(StreamSlice(partition={"key": "value"}, cursor_slice={}), {"key": "value"}, id="test_partition_with_mapping_partition"), - pytest.param(StreamSlice(partition={},cursor_slice={"cursor": "value"}), {}, id="test_partition_with_only_cursor"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={"cursor": "value"}), {}, id="test_partition_nested_empty_and_cursor_value_mapping"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={"cursor": "value"}), cursor_slice={}), {}, id="test_partition_nested_empty_and_cursor_value"), - ] + pytest.param(StreamSlice(partition={}, cursor_slice={}), {}, id="test_partition_with_empty_partition"), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={}), {}, id="test_partition_nested_empty" + ), + pytest.param( + StreamSlice(partition={"key": "value"}, cursor_slice={}), {"key": "value"}, id="test_partition_with_mapping_partition" + ), + pytest.param(StreamSlice(partition={}, cursor_slice={"cursor": "value"}), {}, id="test_partition_with_only_cursor"), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={"cursor": "value"}), + {}, + id="test_partition_nested_empty_and_cursor_value_mapping", + ), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={"cursor": "value"}), cursor_slice={}), + {}, + id="test_partition_nested_empty_and_cursor_value", + ), + ], ) def test_partition(stream_slice, expected_partition): partition = stream_slice.partition @@ -24,14 +36,25 @@ def test_partition(stream_slice, expected_partition): @pytest.mark.parametrize( "stream_slice, expected_cursor_slice", [ - pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_cursor_slice_with_empty_cursor"), - pytest.param(StreamSlice(partition={}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty"), - - pytest.param(StreamSlice(partition={}, cursor_slice={"key": "value"}), {"key": "value"}, id="test_cursor_slice_with_mapping_cursor_slice"), + pytest.param(StreamSlice(partition={}, cursor_slice={}), {}, id="test_cursor_slice_with_empty_cursor"), + pytest.param( + StreamSlice(partition={}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty" + ), + pytest.param( + StreamSlice(partition={}, cursor_slice={"key": "value"}), {"key": "value"}, id="test_cursor_slice_with_mapping_cursor_slice" + ), pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice={}), {}, id="test_cursor_slice_with_only_partition"), - pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty_and_partition_mapping"), - pytest.param(StreamSlice(partition=StreamSlice(partition={"partition": "value"}, cursor_slice={}), cursor_slice={}), {}, id="test_cursor_slice_nested_empty_and_partition"), - ] + pytest.param( + StreamSlice(partition={"partition": "value"}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), + {}, + id="test_cursor_slice_nested_empty_and_partition_mapping", + ), + pytest.param( + StreamSlice(partition=StreamSlice(partition={"partition": "value"}, cursor_slice={}), cursor_slice={}), + {}, + id="test_cursor_slice_nested_empty_and_partition", + ), + ], ) def test_cursor_slice(stream_slice, expected_cursor_slice): cursor_slice = stream_slice.cursor_slice diff --git a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py index d2bad84128e2..7560dc403ecd 100644 --- a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py @@ -6,9 +6,7 @@ from typing import Any, Mapping, Optional from unittest.mock import MagicMock -from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteCatalog, AirbyteLogMessage, AirbyteMessage, @@ -23,6 +21,8 @@ SyncMode, Type, ) +from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration +from airbyte_cdk.utils import AirbyteTracedException class TestIntegration(BaseEmbeddedIntegration): diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py index 5ce69276d974..c233bd7ac9e9 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py @@ -30,5 +30,5 @@ def test_given_from_csv_then_csv_has_header_row(self) -> None: class CsvDelimiterTest(unittest.TestCase): def test_tab_delimter(self): - assert CsvFormat(delimiter=r"\t").delimiter == '\t' + assert CsvFormat(delimiter=r"\t").delimiter == "\t" assert len(CsvFormat(delimiter=r"\t").delimiter) == 1 diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py index 3882d823e196..a45d424b7a2b 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py @@ -222,7 +222,7 @@ def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_js pytest.param(_default_avro_format, "float", 123.456, 123.456, id="test_float"), pytest.param(_default_avro_format, "double", 123.456, 123.456, id="test_double_default_config"), pytest.param(_double_as_string_avro_format, "double", 123.456, "123.456", id="test_double_as_string"), - pytest.param(_default_avro_format, "bytes", b"hello world", b"hello world", id="test_bytes"), + pytest.param(_default_avro_format, "bytes", b"hello world", "hello world", id="test_bytes"), pytest.param(_default_avro_format, "string", "hello world", "hello world", id="test_string"), pytest.param(_default_avro_format, {"logicalType": "decimal"}, 3.1415, "3.1415", id="test_decimal"), pytest.param(_default_avro_format, {"logicalType": "uuid"}, _uuid_value, str(_uuid_value), id="test_uuid"), diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py index dbee93fd57a6..bd9d8338f094 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py @@ -47,12 +47,14 @@ def setup_parser(remote_file): parser = ExcelParser() # Sample data for the mock Excel file - data = pd.DataFrame({ - "column1": [1, 2, 3], - "column2": ["a", "b", "c"], - "column3": [True, False, True], - "column4": pd.to_datetime(["2021-01-01", "2022-01-01", "2023-01-01"]), - }) + data = pd.DataFrame( + { + "column1": [1, 2, 3], + "column2": ["a", "b", "c"], + "column3": [True, False, True], + "column4": pd.to_datetime(["2021-01-01", "2022-01-01", "2023-01-01"]), + } + ) # Convert the DataFrame to an Excel byte stream excel_bytes = BytesIO() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py index 1fa2dcbf66fc..c4768facc7dd 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py @@ -238,7 +238,8 @@ def test_value_dictionary() -> None: pytest.param(pa.decimal256(2), _decimal_as_float_parquet_format, id="test_decimal256_as_float"), pytest.param(pa.map_(pa.int32(), pa.int32()), _default_parquet_format, id="test_map"), pytest.param(pa.null(), _default_parquet_format, id="test_null"), - ]) + ], +) def test_null_value_does_not_throw(parquet_type, parquet_format) -> None: pyarrow_value = pa.scalar(None, type=parquet_type) assert ParquetParser._to_output_value(pyarrow_value, parquet_format) is None diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py index a6de1b290c56..0a2681911211 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py @@ -16,7 +16,7 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy @@ -53,7 +53,7 @@ def __init__( self.files = files self.file_type = file_type self.catalog = catalog - self.configured_catalog = ConfiguredAirbyteCatalog(streams=self.catalog["streams"]) if self.catalog else None + self.configured_catalog = ConfiguredAirbyteCatalogSerializer.load(self.catalog) if self.catalog else None self.config = config self.state = state @@ -224,8 +224,8 @@ def _make_file_contents(self, file_name: str) -> bytes: df = pd.DataFrame(contents) with io.BytesIO() as fp: - writer = pd.ExcelWriter(fp, engine='xlsxwriter') - df.to_excel(writer, index=False, sheet_name='Sheet1') + writer = pd.ExcelWriter(fp, engine="xlsxwriter") + df.to_excel(writer, index=False, sheet_name="Sheet1") writer._save() fp.seek(0) return fp.read() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py index 0b662519f276..e5a7ee419452 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py @@ -2227,12 +2227,12 @@ .set_expected_records( [ { - "history": { - "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-05T03:54:07.000000Z", - "d.csv": "2023-06-05T03:54:07.000000Z", - }, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", } ] ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index 723550cc36d7..dc0a97bc1cf2 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -2,12 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import SyncMode from unit_tests.sources.file_based.helpers import EmptySchemaParser, LowInferenceLimitDiscoveryPolicy from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder @@ -417,8 +416,8 @@ "properties": { "filetype": {"title": "Filetype", "default": "excel", "const": "excel", "type": "string"} }, - "required": ["filetype"] - } + "required": ["filetype"], + }, ], }, "schemaless": { @@ -432,7 +431,7 @@ "description": "The number of resent files which will be used to discover the schema for this stream.", "exclusiveMinimum": 0, "type": "integer", - } + }, }, "required": ["name", "format"], }, @@ -440,6 +439,8 @@ }, "required": ["streams"], }, + "supportsDBT": False, + "supportsNormalization": False, } ) .set_expected_catalog( @@ -505,7 +506,7 @@ "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", - } + }, ] } ) @@ -568,50 +569,52 @@ "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], "is_resumable": True, - } + }, ] } ) - .set_expected_records([ - { - "data": { - "col1": "val11a", - "col2": "val12a", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, - "stream": "stream1", - }, - { - "data": { - "col1": "val21a", - "col2": "val22a", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, - "stream": "stream1", - }, - { - "data": { - "col1": "val11b", - "col2": "val12b", - "col3": "val13b", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv", + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", }, - "stream": "stream2", - }, - { - "data": { - "col1": "val21b", - "col2": "val22b", - "col3": "val23b", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv", + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", }, - "stream": "stream2", - }, - ]) + ] + ) .set_expected_analytics( [ AirbyteAnalyticsTraceMessage(type="file-cdk-csv-stream-count", value="2"), @@ -2094,7 +2097,6 @@ { "data": { "col1": "2", - "col2": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", }, @@ -2305,7 +2307,6 @@ { "data": { "col1": "2", - "col2": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", }, @@ -3174,7 +3175,6 @@ [ { "data": { - "col1": None, "col2": "na", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", @@ -3316,11 +3316,7 @@ "start_date": "2023-06-10T03:54:07.000000Z", } ) - .set_source_builder( - FileBasedSourceBuilder() - .set_files({}) - .set_file_type("csv") - ) + .set_source_builder(FileBasedSourceBuilder().set_files({}).set_file_type("csv")) .set_expected_check_status("FAILED") .set_expected_catalog( { diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py index f92c8420099d..6653296535d5 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py @@ -22,7 +22,11 @@ "a.xlsx": { "contents": [ {"col_double": 20.02, "col_string": "Robbers", "col_album": "The 1975"}, - {"col_double": 20.23, "col_string": "Somebody Else", "col_album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It"}, + { + "col_double": 20.23, + "col_string": "Somebody Else", + "col_album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It", + }, ], "last_modified": "2023-06-05T03:54:07.000Z", }, @@ -60,14 +64,22 @@ {"col_title": "White Lies", "col_album": "IN_RETURN", "col_year": 2014, "col_vocals": True}, {"col_title": "Wide Awake", "col_album": "THE_LAST_GOODBYE", "col_year": 2022, "col_vocals": True}, ], - "last_modified": "2023-06-05T03:54:07.000Z" + "last_modified": "2023-06-05T03:54:07.000Z", }, "california_festivals.xlsx": { "contents": [ - {"col_name": "Lightning in a Bottle", "col_location": {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, "col_attendance": 18000}, - {"col_name": "Outside Lands", "col_location": {"country": "USA", "state": "California", "city": "San Francisco"}, "col_attendance": 220000}, + { + "col_name": "Lightning in a Bottle", + "col_location": {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, + "col_attendance": 18000, + }, + { + "col_name": "Outside Lands", + "col_location": {"country": "USA", "state": "California", "city": "San Francisco"}, + "col_attendance": 220000, + }, ], - "last_modified": "2023-06-06T03:54:07.000Z" + "last_modified": "2023-06-06T03:54:07.000Z", }, } @@ -257,7 +269,7 @@ "col_long": 1992, "col_float": 999.09723456, "col_string": "Love It If We Made It", - "col_date": "2022-05-29T00:00:00", + "col_date": "2022-05-29T00:00:00.000000", "col_time_millis": "06:00:00.456000", "col_time_micros": "12:00:00.456789", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", @@ -407,7 +419,7 @@ "type": "object", "properties": { "col_name": {"type": ["null", "string"]}, - "col_location": {"type": ["null", "string"]}, + "col_location": {"type": ["null", "string"]}, "col_attendance": {"type": ["null", "number"]}, "_ab_source_file_last_modified": {"type": "string"}, "_ab_source_file_url": {"type": "string"}, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py index f3d72ab67e7a..6675df380c7c 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py @@ -33,7 +33,9 @@ def __init__(self) -> None: self._config: Optional[Mapping[str, Any]] = None self._state: Optional[TState] = None - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> InMemoryFilesSource: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> InMemoryFilesSource: if self._file_type is None: raise ValueError("file_type is not set") return InMemoryFilesSource( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py index 25811a9e60ad..8158225ac8f4 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -6,10 +6,15 @@ from dataclasses import dataclass, field from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteStateMessage, SyncMode +from airbyte_cdk.models import ( + AirbyteAnalyticsTraceMessage, + AirbyteStateMessageSerializer, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, + SyncMode, +) from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.source import TState -from airbyte_protocol.models import ConfiguredAirbyteCatalog @dataclass @@ -27,7 +32,9 @@ class SourceBuilder(ABC, Generic[SourceType]): """ @abstractmethod - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> SourceType: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> SourceType: raise NotImplementedError() @@ -78,7 +85,7 @@ def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]] # exception to be raised as part of the actual check/discover/read commands # Note that to avoid a breaking change, we still attempt to automatically generate the catalog based on the streams if self.catalog: - return self.catalog.dict() # type: ignore # dict() is not typed + return ConfiguredAirbyteCatalogSerializer.dump(self.catalog) catalog: Mapping[str, Any] = {"streams": []} for stream in catalog["streams"]: @@ -90,7 +97,7 @@ def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]] "supported_sync_modes": [sync_mode.value], }, "sync_mode": sync_mode.value, - "destination_sync_mode": "append" + "destination_sync_mode": "append", } ) @@ -192,7 +199,9 @@ def build(self) -> "TestScenario[SourceType]": if self.source_builder is None: raise ValueError("source_builder is not set") if self._incremental_scenario_config and self._incremental_scenario_config.input_state: - state = [AirbyteStateMessage.parse_obj(s) for s in self._incremental_scenario_config.input_state] + state = [ + AirbyteStateMessageSerializer.load(s) if isinstance(s, dict) else s for s in self._incremental_scenario_config.input_state + ] else: state = None source = self.source_builder.build( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py index 4257da83e604..97c0c491510a 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py @@ -90,7 +90,6 @@ "content": "# Title 1\n\n## Title 2\n\n### Title 3\n\n#### Title 4\n\n##### Title 5\n\n###### Title 6\n\n", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -100,7 +99,6 @@ "content": "Just some text", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -110,7 +108,6 @@ "content": "Detected via mime type", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -173,7 +170,6 @@ "content": "Just some raw text", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.txt", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -183,7 +179,6 @@ "content": "Detected via mime type", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -290,7 +285,6 @@ { "data": { "document_key": "a.csv", - "content": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=a.csv message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", @@ -358,7 +352,6 @@ "content": "A harmless markdown file", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -439,7 +432,6 @@ "content": "# Hello World", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "sample.pdf", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -449,7 +441,6 @@ "content": "# Content", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "sample.docx", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -459,7 +450,6 @@ "content": "# Title", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", "_ab_source_file_url": "sample.pptx", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -515,7 +505,6 @@ { "data": { "document_key": "sample.pdf", - "content": None, "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=sample.pdf message=No /Root object! - Is this really a PDF?", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "sample.pdf", @@ -587,7 +576,6 @@ "content": "# Hello World", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "pdf_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -597,7 +585,6 @@ "content": "# Content", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "docx_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -607,7 +594,6 @@ "content": "# Title", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", "_ab_source_file_url": "pptx_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py index e83ab5345787..3c10e701c629 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -3,9 +3,9 @@ # +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_protocol.models import SyncMode from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py index f8122da702bb..96c907901a38 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py @@ -30,7 +30,7 @@ def _make_cursor(input_state: Optional[MutableMapping[str, Any]]) -> FileBasedCo None, input_state, MagicMock(), - ConnectorStateManager(state=[AirbyteStateMessage.parse_obj(input_state)] if input_state is not None else None), + ConnectorStateManager(state=[AirbyteStateMessage(input_state)] if input_state is not None else None), CursorField(FileBasedConcurrentCursor.CURSOR_FIELD), ) return cursor diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py index 119dab4bb6fb..9563fe0af9c1 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py @@ -11,14 +11,13 @@ from _pytest.capture import CaptureFixture from _pytest.reports import ExceptionInfo from airbyte_cdk.entrypoint import launch -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalogSerializer, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read from airbyte_cdk.utils import message_utils from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario @@ -112,10 +111,10 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac if hasattr(scenario.source, "cursor_cls") and issubclass(scenario.source.cursor_cls, AbstractConcurrentFileBasedCursor): # Only check the last state emitted because we don't know the order the others will be in. # This may be needed for non-file-based concurrent scenarios too. - assert states[-1].state.stream.stream_state.dict() == expected_states[-1] + assert {k: v for k, v in states[-1].state.stream.stream_state.__dict__.items()} == expected_states[-1] else: for actual, expected in zip(states, expected_states): # states should be emitted in sorted order - assert actual.state.stream.stream_state.dict() == expected + assert {k: v for k, v in actual.state.stream.stream_state.__dict__.items()} == expected if scenario.expected_logs: read_logs = scenario.expected_logs.get("read") @@ -138,8 +137,7 @@ def _verify_state_record_counts(records: List[AirbyteMessage], states: List[Airb for state_message in states: stream_descriptor = message_utils.get_stream_descriptor(state_message) state_record_count_sums[stream_descriptor] = ( - state_record_count_sums.get(stream_descriptor, 0) - + state_message.state.sourceStats.recordCount + state_record_count_sums.get(stream_descriptor, 0) + state_message.state.sourceStats.recordCount ) for stream, actual_count in actual_record_counts.items(): @@ -154,8 +152,8 @@ def _verify_state_record_counts(records: List[AirbyteMessage], states: List[Airb def _verify_analytics(analytics: List[AirbyteMessage], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None: if expected_analytics: assert len(analytics) == len( - expected_analytics), \ - f"Number of actual analytics messages ({len(analytics)}) did not match expected ({len(expected_analytics)})" + expected_analytics + ), f"Number of actual analytics messages ({len(analytics)}) did not match expected ({len(expected_analytics)})" for actual, expected in zip(analytics, expected_analytics): actual_type, actual_value = actual.trace.analytics.type, actual.trace.analytics.value expected_type = expected.type @@ -228,7 +226,7 @@ def read(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: return entrypoint_read( scenario.source, scenario.config, - ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.full_refresh)), + ConfiguredAirbyteCatalogSerializer.load(scenario.configured_catalog(SyncMode.full_refresh)), ) @@ -236,7 +234,7 @@ def read_with_state(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: return entrypoint_read( scenario.source, scenario.config, - ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.incremental)), + ConfiguredAirbyteCatalogSerializer.load(scenario.configured_catalog(SyncMode.incremental)), scenario.input_state(), ) diff --git a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py index 95c8f96a154d..48778b657cb8 100644 --- a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py +++ b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py @@ -12,7 +12,6 @@ MessageRepository, NoopMessageRepository, ) -from pydantic.error_wrappers import ValidationError A_CONTROL = AirbyteControlMessage( type=OrchestratorType.CONNECTOR_CONFIG, @@ -90,14 +89,6 @@ def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_messag repo.log_message(Level.DEBUG, lambda: {"message": "this is a log message"}) assert list(repo.consume_queue()) - def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self): - """ - Pydantic will fail if the log level is unknown but on our side, we should try to log at least - """ - repo = InMemoryMessageRepository(Level.ERROR) - with pytest.raises(ValidationError): - repo.log_message(UNKNOWN_LEVEL, lambda: {"message": "this is a log message"}) - class TestNoopMessageRepository: def test_given_message_emitted_when_consume_queue_then_return_empty(self): diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py index ac7aa179b635..ece5039ba465 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py @@ -9,12 +9,12 @@ import pendulum import requests +from airbyte_cdk.models import ConnectorSpecification, SyncMode from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.streams import CheckpointMixin, IncrementalMixin, Stream from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from airbyte_protocol.models import ConnectorSpecification, SyncMode from requests import HTTPError @@ -23,10 +23,12 @@ class FixtureAvailabilityStrategy(HttpAvailabilityStrategy): Inherit from HttpAvailabilityStrategy with slight modification to 403 error message. """ - def reasons_for_unavailable_status_codes(self, stream: Stream, logger: logging.Logger, source: Source, error: HTTPError) -> Dict[int, str]: + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Source, error: HTTPError + ) -> Dict[int, str]: reasons_for_codes: Dict[int, str] = { requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " - "Please make sure your integration has read access for the resources you are trying to sync" + "Please make sure your integration has read access for the resources you are trying to sync" } return reasons_for_codes @@ -94,28 +96,16 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "first_name": {"type": "string"}, + "last_name": {"type": "string"}, + }, } class Planets(IncrementalIntegrationStream): - def __init__(self, **kwargs): super().__init__(**kwargs) self._state: MutableMapping[str, Any] = {} @@ -129,20 +119,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "name": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "name": {"type": "string"}, + }, } def request_params( @@ -151,10 +132,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "start_date": stream_slice.get("start_date"), - "end_date": stream_slice.get("end_date") - } + return {"start_date": stream_slice.get("start_date"), "end_date": stream_slice.get("end_date")} def stream_slices( self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None @@ -170,7 +148,10 @@ def stream_slices( while start_date < end_date: end_date_slice = min(start_date.add(days=7), end_date) - date_slice = {"start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ")} + date_slice = { + "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ"), + } date_slices.append(date_slice) start_date = end_date_slice @@ -195,20 +176,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "quote": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "quote": {"type": "string"}, + }, } def get_updated_state( @@ -221,11 +193,11 @@ def get_updated_state( return {} def read_records( - self, - sync_mode: SyncMode, - cursor_field: Optional[List[str]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - stream_state: Optional[Mapping[str, Any]] = None, + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) @@ -235,10 +207,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "start_date": stream_slice.get("start_date"), - "end_date": stream_slice.get("end_date") - } + return {"start_date": stream_slice.get("start_date"), "end_date": stream_slice.get("end_date")} def stream_slices( self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None @@ -254,7 +223,10 @@ def stream_slices( while start_date < end_date: end_date_slice = min(start_date.add(days=7), end_date) - date_slice = {"start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ")} + date_slice = { + "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ"), + } date_slices.append(date_slice) start_date = end_date_slice @@ -272,20 +244,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "divide_category": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "divide_category": {"type": "string"}, + }, } def stream_slices( @@ -319,23 +282,12 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "name": { - "type": "string" - }, - "album": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "name": {"type": "string"}, + "album": {"type": "string"}, + }, } @property @@ -360,9 +312,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "page": next_page_token.get("page") - } + return {"page": next_page_token.get("page")} def read_records( self, @@ -433,7 +383,7 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", "examples": ["2020-11-16T00:00:00.000Z"], "type": "string", - "format": "date-time" + "format": "date-time", } } } diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py index 52affbb6d76e..04b65594cf01 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py @@ -5,13 +5,16 @@ from typing import List import pytest -from airbyte_cdk.models import AirbyteMessage, Type -from airbyte_protocol.models import AirbyteStreamStatus +from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, Type def emits_successful_sync_status_messages(status_messages: List[AirbyteStreamStatus]) -> bool: - return (len(status_messages) == 3 and status_messages[0] == AirbyteStreamStatus.STARTED - and status_messages[1] == AirbyteStreamStatus.RUNNING and status_messages[2] == AirbyteStreamStatus.COMPLETE) + return ( + len(status_messages) == 3 + and status_messages[0] == AirbyteStreamStatus.STARTED + and status_messages[1] == AirbyteStreamStatus.RUNNING + and status_messages[2] == AirbyteStreamStatus.COMPLETE + ) def validate_message_order(expected_message_order: List[Type], messages: List[AirbyteMessage]): @@ -20,4 +23,6 @@ def validate_message_order(expected_message_order: List[Type], messages: List[Ai for i, message in enumerate(messages): if message.type != expected_message_order[i]: - pytest.fail(f"At index {i} actual message type {message.type.name} did not match expected message type {expected_message_order[i].name}") + pytest.fail( + f"At index {i} actual message type {message.type.name} did not match expected message type {expected_message_order[i].name}" + ) diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py index 6e68db646675..c7fd2cef433e 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py @@ -114,16 +114,7 @@ def _create_justice_songs_request() -> RequestBuilder: return RequestBuilder.justice_songs_endpoint() -RESPONSE_TEMPLATE = { - "object": "list", - "has_more": False, - "data": [ - { - "id": "123", - "created_at": "2024-01-01T07:04:28.000Z" - } - ] -} +RESPONSE_TEMPLATE = {"object": "list", "has_more": False, "data": [{"id": "123", "created_at": "2024-01-01T07:04:28.000Z"}]} USER_TEMPLATE = { "object": "list", @@ -135,7 +126,7 @@ def _create_justice_songs_request() -> RequestBuilder: "first_name": "Paul", "last_name": "Atreides", } - ] + ], } PLANET_TEMPLATE = { @@ -147,7 +138,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-01-01T07:04:28.000Z", "name": "Giedi Prime", } - ] + ], } LEGACY_TEMPLATE = { @@ -159,7 +150,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "quote": "What do you leave behind?", } - ] + ], } DIVIDER_TEMPLATE = { @@ -171,7 +162,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "divide_category": "dukes", } - ] + ], } @@ -190,8 +181,8 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "name": "dukes", "album": "", - } - ] + }, + ], } @@ -208,7 +199,7 @@ def _create_response(pagination_has_more: bool = False) -> HttpResponseBuilder: return create_response_builder( response_template=RESPONSE_TEMPLATE, records_path=FieldPath("data"), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more) + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more), ) @@ -225,9 +216,7 @@ class FullRefreshStreamTest(TestCase): @HttpMocker() def test_full_refresh_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} http_mocker.get( _create_users_request().build(), @@ -248,17 +237,15 @@ def test_full_refresh_sync(self, http_mocker): @HttpMocker() def test_substream_resumable_full_refresh_with_parent_slices(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} expected_first_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] expected_second_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, - {'partition': {'divide_category': 'mentats'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"divide_category": "mentats"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] http_mocker.get( @@ -277,10 +264,16 @@ def test_substream_resumable_full_refresh_with_parent_slices(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("dividers")) assert len(actual_messages.records) == 4 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) - assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(states=expected_first_substream_per_stream_state) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages + ) + assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob( + states=expected_first_substream_per_stream_state + ) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 - assert actual_messages.state_messages[1].state.stream.stream_state == AirbyteStateBlob(states=expected_second_substream_per_stream_state) + assert actual_messages.state_messages[1].state.stream.stream_state == AirbyteStateBlob( + states=expected_second_substream_per_stream_state + ) assert actual_messages.state_messages[1].state.sourceStats.recordCount == 2.0 @@ -289,20 +282,25 @@ class IncrementalStreamTest(TestCase): @HttpMocker() def test_incremental_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -311,7 +309,10 @@ def test_incremental_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "planets" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -322,20 +323,25 @@ def test_incremental_sync(self, http_mocker): @HttpMocker() def test_incremental_running_as_full_refresh(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -344,7 +350,10 @@ def test_incremental_running_as_full_refresh(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "planets" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) @@ -356,20 +365,25 @@ def test_incremental_running_as_full_refresh(self, http_mocker): @HttpMocker() def test_legacy_incremental_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_legacies_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_legacies_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("legacies").with_cursor(last_record_date_1)).with_record(record=_create_record("legacies").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("legacies").with_cursor(last_record_date_1)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -378,7 +392,10 @@ def test_legacy_incremental_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("legacies")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "legacies" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -389,9 +406,7 @@ def test_legacy_incremental_sync(self, http_mocker): @HttpMocker() def test_legacy_no_records_retains_incoming_state(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( @@ -412,9 +427,7 @@ def test_legacy_no_records_retains_incoming_state(self, http_mocker): @HttpMocker() def test_legacy_no_slices_retains_incoming_state(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_1 = _NOW.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -434,17 +447,15 @@ class MultipleStreamTest(TestCase): @HttpMocker() def test_incremental_and_full_refresh_streams(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} expected_first_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] expected_second_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, - {'partition': {'divide_category': 'mentats'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"divide_category": "mentats"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] # Mocks for users full refresh stream @@ -457,13 +468,20 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) # Mocks for dividers full refresh stream @@ -478,7 +496,13 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): ) source = SourceFixture() - actual_messages = read(source, config=config, catalog=_create_catalog([("users", SyncMode.full_refresh), ("planets", SyncMode.incremental), ("dividers", SyncMode.full_refresh)])) + actual_messages = read( + source, + config=config, + catalog=_create_catalog( + [("users", SyncMode.full_refresh), ("planets", SyncMode.incremental), ("dividers", SyncMode.full_refresh)] + ), + ) assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("users")) assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) @@ -486,24 +510,27 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): assert len(actual_messages.records) == 11 assert len(actual_messages.state_messages) == 5 - validate_message_order([ - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE - ], actual_messages.records_and_state_messages) + validate_message_order( + [ + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + ], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "users" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(__ab_full_refresh_sync_complete=True) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 @@ -514,8 +541,12 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): assert actual_messages.state_messages[2].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_1) assert actual_messages.state_messages[2].state.sourceStats.recordCount == 2.0 assert actual_messages.state_messages[3].state.stream.stream_descriptor.name == "dividers" - assert actual_messages.state_messages[3].state.stream.stream_state == AirbyteStateBlob(states=expected_first_substream_per_stream_state) + assert actual_messages.state_messages[3].state.stream.stream_state == AirbyteStateBlob( + states=expected_first_substream_per_stream_state + ) assert actual_messages.state_messages[3].state.sourceStats.recordCount == 2.0 assert actual_messages.state_messages[4].state.stream.stream_descriptor.name == "dividers" - assert actual_messages.state_messages[4].state.stream.stream_state == AirbyteStateBlob(states=expected_second_substream_per_stream_state) + assert actual_messages.state_messages[4].state.stream.stream_state == AirbyteStateBlob( + states=expected_second_substream_per_stream_state + ) assert actual_messages.state_messages[4].state.sourceStats.recordCount == 2.0 diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py index bc5fe899f343..f5a9e8578ab9 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py @@ -7,7 +7,7 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, SyncMode, Type +from airbyte_cdk.models import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog, FailureType, SyncMode, Type from airbyte_cdk.test.catalog_builder import ConfiguredAirbyteStreamBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest @@ -20,7 +20,6 @@ create_response_builder, ) from airbyte_cdk.test.state_builder import StateBuilder -from airbyte_protocol.models import AirbyteStreamStatus, FailureType from unit_tests.sources.mock_server_tests.mock_source_fixture import SourceFixture from unit_tests.sources.mock_server_tests.test_helpers import emits_successful_sync_status_messages, validate_message_order @@ -64,16 +63,7 @@ def _create_justice_songs_request() -> RequestBuilder: return RequestBuilder.justice_songs_endpoint() -RESPONSE_TEMPLATE = { - "object": "list", - "has_more": False, - "data": [ - { - "id": "123", - "created_at": "2024-01-01T07:04:28.000Z" - } - ] -} +RESPONSE_TEMPLATE = {"object": "list", "has_more": False, "data": [{"id": "123", "created_at": "2024-01-01T07:04:28.000Z"}]} JUSTICE_SONGS_TEMPLATE = { @@ -91,8 +81,8 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "name": "dukes", "album": "", - } - ] + }, + ], } @@ -105,7 +95,7 @@ def _create_response(pagination_has_more: bool = False) -> HttpResponseBuilder: return create_response_builder( response_template=RESPONSE_TEMPLATE, records_path=FieldPath("data"), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more) + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more), ) @@ -126,12 +116,20 @@ def test_resumable_full_refresh_sync(self, http_mocker): http_mocker.get( _create_justice_songs_request().build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(1).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( @@ -145,7 +143,10 @@ def test_resumable_full_refresh_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("justice_songs")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 4 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.STATE, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.STATE, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=1) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 @@ -167,17 +168,31 @@ def test_resumable_full_refresh_second_attempt(self, http_mocker): http_mocker.get( _create_justice_songs_request().with_page(100).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(101).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(102).build(), - _create_response(pagination_has_more=False).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=False) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) source = SourceFixture() @@ -186,7 +201,23 @@ def test_resumable_full_refresh_second_attempt(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("justice_songs")) assert len(actual_messages.records) == 8 assert len(actual_messages.state_messages) == 4 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [ + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.STATE, + ], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=101) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -206,25 +237,37 @@ def test_resumable_full_refresh_failure(self, http_mocker): http_mocker.get( _create_justice_songs_request().build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(1).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get(_create_justice_songs_request().with_page(2).build(), _create_response().with_status_code(status_code=400).build()) source = SourceFixture() - actual_messages = read(source, config=config, catalog=_create_catalog([("justice_songs", SyncMode.full_refresh, {})]), expecting_exception=True) + actual_messages = read( + source, config=config, catalog=_create_catalog([("justice_songs", SyncMode.full_refresh, {})]), expecting_exception=True + ) status_messages = actual_messages.get_stream_statuses("justice_songs") assert status_messages[-1] == AirbyteStreamStatus.INCOMPLETE assert len(actual_messages.records) == 4 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=1) assert actual_messages.state_messages[1].state.stream.stream_descriptor.name == "justice_songs" diff --git a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py index 2ccfaf33e8b6..01ddd363b0d3 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py @@ -310,7 +310,12 @@ def test_legacy_cursor_based_checkpoint_reader_resumable_full_refresh(): {"parent_id": 400, "next_page_token": 2, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 2}}, {"parent_id": 400, "next_page_token": 3, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 3}}, {"parent_id": 400, "next_page_token": 4, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 4}}, - {"parent_id": 400, "__ab_full_refresh_sync_complete": True, "partition": {"parent_id": 400}, "cursor_slice": {"__ab_full_refresh_sync_complete": True}}, + { + "parent_id": 400, + "__ab_full_refresh_sync_complete": True, + "partition": {"parent_id": 400}, + "cursor_slice": {"__ab_full_refresh_sync_complete": True}, + }, ] mocked_state = [ diff --git a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py index eb762ee08f33..4944518535f9 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py @@ -14,22 +14,8 @@ def test_substream_resumable_full_refresh_cursor(): expected_ending_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] } @@ -58,65 +44,18 @@ def test_substream_resumable_full_refresh_cursor_with_state(): """ initial_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "takeshi_aiza" - }, - "cursor": {} - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "takeshi_aiza"}, "cursor": {}}, ] } expected_ending_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "takeshi_aiza" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "emi_igawa" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "takeshi_aiza"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "emi_igawa"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] } @@ -146,9 +85,7 @@ def test_substream_resumable_full_refresh_cursor_with_state(): def test_set_initial_state_invalid_incoming_state(): - bad_state = { - "next_page_token": 2 - } + bad_state = {"next_page_token": 2} cursor = SubstreamResumableFullRefreshCursor() with pytest.raises(AirbyteTracedException): diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py index e6c91686209b..090950aa14c7 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py @@ -6,7 +6,14 @@ import logging from typing import Any, List, Mapping, Optional, Tuple, Union -from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ( + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager @@ -17,7 +24,6 @@ from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter -from airbyte_protocol.models import ConfiguredAirbyteStream from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger @@ -46,7 +52,7 @@ def __init__( self._threadpool = threadpool_manager self._cursor_field = cursor_field self._cursor_boundaries = cursor_boundaries - self._state = [AirbyteStateMessage.parse_obj(s) for s in input_state] if input_state else None + self._state = [AirbyteStateMessage(s) for s in input_state] if input_state else None def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: return True, None @@ -74,10 +80,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: self._cursor_field, self._cursor_boundaries, None, - EpochValueConcurrentStreamStateConverter.get_end_provider() + EpochValueConcurrentStreamStateConverter.get_end_provider(), ) if self._cursor_field - else FinalStateCursor(stream_name=stream.name, stream_namespace=stream.namespace, message_repository=self.message_repository), + else FinalStateCursor( + stream_name=stream.name, stream_namespace=stream.namespace, message_repository=self.message_repository + ), ) for stream, state in zip(self._streams, stream_states) ] @@ -129,6 +137,8 @@ def set_input_state(self, state: List[Mapping[str, Any]]) -> "StreamFacadeSource self._input_state = state return self - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> StreamFacadeSource: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> StreamFacadeSource: threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool") return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py index 43c198916a67..51d83084041e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py @@ -5,7 +5,7 @@ import logging from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union -from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, SyncMode from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository @@ -19,7 +19,6 @@ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.utils.slice_logger import SliceLogger -from airbyte_protocol.models import ConfiguredAirbyteStream from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder @@ -49,7 +48,16 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=self.message_repository), NeverLogSliceLogger(), s._logger) for s in self._streams] + return [ + StreamFacade( + s, + LegacyStream(), + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=self.message_repository), + NeverLogSliceLogger(), + s._logger, + ) + for s in self._streams + ] def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: return ConnectorSpecification(connectionSpecification={}) @@ -58,7 +66,13 @@ def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: return ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), NeverLogSliceLogger(), s._logger).as_airbyte_stream(), + stream=StreamFacade( + s, + LegacyStream(), + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), + NeverLogSliceLogger(), + s._logger, + ).as_airbyte_stream(), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py index 19a4cdb62627..31688999372d 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py @@ -566,7 +566,9 @@ def test_on_exception_return_trace_message_and_on_stream_complete_return_stream_ handler.is_done() @freezegun.freeze_time("2020-01-01T00:00:00") - def test_given_underlying_exception_is_traced_exception_on_exception_return_trace_message_and_on_stream_complete_return_stream_status(self): + def test_given_underlying_exception_is_traced_exception_on_exception_return_trace_message_and_on_stream_complete_return_stream_status( + self, + ): stream_instances_to_read_from = [self._stream, self._another_stream] handler = ConcurrentReadProcessor( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py index b8fa8b2f79e0..3f511c7b51da 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py @@ -102,10 +102,7 @@ def test_given_state_not_sequential_when_close_partition_then_emit_state(self) - self._state_manager.update_state_for_stream.assert_called_once_with( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - { - "slices": [{"end": 0, "start": 0}, {"end": 30, "start": 12}], - "state_type": "date-range" - }, + {"slices": [{"end": 0, "start": 0}, {"end": 30, "start": 12}], "state_type": "date-range"}, ) def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None: @@ -197,7 +194,7 @@ def test_given_one_slice_when_generate_slices_then_create_slice_from_slice_upper "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -225,7 +222,7 @@ def test_given_start_after_slices_when_generate_slices_then_generate_from_start( "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -254,7 +251,7 @@ def test_given_state_with_gap_and_start_after_slices_when_generate_slices_then_g "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 10}, {EpochValueConcurrentStreamStateConverter.START_KEY: 15, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -283,7 +280,7 @@ def test_given_small_slice_range_when_generate_slices_then_create_many_slices(se "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -316,7 +313,7 @@ def test_given_difference_between_slices_match_slice_range_when_generate_slices_ "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 30}, {EpochValueConcurrentStreamStateConverter.START_KEY: 40, EpochValueConcurrentStreamStateConverter.END_KEY: 50}, - ] + ], }, self._message_repository, self._state_manager, @@ -346,7 +343,7 @@ def test_given_non_continuous_state_when_generate_slices_then_create_slices_betw {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 10}, {EpochValueConcurrentStreamStateConverter.START_KEY: 20, EpochValueConcurrentStreamStateConverter.END_KEY: 25}, {EpochValueConcurrentStreamStateConverter.START_KEY: 30, EpochValueConcurrentStreamStateConverter.END_KEY: 40}, - ] + ], }, self._message_repository, self._state_manager, @@ -378,7 +375,7 @@ def test_given_lookback_window_when_generate_slices_then_apply_lookback_on_most_ "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, {EpochValueConcurrentStreamStateConverter.START_KEY: 30, EpochValueConcurrentStreamStateConverter.END_KEY: 40}, - ] + ], }, self._message_repository, self._state_manager, @@ -407,7 +404,7 @@ def test_given_start_is_before_first_slice_lower_boundary_when_generate_slices_t "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 10, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py index d11154e71297..da67ff82588d 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py @@ -68,7 +68,10 @@ def test_given_exception_when_generate_partitions_then_return_exception_and_sent self._partition_generator.generate_partitions(stream) queue_content = self._consume_queue() - assert queue_content == _SOME_PARTITIONS + [StreamThreadException(exception, _A_STREAM_NAME), PartitionGenerationCompletedSentinel(stream)] + assert queue_content == _SOME_PARTITIONS + [ + StreamThreadException(exception, _A_STREAM_NAME), + PartitionGenerationCompletedSentinel(stream), + ] def _partitions_before_raising(self, partitions: List[Partition], exception: Exception) -> Callable[[], Iterable[Partition]]: def inner_function() -> Iterable[Partition]: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py index de795a409b1a..67e7e3503c6c 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py @@ -15,8 +15,9 @@ def test_given_no_arguments_default_backoff_strategy_returns_default_values(): class CustomBackoffStrategy(BackoffStrategy): - - def backoff_time(self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], attempt_count: int) -> Optional[float]: + def backoff_time( + self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], attempt_count: int + ) -> Optional[float]: return response_or_exception.headers["Retry-After"] diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py index e56d97a4fa18..6da3e15b2a69 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py @@ -26,10 +26,12 @@ def test_given_ok_response_http_status_error_handler_returns_success_action(mock "error, expected_action, expected_failure_type, expected_error_message", [ (403, ResponseAction.FAIL, FailureType.config_error, "Forbidden. You don't have permission to access this resource."), - (404, ResponseAction.FAIL, FailureType.system_error, "Not found. The requested resource was not found on the server.") - ] + (404, ResponseAction.FAIL, FailureType.system_error, "Not found. The requested resource was not found on the server."), + ], ) -def test_given_error_code_in_response_http_status_error_handler_returns_expected_actions(error, expected_action, expected_failure_type, expected_error_message): +def test_given_error_code_in_response_http_status_error_handler_returns_expected_actions( + error, expected_action, expected_failure_type, expected_error_message +): response = requests.Response() response.status_code = error error_resolution = HttpStatusErrorHandler(logger).interpret_response(response) @@ -98,14 +100,10 @@ def test_given_injected_error_mapping_returns_expected_action(): assert default_error_resolution.error_message == f"Unexpected HTTP Status Code in error handler: {mock_response.status_code}" mapped_error_resolution = ErrorResolution( - response_action=ResponseAction.IGNORE, - failure_type=FailureType.transient_error, - error_message="Injected mapping" - ) - - error_mapping = { - 509: mapped_error_resolution - } + response_action=ResponseAction.IGNORE, failure_type=FailureType.transient_error, error_message="Injected mapping" + ) + + error_mapping = {509: mapped_error_resolution} actual_error_resolution = HttpStatusErrorHandler(logger, error_mapping).interpret_response(mock_response) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py index 81f838170341..2eff4bc3f05e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py @@ -8,23 +8,21 @@ @pytest.mark.parametrize( - "response_body,expected_error_message", - [ - (b'{"message": "json error message"}', "json error message"), - (b'[{"message": "list error message"}]', "list error message"), - (b'[{"message": "list error message 1"}, {"message": "list error message 2"}]', "list error message 1, list error message 2"), - (b'{"error": "messages error message"}', "messages error message"), - (b'[{"errors": "list error message 1"}, {"errors": "list error message 2"}]', "list error message 1, list error message 2"), - (b'{"failures": "failures error message"}', "failures error message"), - (b'{"failure": "failure error message"}', "failure error message"), - (b'{"detail": "detail error message"}', "detail error message"), - (b'{"err": "err error message"}', "err error message"), - (b'{"error_message": "error_message error message"}', "error_message error message"), - (b'{"msg": "msg error message"}', "msg error message"), - (b'{"reason": "reason error message"}', "reason error message"), - (b'{"status_message": "status_message error message"}', "status_message error message"), - ] - + "response_body,expected_error_message", + [ + (b'{"message": "json error message"}', "json error message"), + (b'[{"message": "list error message"}]', "list error message"), + (b'[{"message": "list error message 1"}, {"message": "list error message 2"}]', "list error message 1, list error message 2"), + (b'{"error": "messages error message"}', "messages error message"), + (b'[{"errors": "list error message 1"}, {"errors": "list error message 2"}]', "list error message 1, list error message 2"), + (b'{"failures": "failures error message"}', "failures error message"), + (b'{"failure": "failure error message"}', "failure error message"), + (b'{"detail": "detail error message"}', "detail error message"), + (b'{"err": "err error message"}', "err error message"), + (b'{"error_message": "error_message error message"}', "error_message error message"), + (b'{"msg": "msg error message"}', "msg error message"), + (b'{"reason": "reason error message"}', "reason error message"), + (b'{"status_message": "status_message error message"}', "status_message error message"),], ) def test_given_error_message_in_response_body_parse_response_error_message_returns_error_message(response_body, expected_error_message): response = requests.Response() @@ -35,6 +33,6 @@ def test_given_error_message_in_response_body_parse_response_error_message_retur def test_given_invalid_json_body_parse_response_error_message_returns_none(): response = requests.Response() - response._content = b'invalid json body' + response._content = b"invalid json body" error_message = JsonErrorMessageParser().parse_response_error_message(response) assert error_message is None diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py index 62cde8d86690..a19d3c8d5fe0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py @@ -4,16 +4,15 @@ import requests import requests_mock +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction, create_fallback_error_resolution from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets -from airbyte_protocol.models import FailureType _A_SECRET = "a-secret" _A_URL = "https://a-url.com" class DefaultErrorResolutionTest(TestCase): - def setUp(self) -> None: update_secrets([_A_SECRET]) @@ -26,7 +25,10 @@ def test_given_none_when_create_fallback_error_resolution_then_return_error_reso assert error_resolution.failure_type == FailureType.system_error assert error_resolution.response_action == ResponseAction.RETRY - assert error_resolution.error_message == "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support" + assert ( + error_resolution.error_message + == "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support" + ) def test_given_exception_when_create_fallback_error_resolution_then_return_error_resolution(self) -> None: exception = ValueError("This is an exception") diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 1300ad8e94df..42975d8ed5a9 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -83,7 +83,7 @@ def read_records(self, *args, **kvargs): http_stream = MockListHttpStream() response = requests.Response() response.status_code = status_code - response.raw = io.BytesIO(json.dumps(json_contents).encode('utf-8')) + response.raw = io.BytesIO(json.dumps(json_contents).encode("utf-8")) mocker.patch.object(requests.Session, "send", return_value=response) actual_is_available, reason = HttpAvailabilityStrategy().check_availability(http_stream, logger) @@ -104,7 +104,9 @@ def test_http_availability_raises_unhandled_error(mocker): req.status_code = 404 mocker.patch.object(requests.Session, "send", return_value=req) - assert (False, 'Not found. The requested resource was not found on the server.') == HttpAvailabilityStrategy().check_availability(http_stream, logger) + assert (False, "Not found. The requested resource was not found on the server.") == HttpAvailabilityStrategy().check_availability( + http_stream, logger + ) def test_send_handles_retries_when_checking_availability(mocker, caplog): @@ -120,7 +122,7 @@ def test_send_handles_retries_when_checking_availability(mocker, caplog): mock_send = mocker.patch.object(requests.Session, "send", side_effect=[req_1, req_2, req_3]) with caplog.at_level(logging.INFO): - stream_is_available, _ = HttpAvailabilityStrategy().check_availability(stream=http_stream,logger=logger) + stream_is_available, _ = HttpAvailabilityStrategy().check_availability(stream=http_stream, logger=logger) assert stream_is_available assert mock_send.call_count == 3 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 27c7c7414e36..8737289a780f 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -490,15 +490,13 @@ def should_retry(self, *args, **kwargs): [ (300, True, True, ResponseAction.RETRY), (200, False, True, ResponseAction.SUCCESS), - (503, False,True, ResponseAction.FAIL), - (503,False,False, ResponseAction.IGNORE) - ] + (503, False, True, ResponseAction.FAIL), + (503, False, False, ResponseAction.IGNORE), + ], ) -def test_http_stream_adapter_http_status_error_handler_should_retry_false_raise_on_http_errors(mocker, - response_status_code: int, - should_retry: bool, - raise_on_http_errors: bool, - expected_response_action: ResponseAction): +def test_http_stream_adapter_http_status_error_handler_should_retry_false_raise_on_http_errors( + mocker, response_status_code: int, should_retry: bool, raise_on_http_errors: bool, expected_response_action: ResponseAction +): stream = AutoFailTrueHttpStream() mocker.patch.object(stream, "should_retry", return_value=should_retry) mocker.patch.object(stream, "raise_on_http_errors", raise_on_http_errors) @@ -664,9 +662,19 @@ def test_duplicate_request_params_are_deduped(deduplicate_query_params, path, pa if expected_url is None: with pytest.raises(ValueError): - stream._http_client._create_prepared_request(http_method=stream.http_method, url=stream._join_url(stream.url_base, path), params=params, dedupe_query_params=deduplicate_query_params) + stream._http_client._create_prepared_request( + http_method=stream.http_method, + url=stream._join_url(stream.url_base, path), + params=params, + dedupe_query_params=deduplicate_query_params, + ) else: - prepared_request = stream._http_client._create_prepared_request(http_method=stream.http_method, url=stream._join_url(stream.url_base, path), params=params, dedupe_query_params=deduplicate_query_params) + prepared_request = stream._http_client._create_prepared_request( + http_method=stream.http_method, + url=stream._join_url(stream.url_base, path), + params=params, + dedupe_query_params=deduplicate_query_params, + ) assert prepared_request.url == expected_url @@ -689,8 +697,13 @@ def __init__(self, records: List[Mapping[str, Any]]): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -709,12 +722,12 @@ def _read_single_page( self.state = {"__ab_full_refresh_sync_complete": True} def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -736,8 +749,13 @@ def __init__(self, record_pages: List[List[Mapping[str, Any]]]): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -759,12 +777,12 @@ def read_records( self.state = {"__ab_full_refresh_sync_complete": True} def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -779,8 +797,13 @@ class StubHttpSubstream(HttpSubStream): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -800,12 +823,12 @@ def _read_pages( ] def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -841,7 +864,7 @@ def test_substream_with_resumable_full_refresh_parent(): [ {"id": "page_3_abc"}, {"id": "page_3_def"}, - ] + ], ] expected_slices = [ @@ -987,10 +1010,7 @@ def test_resumable_full_refresh_read_from_state(mocker): mocker.patch.object(stream, method, wraps=getattr(stream, method)) checkpoint_reader = stream._get_checkpoint_reader( - cursor_field=[], - logger=logging.getLogger("airbyte"), - sync_mode=SyncMode.full_refresh, - stream_state={"page": 3} + cursor_field=[], logger=logging.getLogger("airbyte"), sync_mode=SyncMode.full_refresh, stream_state={"page": 3} ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1036,10 +1056,7 @@ def test_resumable_full_refresh_legacy_stream_slice(mocker): mocker.patch.object(stream, method, wraps=getattr(stream, method)) checkpoint_reader = stream._get_checkpoint_reader( - cursor_field=[], - logger=logging.getLogger("airbyte"), - sync_mode=SyncMode.full_refresh, - stream_state={"page": 2} + cursor_field=[], logger=logging.getLogger("airbyte"), sync_mode=SyncMode.full_refresh, stream_state={"page": 2} ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1082,8 +1099,13 @@ def __init__(self, parent: HttpStream, partition_id_to_child_records: Mapping[st def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return f"/parents/{stream_slice.get('parent_id')}/children" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -1113,12 +1135,12 @@ def _fetch_next_page( return requests.PreparedRequest(), requests.Response() def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: partition_id = stream_slice.get("parent").get("parent_id") if partition_id in self._partition_id_to_child_records: @@ -1141,14 +1163,21 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): {"parent_id": "100", "name": "christopher_nolan"}, {"parent_id": "101", "name": "celine_song"}, {"parent_id": "102", "name": "david_fincher"}, - ] parent_stream = StubParentHttpStream(records=parent_records) parents_to_children_records = { - "100": [{"id": "a200", "parent_id": "100", "film": "interstellar"}, {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, {"id": "a202", "parent_id": "100", "film": "inception"}], + "100": [ + {"id": "a200", "parent_id": "100", "film": "interstellar"}, + {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, + {"id": "a202", "parent_id": "100", "film": "inception"}, + ], "101": [{"id": "b200", "parent_id": "101", "film": "past_lives"}, {"id": "b201", "parent_id": "101", "film": "materialists"}], - "102": [{"id": "c200", "parent_id": "102", "film": "the_social_network"}, {"id": "c201", "parent_id": "102", "film": "gone_girl"}, {"id": "c202", "parent_id": "102", "film": "the_curious_case_of_benjamin_button"}], + "102": [ + {"id": "c200", "parent_id": "102", "film": "the_social_network"}, + {"id": "c201", "parent_id": "102", "film": "gone_girl"}, + {"id": "c202", "parent_id": "102", "film": "the_curious_case_of_benjamin_button"}, + ], } stream = StubSubstreamResumableFullRefreshStream(parent=parent_stream, partition_id_to_child_records=parents_to_children_records) @@ -1168,61 +1197,31 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, } ] }, { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } - } + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, ] }, { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "david_fincher", "parent_id": "102"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "david_fincher", "parent_id": "102"} - } - } ] }, ] @@ -1239,46 +1238,14 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): assert getattr(stream, "_read_pages").call_count == 3 expected = [ - { - "film": "interstellar", - "id": "a200", - "parent_id": "100" - }, - { - "film": "oppenheimer", - "id": "a201", - "parent_id": "100" - }, - { - "film": "inception", - "id": "a202", - "parent_id": "100" - }, - { - "film": "past_lives", - "id": "b200", - "parent_id": "101" - }, - { - "film": "materialists", - "id": "b201", - "parent_id": "101" - }, - { - "film": "the_social_network", - "id": "c200", - "parent_id": "102" - }, - { - "film": "gone_girl", - "id": "c201", - "parent_id": "102" - }, - { - "film": "the_curious_case_of_benjamin_button", - "id": "c202", - "parent_id": "102" - } + {"film": "interstellar", "id": "a200", "parent_id": "100"}, + {"film": "oppenheimer", "id": "a201", "parent_id": "100"}, + {"film": "inception", "id": "a202", "parent_id": "100"}, + {"film": "past_lives", "id": "b200", "parent_id": "101"}, + {"film": "materialists", "id": "b201", "parent_id": "101"}, + {"film": "the_social_network", "id": "c200", "parent_id": "102"}, + {"film": "gone_girl", "id": "c201", "parent_id": "102"}, + {"film": "the_curious_case_of_benjamin_button", "id": "c202", "parent_id": "102"}, ] assert records == expected @@ -1294,13 +1261,15 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): parent_records = [ {"parent_id": "100", "name": "christopher_nolan"}, {"parent_id": "101", "name": "celine_song"}, - ] parent_stream = StubParentHttpStream(records=parent_records) parents_to_children_records = { - "100": [{"id": "a200", "parent_id": "100", "film": "interstellar"}, {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, - {"id": "a202", "parent_id": "100", "film": "inception"}], + "100": [ + {"id": "a200", "parent_id": "100", "film": "interstellar"}, + {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, + {"id": "a202", "parent_id": "100", "film": "inception"}, + ], "101": [{"id": "b200", "parent_id": "101", "film": "past_lives"}, {"id": "b201", "parent_id": "101", "film": "materialists"}], } stream = StubSubstreamResumableFullRefreshStream(parent=parent_stream, partition_id_to_child_records=parents_to_children_records) @@ -1318,15 +1287,11 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): stream_state={ "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, ] - } + }, ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1335,21 +1300,10 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } - } + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, ] }, ] @@ -1366,16 +1320,8 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): assert getattr(stream, "_read_pages").call_count == 1 expected = [ - { - "film": "past_lives", - "id": "b200", - "parent_id": "101" - }, - { - "film": "materialists", - "id": "b201", - "parent_id": "101" - }, + {"film": "past_lives", "id": "b200", "parent_id": "101"}, + {"film": "materialists", "id": "b201", "parent_id": "101"}, ] assert records == expected @@ -1398,8 +1344,13 @@ def cursor_field(self) -> Union[str, List[str]]: pytest.param([], False, ResumableFullRefreshCursor(), id="test_stream_supports_resumable_full_refresh_cursor"), pytest.param(["updated_at"], False, None, id="test_incremental_stream_does_not_use_cursor"), pytest.param(["updated_at"], True, None, id="test_incremental_substream_does_not_use_cursor"), - pytest.param([], True, SubstreamResumableFullRefreshCursor(), id="test_full_refresh_substream_automatically_applies_substream_resumable_full_refresh_cursor"), - ] + pytest.param( + [], + True, + SubstreamResumableFullRefreshCursor(), + id="test_full_refresh_substream_automatically_applies_substream_resumable_full_refresh_cursor", + ), + ], ) def test_get_cursor(cursor_field, is_substream, expected_cursor): stream = StubWithCursorFields(set_cursor_field=cursor_field, has_multiple_slices=is_substream) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py index efb4a9f1f4bd..f7b0a11f69f4 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py @@ -48,19 +48,23 @@ def test_request_session_returns_valid_session(use_cache, expected_session): True, "https://test_base_url.com/v1/endpoint?param1=value1", {}, - "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_only_in_path", ), pytest.param( True, "https://test_base_url.com/v1/endpoint", {"param1": "value1"}, - "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_only_in_path", ), pytest.param( True, "https://test_base_url.com/v1/endpoint", None, - "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), + "https://test_base_url.com/v1/endpoint", + id="test_params_is_none_and_no_params_in_path", + ), pytest.param( True, "https://test_base_url.com/v1/endpoint?param1=value1", @@ -119,7 +123,9 @@ def test_duplicate_request_params_are_deduped(deduplicate_query_params, url, par with pytest.raises(ValueError): http_client._create_prepared_request(http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params) else: - prepared_request = http_client._create_prepared_request(http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params) + prepared_request = http_client._create_prepared_request( + http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params + ) assert prepared_request.url == expected_url @@ -127,7 +133,9 @@ def test_create_prepared_response_given_given_both_json_and_data_raises_request_ http_client = test_http_client() with pytest.raises(RequestBodyException): - http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint", json={"test": "json"}, data={"test": "data"}) + http_client._create_prepared_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", json={"test": "json"}, data={"test": "data"} + ) @pytest.mark.parametrize( @@ -139,7 +147,9 @@ def test_create_prepared_response_given_given_both_json_and_data_raises_request_ ) def test_create_prepared_response_given_either_json_or_data_returns_valid_request(json, data): http_client = test_http_client() - prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint", json=json, data=data) + prepared_request = http_client._create_prepared_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", json=json, data=data + ) assert prepared_request assert isinstance(prepared_request, requests.PreparedRequest) @@ -155,7 +165,9 @@ def test_valid_basic_send_request(mocker): mocked_response.status_code = 200 mocked_response.headers = {} mocker.patch.object(requests.Session, "send", return_value=mocked_response) - returned_request, returned_response = http_client.send_request(http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}) + returned_request, returned_response = http_client.send_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={} + ) assert isinstance(returned_request, requests.PreparedRequest) assert returned_response == mocked_response @@ -166,8 +178,10 @@ def test_send_raises_airbyte_traced_exception_with_fail_response_action(): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={400: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test error message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={400: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test error message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() mocked_response = requests.Response() @@ -190,8 +204,10 @@ def test_send_ignores_with_ignore_reponse_action_and_returns_response(): http_client = HttpClient( name="test", logger=mocked_logger, - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={300: ErrorResolution(ResponseAction.IGNORE, FailureType.system_error, "test ignore message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={300: ErrorResolution(ResponseAction.IGNORE, FailureType.system_error, "test ignore message")} + ), + session=mocked_session, ) prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint") @@ -204,7 +220,6 @@ def test_send_ignores_with_ignore_reponse_action_and_returns_response(): class CustomBackoffStrategy(BackoffStrategy): - def __init__(self, backoff_time_value: float) -> None: self._backoff_time_value = backoff_time_value @@ -212,19 +227,15 @@ def backoff_time(self, *args, **kwargs) -> float: return self._backoff_time_value -@pytest.mark.parametrize( - "backoff_time_value, exception_type", - [ - (0.1, UserDefinedBackoffException), - (None, DefaultBackoffException) - ] -) +@pytest.mark.parametrize("backoff_time_value, exception_type", [(0.1, UserDefinedBackoffException), (None, DefaultBackoffException)]) def test_raises_backoff_exception_with_retry_response_action(mocker, backoff_time_value, exception_type): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")}), - backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value) + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")} + ), + backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value), ) prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint") mocked_response = MagicMock(spec=requests.Response) @@ -233,25 +244,25 @@ def test_raises_backoff_exception_with_retry_response_action(mocker, backoff_tim http_client._logger.info = MagicMock() mocker.patch.object(requests.Session, "send", return_value=mocked_response) - mocker.patch.object(http_client._error_handler, "interpret_response", return_value=ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")) + mocker.patch.object( + http_client._error_handler, + "interpret_response", + return_value=ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message"), + ) with pytest.raises(exception_type): http_client._send(prepared_request, {}) -@pytest.mark.parametrize( - "backoff_time_value, exception_type", - [ - (0.1, UserDefinedBackoffException), - (None, DefaultBackoffException) - ] -) +@pytest.mark.parametrize("backoff_time_value, exception_type", [(0.1, UserDefinedBackoffException), (None, DefaultBackoffException)]) def test_raises_backoff_exception_with_response_with_unmapped_error(mocker, backoff_time_value, exception_type): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")}), - backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value) + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")} + ), + backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value), ) prepared_request = requests.PreparedRequest() mocked_response = MagicMock(spec=requests.Response) @@ -289,8 +300,10 @@ def update_response(*args, **kwargs): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() @@ -302,15 +315,15 @@ def update_response(*args, **kwargs): def test_session_request_exception_raises_backoff_exception(): - error_handler = HttpStatusErrorHandler(logger=MagicMock(), error_mapping={requests.exceptions.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}) - mocked_session = MagicMock(spec=requests.Session) - mocked_session.send.side_effect = requests.RequestException - http_client = HttpClient( - name="test", + error_handler = HttpStatusErrorHandler( logger=MagicMock(), - error_handler=error_handler, - session=mocked_session + error_mapping={ + requests.exceptions.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message") + }, ) + mocked_session = MagicMock(spec=requests.Session) + mocked_session.send.side_effect = requests.RequestException + http_client = HttpClient(name="test", logger=MagicMock(), error_handler=error_handler, session=mocked_session) prepared_request = requests.PreparedRequest() with pytest.raises(DefaultBackoffException): @@ -347,12 +360,7 @@ def test_send_handles_response_action_given_session_send_raises_request_exceptio mocked_session = MagicMock(spec=requests.Session) mocked_session.send.side_effect = requests.RequestException - http_client = HttpClient( - name="test", - logger=MagicMock(), - error_handler=custom_error_handler, - session=mocked_session - ) + http_client = HttpClient(name="test", logger=MagicMock(), error_handler=custom_error_handler, session=mocked_session) prepared_request = requests.PreparedRequest() with pytest.raises(AirbyteTracedException) as e: @@ -383,8 +391,10 @@ def update_response(*args, **kwargs): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() @@ -400,7 +410,13 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy(), disable_retries=True) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock()), + backoff_strategy=BackoffStrategy(), + disable_retries=True, + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -421,7 +437,9 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy() + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -444,7 +462,12 @@ def backoff_time(self, *args, **kwargs): retries = 3 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=retries), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=retries), + backoff_strategy=BackoffStrategy(), + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -461,7 +484,12 @@ def backoff_time(self, *args, **kwargs): @pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_max_time(): - error_handler = HttpStatusErrorHandler(logger=MagicMock(), error_mapping={requests.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}, max_retries=10, max_time=timedelta(seconds=2)) + error_handler = HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={requests.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}, + max_retries=10, + max_time=timedelta(seconds=2), + ) class BackoffStrategy: def backoff_time(self, *args, **kwargs): @@ -488,7 +516,9 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy() + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -505,7 +535,9 @@ def backoff_time(self, *args, **kwargs): assert len(trace_messages) == mocked_send.call_count -@pytest.mark.parametrize("exit_on_rate_limit, expected_call_count, expected_error",[[True, 6, DefaultBackoffException] ,[False, 38, OverflowError]]) +@pytest.mark.parametrize( + "exit_on_rate_limit, expected_call_count, expected_error", [[True, 6, DefaultBackoffException], [False, 38, OverflowError]] +) @pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_endless(exit_on_rate_limit, expected_call_count, expected_error): http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock())) @@ -519,5 +551,7 @@ def test_backoff_strategy_endless(exit_on_rate_limit, expected_call_count, expec with patch.object(requests.Session, "send", return_value=mocked_response) as mocked_send: with pytest.raises(expected_error): - http_client.send_request(http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}, exit_on_rate_limit=exit_on_rate_limit) + http_client.send_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}, exit_on_rate_limit=exit_on_rate_limit + ) assert mocked_send.call_count == expected_call_count diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py index b40f93ed0327..9f6f943e0840 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py @@ -470,7 +470,9 @@ def test_configured_json_schema(): }, } - configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies(current_json_schema) + configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies( + current_json_schema + ) records = [ {"id": 1, "partition": 1}, {"id": 2, "partition": 1}, @@ -506,7 +508,9 @@ def test_configured_json_schema_with_invalid_properties(): del stream_schema["properties"][old_user_insights] del stream_schema["properties"][old_feature_info] - configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies(configured_json_schema) + configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies( + configured_json_schema + ) records = [ {"id": 1, "partition": 1}, {"id": 2, "partition": 1}, @@ -521,7 +525,9 @@ def test_configured_json_schema_with_invalid_properties(): assert old_user_insights not in configured_json_schema_properties assert old_feature_info not in configured_json_schema_properties for stream_schema_property in stream_schema["properties"]: - assert stream_schema_property in configured_json_schema_properties, f"Stream schema property: {stream_schema_property} missing in configured schema" + assert ( + stream_schema_property in configured_json_schema_properties + ), f"Stream schema property: {stream_schema_property} missing in configured schema" assert stream_schema["properties"][stream_schema_property] == configured_json_schema_properties[stream_schema_property] diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py index 019705d1cd75..9f356b5c80bb 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py @@ -46,6 +46,7 @@ class StreamStubIncremental(Stream, CheckpointMixin): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -74,6 +75,7 @@ class StreamStubResumableFullRefresh(Stream, CheckpointMixin): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -100,6 +102,7 @@ class StreamStubLegacyStateInterface(Stream): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -154,17 +157,22 @@ def url_base(self) -> str: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: pass - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -203,17 +211,22 @@ def stream_slices( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: pass - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -234,17 +247,22 @@ def read_records( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/parent" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -374,11 +392,25 @@ def test_get_json_schema_is_cached(mocked_method): [ pytest.param(StreamStubIncremental(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader"), pytest.param(StreamStubFullRefresh(), {}, FullRefreshCheckpointReader, id="test_full_refresh_checkpoint_reader"), - pytest.param(StreamStubResumableFullRefresh(), {}, ResumableFullRefreshCheckpointReader, id="test_resumable_full_refresh_checkpoint_reader"), - pytest.param(StreamStubLegacyStateInterface(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader_with_legacy_state"), - pytest.param(CursorBasedStreamStubFullRefresh(), {"next_page_token": 10}, CursorBasedCheckpointReader, id="test_checkpoint_reader_using_rfr_cursor"), - pytest.param(LegacyCursorBasedStreamStubFullRefresh(), {}, LegacyCursorBasedCheckpointReader, id="test_full_refresh_checkpoint_reader_for_legacy_slice_format"), - ] + pytest.param( + StreamStubResumableFullRefresh(), {}, ResumableFullRefreshCheckpointReader, id="test_resumable_full_refresh_checkpoint_reader" + ), + pytest.param( + StreamStubLegacyStateInterface(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader_with_legacy_state" + ), + pytest.param( + CursorBasedStreamStubFullRefresh(), + {"next_page_token": 10}, + CursorBasedCheckpointReader, + id="test_checkpoint_reader_using_rfr_cursor", + ), + pytest.param( + LegacyCursorBasedStreamStubFullRefresh(), + {}, + LegacyCursorBasedCheckpointReader, + id="test_full_refresh_checkpoint_reader_for_legacy_slice_format", + ), + ], ) def test_get_checkpoint_reader(stream: Stream, stream_state, expected_checkpoint_reader_type): checkpoint_reader = stream._get_checkpoint_reader( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py b/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py index 8cf1996853fd..da76a78714d7 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py @@ -11,8 +11,7 @@ def __init__(self, records, exit_on_rate_limit=True): self.records = records self._exit_on_rate_limit = exit_on_rate_limit type(self).exit_on_rate_limit = property( - lambda self: self._get_exit_on_rate_limit(), - lambda self, value: self._set_exit_on_rate_limit(value) + lambda self: self._get_exit_on_rate_limit(), lambda self, value: self._set_exit_on_rate_limit(value) ) def _get_exit_on_rate_limit(self): @@ -31,7 +30,7 @@ def read_records(self, sync_mode, stream_slice): ([{"id": 1}], None, True, {"id": 1}, False), # Single record, with setter ([{"id": 1}, {"id": 2}], None, True, {"id": 1}, False), # Multiple records, with setter ([], None, True, None, True), # No records, with setter - ] + ], ) def test_get_first_record_for_slice(records, stream_slice, exit_on_rate_limit, expected_result, raises_exception): stream = MockStream(records, exit_on_rate_limit) diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index af6a8b0a5f03..9de46b9e116f 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -198,7 +198,6 @@ def __init__(self, inputs_and_mocked_outputs: List[Tuple[Mapping[str, Any], Iter class MockStreamWithState(MockStreamWithCursor): - def __init__(self, inputs_and_mocked_outputs: List[Tuple[Mapping[str, Any], Iterable[Mapping[str, Any]]]], name: str, state=None): super().__init__(inputs_and_mocked_outputs, name) self._state = state @@ -422,7 +421,7 @@ def _as_state(stream_name: str = "", per_stream_state: Dict[str, Any] = None): state=AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob.parse_obj(per_stream_state) + stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(per_stream_state) ), ), ) @@ -606,9 +605,7 @@ def test_with_state_attribute(self, mocker): input_state = [ AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob.parse_obj(old_state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob(old_state)), ), ] new_state_from_connector = {"cursor": "new_value"} @@ -860,13 +857,7 @@ def test_with_slices(self, mocker): assert messages == expected - @pytest.mark.parametrize( - "slices", - [ - pytest.param([], id="test_slices_as_list"), - pytest.param(iter([]), id="test_slices_as_iterator") - ] - ) + @pytest.mark.parametrize("slices", [pytest.param([], id="test_slices_as_list"), pytest.param(iter([]), id="test_slices_as_iterator")]) def test_no_slices(self, mocker, slices): """ Tests that an incremental read returns at least one state messages even if no records were read: @@ -876,15 +867,11 @@ def test_no_slices(self, mocker, slices): input_state = [ AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob.parse_obj(state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob(state)), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s2"), stream_state=AirbyteStateBlob.parse_obj(state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s2"), stream_state=AirbyteStateBlob(state)), ), ] @@ -1185,15 +1172,12 @@ def test_without_state_attribute_for_stream_with_desc_records(self, mocker): AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob.parse_obj(initial_state) + stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(initial_state) ), ), ] stream_with_cursor = MockStreamWithCursor( - [ - ( - {"sync_mode": SyncMode.incremental, "stream_slice": {}, "stream_state": initial_state}, stream_output) - ], + [({"sync_mode": SyncMode.incremental, "stream_slice": {}, "stream_state": initial_state}, stream_output)], name=stream_name, ) @@ -1201,6 +1185,7 @@ def mock_get_updated_state(current_stream, current_stream_state, latest_record): state_cursor_value = current_stream_state.get(current_stream.cursor_field, 0) latest_record_value = latest_record.get(current_stream.cursor_field) return {current_stream.cursor_field: max(latest_record_value, state_cursor_value)} + mocker.patch.object(MockStreamWithCursor, "get_updated_state", mock_get_updated_state) mocker.patch.object(MockStreamWithCursor, "get_json_schema", return_value={}) src = MockSource(streams=[stream_with_cursor]) @@ -1306,7 +1291,7 @@ def test_resumable_full_refresh_with_incoming_state(self, mocker): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s1"), - stream_state=AirbyteStateBlob.parse_obj({"page": 10}), + stream_state=AirbyteStateBlob({"page": 10}), ), ) ] @@ -1433,16 +1418,16 @@ def test_resumable_full_refresh_skip_prior_successful_streams(self, mocker): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s1"), - stream_state=AirbyteStateBlob.parse_obj({"__ab_full_refresh_sync_complete": True}), + stream_state=AirbyteStateBlob({"__ab_full_refresh_sync_complete": True}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s2"), - stream_state=AirbyteStateBlob.parse_obj({"page": 10}), + stream_state=AirbyteStateBlob({"page": 10}), ), - ) + ), ] src = MockSource(streams=[s1, s2]) @@ -1713,8 +1698,10 @@ def test_read_nonexistent_stream_emit_incomplete_stream_status(mocker, remove_st expected = _fix_emitted_at([as_stream_status("this_stream_doesnt_exist_in_the_source", AirbyteStreamStatus.INCOMPLETE)]) - expected_error_message = "The stream 'this_stream_doesnt_exist_in_the_source' in your connection configuration was not found in the " \ - "source. Refresh the schema in your replication settings and remove this stream from future sync attempts." + expected_error_message = ( + "The stream 'this_stream_doesnt_exist_in_the_source' in your connection configuration was not found in the " + "source. Refresh the schema in your replication settings and remove this stream from future sync attempts." + ) with pytest.raises(AirbyteTracedException) as exc_info: messages = [remove_stack_trace(message) for message in src.read(logger, {}, catalog)] diff --git a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py index bcef13b9783c..1a5526b105d5 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py +++ b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py @@ -6,7 +6,15 @@ from typing import List import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateMessageSerializer, + AirbyteStateType, + AirbyteStreamState, + StreamDescriptor, +) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor @@ -17,24 +25,24 @@ pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mando_michael"}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), - HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), + HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob({"id": "seehorn_rhea"}), }, does_not_raise(), id="test_incoming_per_stream_state", ), pytest.param([], {}, does_not_raise(), id="test_incoming_empty_stream_state"), pytest.param( - [{"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}}}], + [{"type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}}}], {HashableStreamDescriptor(name="actresses", namespace="public"): None}, does_not_raise(), id="test_stream_states_that_have_none_state_blob", @@ -42,25 +50,25 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "shared_state": {"television": "better_call_saul"}, "stream_states": [ { - "stream_descriptor": StreamDescriptor(name="actors", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + "stream_descriptor": {"name": "actors", "namespace": "public"}, + "stream_state": {"id": "mando_michael"}, }, { - "stream_descriptor": StreamDescriptor(name="actresses", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + "stream_descriptor": {"name": "actresses", "namespace": "public"}, + "stream_state": {"id": "seehorn_rhea"}, }, ], }, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), - HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), + HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob({"id": "seehorn_rhea"}), }, pytest.raises(ValueError), id="test_incoming_global_state_with_shared_state_throws_error", @@ -68,7 +76,7 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "stream_states": [ {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mando_michael"}}, @@ -77,7 +85,7 @@ }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), }, does_not_raise(), id="test_incoming_global_state_without_shared", @@ -85,20 +93,20 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "shared_state": None, "stream_states": [ { - "stream_descriptor": StreamDescriptor(name="actors", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + "stream_descriptor": {"name": "actors", "namespace": "public"}, + "stream_state": {"id": "mando_michael"}, }, ], }, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), }, does_not_raise(), id="test_incoming_global_state_with_none_shared", @@ -106,7 +114,7 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "stream_states": [ {"stream_descriptor": {"name": "actresses", "namespace": "public"}}, @@ -122,7 +130,7 @@ ) def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error): if isinstance(input_stream_state, List): - input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)] + input_stream_state = [AirbyteStateMessageSerializer.load(state_obj) for state_obj in list(input_stream_state)] with expected_error: state_manager = ConnectorStateManager(input_stream_state) @@ -136,11 +144,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -152,10 +160,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users"}, "stream_state": {"created_at": 12345}}, }, - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, ], "users", None, @@ -164,8 +172,8 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp ), pytest.param( [ - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "users"}}}, - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "users"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, ], "users", None, @@ -175,11 +183,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -191,11 +199,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -208,7 +216,7 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": None}, }, ], @@ -220,7 +228,7 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp ], ) def test_get_stream_state(input_state, stream_name, namespace, expected_state): - state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)] + state_messages = [AirbyteStateMessageSerializer.load(state_obj) for state_obj in list(input_state)] state_manager = ConnectorStateManager(state_messages) actual_state = state_manager.get_stream_state(stream_name, namespace) @@ -234,7 +242,7 @@ def test_get_state_returns_deep_copy(): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": [109]}), + stream_state=AirbyteStateBlob({"id": [109]}), ), ) ] @@ -252,11 +260,11 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mckean_michael"}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, }, ], @@ -275,7 +283,7 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, } ], @@ -287,7 +295,7 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, } ], @@ -299,14 +307,14 @@ def test_get_state_returns_deep_copy(): ], ) def test_update_state_for_stream(start_state, update_name, update_namespace, update_value): - state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(start_state)] + state_messages = [AirbyteStateMessage(state_obj) for state_obj in list(start_state)] state_manager = ConnectorStateManager(state_messages) state_manager.update_state_for_stream(update_name, update_namespace, update_value) - assert state_manager.per_stream_states[ - HashableStreamDescriptor(name=update_name, namespace=update_namespace) - ] == AirbyteStateBlob.parse_obj(update_value) + assert state_manager.per_stream_states[HashableStreamDescriptor(name=update_name, namespace=update_namespace)] == AirbyteStateBlob( + update_value + ) @pytest.mark.parametrize( @@ -318,14 +326,14 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2022_05_22"}), + stream_state=AirbyteStateBlob({"created_at": "2022_05_22"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="seasons", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 1}), + stream_state=AirbyteStateBlob({"id": 1}), ), ), ], @@ -337,7 +345,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2022_05_22"}), + stream_state=AirbyteStateBlob({"created_at": "2022_05_22"}), ), ), ), @@ -373,7 +381,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 507}), + stream_state=AirbyteStateBlob({"id": 507}), ), ) ], @@ -396,7 +404,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 507}), + stream_state=AirbyteStateBlob({"id": 507}), ), ) ], diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index c7b8e884653b..d548a51b1ebb 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -13,9 +13,11 @@ AirbyteGlobalState, AirbyteStateBlob, AirbyteStateMessage, + AirbyteStateMessageSerializer, AirbyteStateType, AirbyteStreamState, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, StreamDescriptor, SyncMode, Type, @@ -24,7 +26,8 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pydantic import ValidationError +from orjson import orjson +from serpyco_rs import SchemaValidationError class MockSource(Source): @@ -74,7 +77,7 @@ def catalog(): }, ] } - return ConfiguredAirbyteCatalog.model_validate(configured_catalog) + return ConfiguredAirbyteCatalogSerializer.load(configured_catalog) @pytest.fixture @@ -154,7 +157,7 @@ def streams(self, config): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), ), ) ], @@ -190,21 +193,21 @@ def streams(self, config): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="directors", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": "villeneuve_denis"}), + stream_state=AirbyteStateBlob({"id": "villeneuve_denis"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="actors", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "1995-12-27"}), + stream_state=AirbyteStateBlob({"created_at": "1995-12-27"}), ), ), ], @@ -224,19 +227,17 @@ def streams(self, config): } ], [ - AirbyteStateMessage.parse_obj( - { - "type": AirbyteStateType.GLOBAL, - "global": AirbyteGlobalState( - shared_state=AirbyteStateBlob.parse_obj({"shared_key": "shared_val"}), - stream_states=[ - AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), - ) - ], - ), - } + AirbyteStateMessage( + type=AirbyteStateType.GLOBAL, + global_=AirbyteGlobalState( + shared_state=AirbyteStateBlob({"shared_key": "shared_val"}), + stream_states=[ + AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="movies", namespace="public"), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), + ) + ], + ), ), ], does_not_raise(), @@ -255,19 +256,19 @@ def streams(self, config): } ], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_stream_state_invalid_type", ), pytest.param( [{"type": "STREAM", "stream": {"stream_state": {"created_at": "2009-07-19"}}}], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_stream_state_missing_descriptor", ), pytest.param( [{"type": "GLOBAL", "global": {"shared_state": {"shared_key": "shared_val"}}}], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_global_state_missing_streams", ), pytest.param( @@ -284,7 +285,7 @@ def streams(self, config): } ], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_global_state_streams_not_list", ), ], @@ -295,7 +296,8 @@ def test_read_state(source, incoming_state, expected_state, expected_error): state_file.flush() with expected_error: actual = source.read_state(state_file.name) - assert actual == expected_state + if expected_state and actual: + assert AirbyteStateMessageSerializer.dump(actual[0]) == AirbyteStateMessageSerializer.dump(expected_state[0]) def test_read_invalid_state(source): @@ -330,9 +332,9 @@ def test_read_catalog(source): } ] } - expected = ConfiguredAirbyteCatalog.parse_obj(configured_catalog) + expected = ConfiguredAirbyteCatalogSerializer.load(configured_catalog) with tempfile.NamedTemporaryFile("w") as catalog_file: - catalog_file.write(expected.json(exclude_unset=True)) + catalog_file.write(orjson.dumps(ConfiguredAirbyteCatalogSerializer.dump(expected)).decode()) catalog_file.flush() actual = source.read_catalog(catalog_file.name) assert actual == expected diff --git a/airbyte-cdk/python/unit_tests/sources/test_source_read.py b/airbyte-cdk/python/unit_tests/sources/test_source_read.py index 00471ae86f82..05c71d1eae39 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source_read.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source_read.py @@ -309,11 +309,19 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e def _assert_status_messages(messages_from_abstract_source, messages_from_concurrent_source): - status_from_concurrent_source = [message for message in messages_from_concurrent_source if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS] + status_from_concurrent_source = [ + message + for message in messages_from_concurrent_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS + ] assert status_from_concurrent_source _verify_messages( - [message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS], + [ + message + for message in messages_from_abstract_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS + ], status_from_concurrent_source, ) @@ -329,8 +337,14 @@ def _assert_record_messages(messages_from_abstract_source, messages_from_concurr def _assert_errors(messages_from_abstract_source, messages_from_concurrent_source): - errors_from_concurrent_source = [message for message in messages_from_concurrent_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR] - errors_from_abstract_source = [message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR] + errors_from_concurrent_source = [ + message + for message in messages_from_concurrent_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR + ] + errors_from_abstract_source = [ + message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR + ] assert errors_from_concurrent_source # exceptions might differ from both framework hence we only assert the count @@ -352,7 +366,13 @@ def _init_sources(stream_slice_to_partitions, state, logger): def _init_source(stream_slice_to_partitions, state, logger, source): streams = [ - StreamFacade.create_from_stream(_MockStream(stream_slices, f"stream{i}"), source, logger, state, FinalStateCursor(stream_name=f"stream{i}", stream_namespace=None, message_repository=InMemoryMessageRepository())) + StreamFacade.create_from_stream( + _MockStream(stream_slices, f"stream{i}"), + source, + logger, + state, + FinalStateCursor(stream_name=f"stream{i}", stream_namespace=None, message_repository=InMemoryMessageRepository()), + ) for i, stream_slices in enumerate(stream_slice_to_partitions) ] source.set_streams(streams) diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py deleted file mode 100644 index 8f4862332ea8..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import AirbyteCatalog, AirbyteStream, SyncMode -from airbyte_cdk.sources.utils.catalog_helpers import CatalogHelper - - -def test_coerce_catalog_as_full_refresh(): - incremental = AirbyteStream( - name="1", - json_schema={"k": "v"}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], - source_defined_cursor=True, - default_cursor_field=["cursor"], - ) - full_refresh = AirbyteStream( - name="2", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh], source_defined_cursor=False - ) - input = AirbyteCatalog(streams=[incremental, full_refresh]) - - expected = AirbyteCatalog( - streams=[ - AirbyteStream(name="1", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh], source_defined_cursor=False), - full_refresh, - ] - ) - - assert CatalogHelper.coerce_catalog_as_full_refresh(input) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index 0b76f5eef5c2..76b7a9b1c772 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -14,7 +14,7 @@ import jsonref import pytest -from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification, FailureType +from airbyte_cdk.models import ConnectorSpecification, ConnectorSpecificationSerializer, FailureType from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest import fixture @@ -42,7 +42,7 @@ def create_schema(name: str, content: Mapping): @fixture -def spec_object(): +def spec_object() -> ConnectorSpecification: spec = { "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", @@ -54,7 +54,7 @@ def spec_object(): }, }, } - yield ConnectorSpecification.parse_obj(spec) + yield ConnectorSpecificationSerializer.load(spec) def test_check_config_against_spec_or_exit_does_not_print_schema(capsys, spec_object): diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py deleted file mode 100644 index 1ef6b23349e7..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import List, Optional - -from airbyte_cdk.sources.utils.schema_models import AllOptional, BaseSchemaModel - - -class InnerClass(BaseSchemaModel): - field1: Optional[str] - field2: int - - -class SchemaWithFewNullables(BaseSchemaModel): - name: Optional[str] - optional_item: Optional[InnerClass] - items: List[InnerClass] - - -class SchemaWithAllOptional(BaseSchemaModel, metaclass=AllOptional): - object_id: int - item: InnerClass - - -class TestSchemaWithFewNullables: - EXPECTED_SCHEMA = { - "type": "object", - "properties": { - "name": {"type": ["null", "string"]}, - "optional_item": { - "oneOf": [ - {"type": "null"}, - {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - ] - }, - "items": { - "type": "array", - "items": {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - }, - }, - } - - def test_schema_postprocessing(self): - schema = SchemaWithFewNullables.schema() - assert schema == self.EXPECTED_SCHEMA - - -class TestSchemaWithAllOptional: - EXPECTED_SCHEMA = { - "type": "object", - "properties": { - "object_id": {"type": ["null", "integer"]}, - "item": { - "oneOf": [ - {"type": "null"}, - {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - ] - }, - }, - } - - def test_schema_postprocessing(self): - schema = SchemaWithAllOptional.schema() - assert schema == self.EXPECTED_SCHEMA diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py index 328db535ca36..c8ccdc41b9bf 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py @@ -46,9 +46,7 @@ def _any_record_builder() -> RecordBuilder: def _response_builder( - response_template: Dict[str, Any], - records_path: Union[FieldPath, NestedPath], - pagination_strategy: Optional[PaginationStrategy] = None + response_template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy] = None ) -> HttpResponseBuilder: return create_response_builder(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy) @@ -64,7 +62,9 @@ def test_given_with_id_when_build_then_set_id(self) -> None: assert record[_ID_FIELD] == "another id" def test_given_nested_id_when_build_then_set_id(self) -> None: - builder = _record_builder({_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD])) + builder = _record_builder( + {_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD]) + ) record = builder.with_id("another id").build() assert record["nested"][_ID_FIELD] == "another id" @@ -79,9 +79,7 @@ def test_given_no_id_in_template_for_path_when_build_then_raise_error(self) -> N def test_given_with_cursor_when_build_then_set_id(self) -> None: builder = _record_builder( - {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, - FieldPath(_RECORDS_FIELD), - record_cursor_path=FieldPath(_CURSOR_FIELD) + {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, FieldPath(_RECORDS_FIELD), record_cursor_path=FieldPath(_CURSOR_FIELD) ) record = builder.with_cursor("another cursor").build() assert record[_CURSOR_FIELD] == "another cursor" @@ -90,7 +88,7 @@ def test_given_nested_cursor_when_build_then_set_cursor(self) -> None: builder = _record_builder( {_RECORDS_FIELD: [{"nested": {_CURSOR_FIELD: "a cursor"}}]}, FieldPath(_RECORDS_FIELD), - record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]), ) record = builder.with_cursor("another cursor").build() assert record["nested"][_CURSOR_FIELD] == "another cursor" @@ -115,7 +113,7 @@ def test_given_no_cursor_in_template_for_path_when_build_then_raise_error(self) _record_builder( {_RECORDS_FIELD: [{"record without cursor": "should fail"}]}, FieldPath(_RECORDS_FIELD), - record_cursor_path=FieldPath(_ID_FIELD) + record_cursor_path=FieldPath(_ID_FIELD), ) @@ -150,7 +148,7 @@ def test_given_pagination_with_strategy_when_build_then_apply_strategy(self) -> builder = _response_builder( {"has_more_pages": False} | _SOME_RECORDS, FieldPath(_RECORDS_FIELD), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page") + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page"), ) response = builder.with_pagination().build() diff --git a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py index 8e0bbe9fc93c..11dfc5877572 100644 --- a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py +++ b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py @@ -7,32 +7,40 @@ from unittest import TestCase from unittest.mock import Mock, patch -from airbyte_cdk.sources.abstract_source import AbstractSource -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, discover, read -from airbyte_cdk.test.state_builder import StateBuilder -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteAnalyticsTraceMessage, AirbyteCatalog, AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStreamState, + AirbyteStreamStateSerializer, AirbyteStreamStatus, AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, - ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, StreamDescriptor, TraceType, Type, ) +from airbyte_cdk.sources.abstract_source import AbstractSource +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, discover, read +from airbyte_cdk.test.state_builder import StateBuilder +from orjson import orjson def _a_state_message(stream_name: str, stream_state: Mapping[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(**stream_state)))) + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(**stream_state)) + ), + ) def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteMessage: @@ -77,7 +85,7 @@ def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteM _A_STREAM_NAME = "a stream name" _A_CONFIG = {"config_key": "config_value"} -_A_CATALOG = ConfiguredAirbyteCatalog.parse_obj( +_A_CATALOG = ConfiguredAirbyteCatalogSerializer.load( { "streams": [ { @@ -97,7 +105,7 @@ def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteM def _to_entrypoint_output(messages: List[AirbyteMessage]) -> Iterator[str]: - return (message.json(exclude_unset=True) for message in messages) + return (orjson.dumps(AirbyteMessageSerializer.dump(message)).decode() for message in messages) def _a_mocked_source() -> AbstractSource: @@ -112,7 +120,11 @@ def _validate_tmp_json_file(expected, file_path) -> None: def _validate_tmp_catalog(expected, file_path) -> None: - assert ConfiguredAirbyteCatalog.parse_file(file_path) == expected + assert ConfiguredAirbyteCatalogSerializer.load( + orjson.loads( + open(file_path).read() + ) + ) == expected def _create_tmp_file_validation(entrypoint, expected_config, expected_catalog: Optional[Any] = None, expected_state: Optional[Any] = None): @@ -176,19 +188,19 @@ def _do_some_logging(self): def test_given_record_when_discover_then_output_has_record(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_CATALOG_MESSAGE]) output = discover(self._a_source, _A_CONFIG) - assert output.catalog == _A_CATALOG_MESSAGE + assert AirbyteMessageSerializer.dump(output.catalog) == AirbyteMessageSerializer.dump(_A_CATALOG_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_log_when_discover_then_output_has_log(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) output = discover(self._a_source, _A_CONFIG) - assert output.logs == [_A_LOG] + assert AirbyteMessageSerializer.dump(output.logs[0]) == AirbyteMessageSerializer.dump(_A_LOG) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_trace_message_when_discover_then_output_has_trace_messages(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_AN_ANALYTIC_MESSAGE]) output = discover(self._a_source, _A_CONFIG) - assert output.analytics_messages == [_AN_ANALYTIC_MESSAGE] + assert AirbyteMessageSerializer.dump(output.analytics_messages[0]) == AirbyteMessageSerializer.dump(_AN_ANALYTIC_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.print", create=True) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") @@ -254,41 +266,45 @@ def _do_some_logging(self): def test_given_record_when_read_then_output_has_record(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.records == [_A_RECORD] + assert AirbyteMessageSerializer.dump(output.records[0]) == AirbyteMessageSerializer.dump(_A_RECORD) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_state_message_when_read_then_output_has_state_message(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.state_messages == [_A_STATE_MESSAGE] + assert AirbyteMessageSerializer.dump(output.state_messages[0]) == AirbyteMessageSerializer.dump(_A_STATE_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_state_message_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD, _A_STATE_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.records_and_state_messages == [_A_RECORD, _A_STATE_MESSAGE] + assert [AirbyteMessageSerializer.dump(message) for message in output.records_and_state_messages] == [ + AirbyteMessageSerializer.dump(message) for message in (_A_RECORD, _A_STATE_MESSAGE) + ] @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_many_state_messages_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): state_value = {"state_key": "last state value"} - last_emitted_state = AirbyteStreamState(stream_descriptor=StreamDescriptor(name="stream_name"), stream_state=AirbyteStateBlob(**state_value)) + last_emitted_state = AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="stream_name"), stream_state=AirbyteStateBlob(**state_value) + ) entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE, _a_state_message("stream_name", state_value)]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.most_recent_state == last_emitted_state + assert AirbyteStreamStateSerializer.dump(output.most_recent_state) == AirbyteStreamStateSerializer.dump(last_emitted_state) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_log_when_read_then_output_has_log(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.logs == [_A_LOG] + assert AirbyteMessageSerializer.dump(output.logs[0]) == AirbyteMessageSerializer.dump(_A_LOG) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_trace_message_when_read_then_output_has_trace_messages(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_AN_ANALYTIC_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.analytics_messages == [_AN_ANALYTIC_MESSAGE] + assert AirbyteMessageSerializer.dump(output.analytics_messages[0]) == AirbyteMessageSerializer.dump(_AN_ANALYTIC_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_stream_statuses_when_read_then_return_statuses(self, entrypoint): diff --git a/airbyte-cdk/python/unit_tests/test_connector.py b/airbyte-cdk/python/unit_tests/test_connector.py index 444397b4b0d6..ea7de2e40695 100644 --- a/airbyte-cdk/python/unit_tests/test_connector.py +++ b/airbyte-cdk/python/unit_tests/test_connector.py @@ -15,7 +15,6 @@ import yaml from airbyte_cdk import Connector from airbyte_cdk.models import AirbyteConnectionStatus -from pydantic import AnyUrl logger = logging.getLogger("airbyte") @@ -113,7 +112,7 @@ def use_yaml_spec(self): def test_spec_from_json_file(self, integration, use_json_spec): connector_spec = integration.spec(logger) - assert connector_spec.documentationUrl == AnyUrl("https://airbyte.com/#json") + assert connector_spec.documentationUrl == "https://airbyte.com/#json" assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION def test_spec_from_improperly_formatted_json_file(self, integration, use_invalid_json_spec): @@ -122,7 +121,7 @@ def test_spec_from_improperly_formatted_json_file(self, integration, use_invalid def test_spec_from_yaml_file(self, integration, use_yaml_spec): connector_spec = integration.spec(logger) - assert connector_spec.documentationUrl == AnyUrl("https://airbyte.com/#yaml") + assert connector_spec.documentationUrl == "https://airbyte.com/#yaml" assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION def test_multiple_spec_files_raises_exception(self, integration, use_yaml_spec, use_json_spec): diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index 1c5f8427bbb0..571042e202d1 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -20,9 +20,11 @@ AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateBlob, AirbyteStateMessage, + AirbyteStateStats, AirbyteStateType, AirbyteStream, AirbyteStreamState, @@ -37,10 +39,10 @@ TraceType, Type, ) -from airbyte_cdk.models.airbyte_protocol import AirbyteStateStats from airbyte_cdk.sources import Source from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor from airbyte_cdk.utils import AirbyteTracedException +from orjson import orjson class MockSource(Source): @@ -106,14 +108,14 @@ def test_airbyte_entrypoint_init(mocker): ("check", {"config": "config_path"}, {"command": "check", "config": "config_path", "debug": False}), ("discover", {"config": "config_path", "debug": ""}, {"command": "discover", "config": "config_path", "debug": True}), ( - "read", - {"config": "config_path", "catalog": "catalog_path", "state": "None"}, - {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "None", "debug": False}, + "read", + {"config": "config_path", "catalog": "catalog_path", "state": "None"}, + {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "None", "debug": False}, ), ( - "read", - {"config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": ""}, - {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": True}, + "read", + {"config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": ""}, + {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": True}, ), ], ) @@ -152,7 +154,7 @@ def _wrap_message(submessage: Union[AirbyteConnectionStatus, ConnectorSpecificat else: raise Exception(f"Unknown message type: {submessage}") - return message.json(exclude_unset=True) + return orjson.dumps(AirbyteMessageSerializer.dump(message)).decode() def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): @@ -162,7 +164,7 @@ def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages @pytest.fixture @@ -181,9 +183,9 @@ def config_mock(mocker, request): ({"username": "fake"}, {"type": "object", "properties": {"user": {"type": "string"}}}, True), ({"username": "fake"}, {"type": "object", "properties": {"user": {"type": "string", "airbyte_secret": True}}}, True), ( - {"username": "fake", "_limit": 22}, - {"type": "object", "properties": {"username": {"type": "string"}}, "additionalProperties": False}, - True, + {"username": "fake", "_limit": 22}, + {"type": "object", "properties": {"username": {"type": "string"}}, "additionalProperties": False}, + True, ), ], indirect=["config_mock"], @@ -196,14 +198,14 @@ def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, sch messages = list(entrypoint.run(parsed_args)) if config_valid: - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(check_value)] == messages else: assert len(messages) == 2 - assert messages[0] == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) - connection_status_message = AirbyteMessage.parse_raw(messages[1]) - assert connection_status_message.type == Type.CONNECTION_STATUS - assert connection_status_message.connectionStatus.status == Status.FAILED - assert connection_status_message.connectionStatus.message.startswith("Config validation error:") + assert messages[0] == orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode() + connection_status_message = AirbyteMessage(**orjson.loads(messages[1])) + assert connection_status_message.type == Type.CONNECTION_STATUS.value + assert connection_status_message.connectionStatus.get("status") == Status.FAILED.value + assert connection_status_message.connectionStatus.get("message").startswith("Config validation error:") def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -213,7 +215,7 @@ def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(check_value)] == messages assert spec_mock.called @@ -223,7 +225,7 @@ def test_run_check_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mo with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -233,7 +235,7 @@ def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_m messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages assert spec_mock.called @@ -243,7 +245,7 @@ def test_run_discover_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -255,18 +257,18 @@ def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock) messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages assert spec_mock.called def test_given_message_emitted_during_config_when_read_then_emit_message_before_next_steps( - entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock + entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock ): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") mocker.patch.object(MockSource, "read_catalog", side_effect=ValueError) messages = entrypoint.run(parsed_args) - assert next(messages) == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) + assert next(messages) == orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode() with pytest.raises(ValueError): next(messages) @@ -279,7 +281,7 @@ def test_run_read_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_moc with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): @@ -334,12 +336,26 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_record_message", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")), - sourceStats=AirbyteStateStats(recordCount=100.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + sourceStats=AirbyteStateStats(recordCount=100.0), + ), + ), {HashableStreamDescriptor(name="customers"): 0.0}, id="test_handle_state_message", ), @@ -351,15 +367,27 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_first_record_message", ), pytest.param( - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.STREAM_STATUS, - stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name="customers"), - status=AirbyteStreamStatus.COMPLETE), emitted_at=1)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="customers"), status=AirbyteStreamStatus.COMPLETE + ), + emitted_at=1, + ), + ), {HashableStreamDescriptor(name="customers"): 5.0}, - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.STREAM_STATUS, - stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name="customers"), - status=AirbyteStreamStatus.COMPLETE), emitted_at=1)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="customers"), status=AirbyteStreamStatus.COMPLETE + ), + emitted_at=1, + ), + ), {HashableStreamDescriptor(name="customers"): 5.0}, id="test_handle_other_message_type", ), @@ -371,48 +399,96 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_record_message_for_other_stream", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0, HashableStreamDescriptor(name="others"): 27.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")), - sourceStats=AirbyteStateStats(recordCount=27.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + sourceStats=AirbyteStateStats(recordCount=27.0), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0, HashableStreamDescriptor(name="others"): 0.0}, id="test_handle_state_message_for_other_stream", ), pytest.param( - AirbyteMessage(type=Type.RECORD, - record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1)), + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1) + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.RECORD, - record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1)), + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1) + ), {HashableStreamDescriptor(name="customers", namespace="public"): 101.0}, id="test_handle_record_message_with_descriptor", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")), sourceStats=AirbyteStateStats(recordCount=100.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + sourceStats=AirbyteStateStats(recordCount=100.0), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 0.0}, id="test_handle_state_message_with_descriptor", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")), sourceStats=AirbyteStateStats(recordCount=0.0))), - {HashableStreamDescriptor(name="customers", namespace="public"): 100.0, - HashableStreamDescriptor(name="others", namespace="public"): 0.0}, + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + sourceStats=AirbyteStateStats(recordCount=0.0), + ), + ), + { + HashableStreamDescriptor(name="customers", namespace="public"): 100.0, + HashableStreamDescriptor(name="others", namespace="public"): 0.0, + }, id="test_handle_state_message_no_records", ), - ] + ], ) def test_handle_record_counts(incoming_message, stream_message_count, expected_message, expected_records_by_stream): entrypoint = AirbyteEntrypoint(source=MockSource()) diff --git a/airbyte-cdk/python/unit_tests/test_exception_handler.py b/airbyte-cdk/python/unit_tests/test_exception_handler.py index 42819942ade1..f135c19fd5a9 100644 --- a/airbyte-cdk/python/unit_tests/test_exception_handler.py +++ b/airbyte-cdk/python/unit_tests/test_exception_handler.py @@ -9,7 +9,17 @@ import pytest from airbyte_cdk.exception_handler import assemble_uncaught_exception -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage +from airbyte_cdk.models import ( + AirbyteErrorTraceMessage, + AirbyteLogMessage, + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteTraceMessage, + FailureType, + Level, + TraceType, +) +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -43,16 +53,16 @@ def test_uncaught_exception_handler(): ) expected_log_message = AirbyteMessage( - type="LOG", log=AirbyteLogMessage(level="FATAL", message=f"{exception_message}\n{exception_trace}") + type=MessageType.LOG, log=AirbyteLogMessage(level=Level.FATAL, message=f"{exception_message}\n{exception_trace}") ) expected_trace_message = AirbyteMessage( - type="TRACE", + type=MessageType.TRACE, trace=AirbyteTraceMessage( - type="ERROR", + type=TraceType.ERROR, emitted_at=0.0, error=AirbyteErrorTraceMessage( - failure_type="system_error", + failure_type=FailureType.system_error, message="Something went wrong in the connector. See the logs for more details.", internal_message=exception_message, stack_trace=f"{exception_trace}\n", @@ -70,10 +80,10 @@ def test_uncaught_exception_handler(): log_output, trace_output = stdout_lines - out_log_message = AirbyteMessage.parse_obj(json.loads(log_output)) + out_log_message = AirbyteMessageSerializer.load(json.loads(log_output)) assert out_log_message == expected_log_message, "Log message should be emitted in expected form" - out_trace_message = AirbyteMessage.parse_obj(json.loads(trace_output)) + out_trace_message = AirbyteMessageSerializer.load(json.loads(trace_output)) assert out_trace_message.trace.emitted_at > 0 out_trace_message.trace.emitted_at = 0.0 # set a specific emitted_at value for testing assert out_trace_message == expected_trace_message, "Trace message should be emitted in expected form" diff --git a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py index 766007467184..5e76b9cfa193 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py +++ b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py @@ -5,7 +5,7 @@ from typing import Dict, List import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer NOW = 1234567 diff --git a/airbyte-cdk/python/unit_tests/utils/test_message_utils.py b/airbyte-cdk/python/unit_tests/utils/test_message_utils.py index 496360ea46f3..84fabf1a8fa8 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_message_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_message_utils.py @@ -1,9 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import pytest -from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor -from airbyte_cdk.utils.message_utils import get_stream_descriptor -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, @@ -17,6 +15,8 @@ StreamDescriptor, Type, ) +from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor +from airbyte_cdk.utils.message_utils import get_stream_descriptor def test_get_record_message_stream_descriptor(): @@ -36,9 +36,7 @@ def test_get_record_message_stream_descriptor(): def test_get_record_message_stream_descriptor_no_namespace(): message = AirbyteMessage( type=Type.RECORD, - record=AirbyteRecordMessage( - stream="test_stream", data={"id": "12345"}, emitted_at=1 - ), + record=AirbyteRecordMessage(stream="test_stream", data={"id": "12345"}, emitted_at=1), ) expected_descriptor = HashableStreamDescriptor(name="test_stream", namespace=None) assert get_stream_descriptor(message) == expected_descriptor @@ -50,9 +48,7 @@ def test_get_state_message_stream_descriptor(): state=AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor( - name="test_stream", namespace="test_namespace" - ), + stream_descriptor=StreamDescriptor(name="test_stream", namespace="test_namespace"), stream_state=AirbyteStateBlob(updated_at="2024-02-02"), ), sourceStats=AirbyteStateStats(recordCount=27.0), diff --git a/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py index 51a055401a29..98d227c40ec6 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py +++ b/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py @@ -5,7 +5,7 @@ from typing import List, Mapping import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException NOW = 1234567 @@ -133,7 +133,10 @@ { "my_stream": { "field_A": {"type": ["string", "null"]}, - "nested": {"type": ["array", "null"], "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}}, + "nested": { + "type": ["array", "null"], + "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}, + }, } }, id="test_array_nested_null", @@ -146,7 +149,10 @@ { "my_stream": { "field_A": {"type": ["string", "null"]}, - "nested": {"type": ["array", "null"], "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}}, + "nested": { + "type": ["array", "null"], + "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}, + }, } }, id="test_array_top_level_null", @@ -166,80 +172,42 @@ "data": { "root_property_object": { "property_array": [ - { - "title": "Nested_1", - "type": "multi-value", - "value": ["XL"] - }, + {"title": "Nested_1", "type": "multi-value", "value": ["XL"]}, { "title": "Nested_2", "type": "location", - "value": { - "nested_key_1": "GB", - "nested_key_2": "United Kingdom" - } - } + "value": {"nested_key_1": "GB", "nested_key_2": "United Kingdom"}, + }, ], } - } + }, }, ], { "data_with_nested_arrays": { "root_property_object": { - "type": [ - "object", - "null" - ], + "type": ["object", "null"], "properties": { "property_array": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { - "type": [ - "object", - "null" - ], + "type": ["object", "null"], "properties": { - "title": { - "type": [ - "string", - "null" - ] - }, - "type": { - "type": [ - "string", - "null" - ] - }, + "title": {"type": ["string", "null"]}, + "type": {"type": ["string", "null"]}, "value": { "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, + {"type": "array", "items": {"type": "string"}}, { "type": "object", - "properties": { - "nested_key_1": { - "type": "string" - }, - "nested_key_2": { - "type": "string" - } - } - } + "properties": {"nested_key_1": {"type": "string"}, "nested_key_2": {"type": "string"}}, + }, ] - } - } - } + }, + }, + }, } - } + }, } } }, @@ -277,7 +245,7 @@ def _create_inferrer_with_required_field(is_pk: bool, field: List[List[str]]) -> [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_on_root(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property"]]) @@ -293,7 +261,7 @@ def test_field_is_on_root(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_nested(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property", "nested_property"]]) @@ -310,11 +278,13 @@ def test_field_is_nested(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_composite(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property 1"], ["property 2"]]) - inferrer.accumulate(AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": _ANY_VALUE, "property 2": _ANY_VALUE}, emitted_at=NOW)) + inferrer.accumulate( + AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": _ANY_VALUE, "property 2": _ANY_VALUE}, emitted_at=NOW) + ) assert inferrer.get_stream_schema(_STREAM_NAME)["required"] == ["property 1", "property 2"] @@ -323,12 +293,14 @@ def test_field_is_composite(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_composite_and_nested(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property 1", "nested"], ["property 2"]]) - inferrer.accumulate(AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": {"nested": _ANY_VALUE}, "property 2": _ANY_VALUE}, emitted_at=NOW)) + inferrer.accumulate( + AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": {"nested": _ANY_VALUE}, "property 2": _ANY_VALUE}, emitted_at=NOW) + ) assert inferrer.get_stream_schema(_STREAM_NAME)["required"] == ["property 1", "property 2"] assert inferrer.get_stream_schema(_STREAM_NAME)["properties"]["property 1"]["type"] == "object" diff --git a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py index e0d3b9a50353..ea559a319467 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py +++ b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py @@ -2,20 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json import pytest -from airbyte_cdk.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteErrorTraceMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteTraceMessage, FailureType, Status, + StreamDescriptor, TraceType, ) -from airbyte_cdk.models.airbyte_protocol import Type as MessageType +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import StreamDescriptor +from orjson import orjson _AN_EXCEPTION = ValueError("An exception") _A_STREAM_DESCRIPTOR = StreamDescriptor(name="a_stream") @@ -90,12 +91,12 @@ def test_emit_message(capsys): ) expected_message = AirbyteMessage( - type="TRACE", + type=MessageType.TRACE, trace=AirbyteTraceMessage( - type="ERROR", + type=TraceType.ERROR, emitted_at=0.0, error=AirbyteErrorTraceMessage( - failure_type="system_error", + failure_type=FailureType.system_error, message="user-friendly message", internal_message="internal message", stack_trace="RuntimeError: oh no\n", @@ -106,9 +107,8 @@ def test_emit_message(capsys): traced_exc.emit_message() stdout = capsys.readouterr().out - printed_message = AirbyteMessage.parse_obj(json.loads(stdout)) + printed_message = AirbyteMessageSerializer.load(orjson.loads(stdout)) printed_message.trace.emitted_at = 0.0 - assert printed_message == expected_message