From 7f1557b348b7935e3586c90c8dec15fdf6cd8665 Mon Sep 17 00:00:00 2001 From: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> Date: Wed, 3 Apr 2024 20:19:24 +0200 Subject: [PATCH] feat: Lint with ruff (#4043) * feat: Lin with ruff Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * fea: replace black with ruff format options Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * fea: files reformatted due to ruff deviations from black Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * fix: restored mypy in lint-python target Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * fix: fixed formatting error Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * adding ruff format --check Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * Formatting updated files Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> * Adding section separator Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> --------- Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> --- Makefile | 13 ++---- docs/project/development-guide.md | 4 +- pyproject.toml | 44 +++++++++---------- sdk/python/feast/__init__.py | 3 +- sdk/python/feast/data_source.py | 1 - sdk/python/feast/diff/registry_diff.py | 12 ++--- sdk/python/feast/dqm/profilers/profiler.py | 12 ++--- .../embedded_go/online_features_service.py | 1 - sdk/python/feast/feature_logging.py | 18 ++++---- sdk/python/feast/feature_store.py | 28 +++++++----- .../infra/materialization/aws_lambda/app.py | 1 - .../batch_materialization_engine.py | 15 +++---- .../bytewax/bytewax_materialization_engine.py | 3 +- .../spark/spark_materialization_engine.py | 2 - .../infra/materialization/snowflake_engine.py | 8 ++-- .../feast/infra/offline_stores/bigquery.py | 6 +-- .../contrib/athena_offline_store/athena.py | 3 -- .../athena_offline_store/athena_source.py | 1 - .../athena_offline_store/tests/data_source.py | 2 - .../contrib/ibis_offline_store/ibis.py | 6 +-- .../postgres_offline_store/postgres.py | 10 +++-- .../postgres_offline_store/postgres_source.py | 1 - .../contrib/spark_offline_store/spark.py | 2 - .../trino_offline_store/connectors/upload.py | 1 + .../trino_offline_store/tests/data_source.py | 1 - .../trino_offline_store/trino_source.py | 1 - sdk/python/feast/infra/offline_stores/file.py | 31 +++++++------ .../infra/offline_stores/redshift_source.py | 6 +-- .../feast/infra/offline_stores/snowflake.py | 9 ---- .../infra/offline_stores/snowflake_source.py | 1 - .../hazelcast_online_store.py | 2 +- .../contrib/hbase_online_store/hbase.py | 6 +-- .../contrib/mysql_online_store/mysql.py | 3 -- .../feast/infra/online_stores/datastore.py | 2 - .../feast/infra/online_stores/sqlite.py | 7 ++- .../feast/infra/registry/base_registry.py | 19 ++++---- .../infra/utils/snowflake/snowflake_utils.py | 2 - sdk/python/feast/on_demand_feature_view.py | 8 ++-- sdk/python/feast/repo_operations.py | 1 - .../athena/feature_repo/test_workflow.py | 2 - .../feast/templates/snowflake/bootstrap.py | 1 - sdk/python/feast/utils.py | 6 +-- sdk/python/pyproject.toml | 15 +++++++ .../requirements/py3.10-ci-requirements.txt | 25 +++-------- .../requirements/py3.9-ci-requirements.txt | 26 +++-------- sdk/python/setup.cfg | 22 ---------- .../feature_repos/repo_configuration.py | 6 +-- .../universal/data_sources/bigquery.py | 2 - .../universal/data_sources/file.py | 2 - .../universal/data_sources/redshift.py | 2 - .../universal/data_sources/snowflake.py | 2 - .../universal/online_store/hazelcast.py | 1 - .../contrib/spark/test_spark.py | 1 - .../integration/registration/test_registry.py | 1 - .../registration/test_universal_types.py | 2 +- sdk/python/tests/unit/cli/test_cli.py | 2 - .../offline_stores/test_offline_store.py | 1 - .../infra/scaffolding/test_repo_config.py | 1 - .../unit/infra/test_inference_unit_tests.py | 2 +- .../online_store/test_online_retrieval.py | 2 +- sdk/python/tests/unit/test_type_map.py | 1 - sdk/python/tests/utils/e2e_test_validation.py | 1 - sdk/python/tests/utils/feature_records.py | 3 +- setup.cfg | 23 ---------- setup.py | 4 +- 65 files changed, 162 insertions(+), 290 deletions(-) create mode 100644 sdk/python/pyproject.toml delete mode 100644 sdk/python/setup.cfg diff --git a/Makefile b/Makefile index d232d9c93f..26e79cf6a9 100644 --- a/Makefile +++ b/Makefile @@ -294,18 +294,13 @@ test-python-universal: python -m pytest -n 8 --integration sdk/python/tests format-python: - # Sort - cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ - - # Format - cd ${ROOT_DIR}/sdk/python; python -m black --target-version py38 feast tests + cd ${ROOT_DIR}/sdk/python; python -m ruff check --fix feast/ tests/ + cd ${ROOT_DIR}/sdk/python; python -m ruff format feast/ tests/ lint-python: cd ${ROOT_DIR}/sdk/python; python -m mypy feast - cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ --check-only - cd ${ROOT_DIR}/sdk/python; python -m flake8 feast/ tests/ - cd ${ROOT_DIR}/sdk/python; python -m black --check feast tests - + cd ${ROOT_DIR}/sdk/python; python -m ruff check feast/ tests/ + cd ${ROOT_DIR}/sdk/python; python -m ruff format --check feast/ tests # Java install-java-ci-dependencies: diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 43dae1d678..28baa789bb 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -168,8 +168,8 @@ docker build -t docker-whale -f ./sdk/python/feast/infra/feature_servers/multicl Feast Python SDK / CLI codebase: - Conforms to [Black code style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) - Has type annotations as enforced by `mypy` -- Has imports sorted by `isort` -- Is lintable by `flake8` +- Has imports sorted by `ruff` (see [isort (I) rules](https://docs.astral.sh/ruff/rules/#isort-i)) +- Is lintable by `ruff` To ensure your Python code conforms to Feast Python code standards: - Autoformat your code to conform to the code style: diff --git a/pyproject.toml b/pyproject.toml index bfe2bc9fd0..00170ab443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,27 +5,25 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] # Including this section is comparable to supplying use_scm_version=True in setup.py. -[tool.black] +[tool.ruff] line-length = 88 -target-version = ['py39'] -include = '\.pyi?$' -exclude = ''' -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | pb2.py - | \.pyi - | protos - | sdk/python/feast/embedded_go/lib - )/ -) -''' +target-version = "py39" +include = ["*.py", "*.pyi"] + +[tool.ruff.format] +# exclude a few common directories in the root of the project +exclude = [ + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "pb2.py", + ".pyi", + "protos", + "sdk/python/feast/embedded_go/lib"] diff --git a/sdk/python/feast/__init__.py b/sdk/python/feast/__init__.py index 3eff91d65f..f51eb2983c 100644 --- a/sdk/python/feast/__init__.py +++ b/sdk/python/feast/__init__.py @@ -2,7 +2,8 @@ from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _version except ModuleNotFoundError: - from importlib_metadata import PackageNotFoundError, version as _version # type: ignore + from importlib_metadata import PackageNotFoundError # type: ignore + from importlib_metadata import version as _version from feast.infra.offline_stores.bigquery_source import BigQuerySource from feast.infra.offline_stores.contrib.athena_offline_store.athena_source import ( diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 3421fd5d30..17fbfd5fcf 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -576,7 +576,6 @@ def from_proto(data_source: DataSourceProto): ) def to_proto(self) -> DataSourceProto: - schema_pb = [] if isinstance(self.schema, Dict): diff --git a/sdk/python/feast/diff/registry_diff.py b/sdk/python/feast/diff/registry_diff.py index 106d34bf48..b608757496 100644 --- a/sdk/python/feast/diff/registry_diff.py +++ b/sdk/python/feast/diff/registry_diff.py @@ -216,12 +216,12 @@ def extract_objects_for_keep_delete_update_add( objs_to_update = {} objs_to_add = {} - registry_object_type_to_objects: Dict[ - FeastObjectType, List[Any] - ] = FeastObjectType.get_objects_from_registry(registry, current_project) - registry_object_type_to_repo_contents: Dict[ - FeastObjectType, List[Any] - ] = FeastObjectType.get_objects_from_repo_contents(desired_repo_contents) + registry_object_type_to_objects: Dict[FeastObjectType, List[Any]] = ( + FeastObjectType.get_objects_from_registry(registry, current_project) + ) + registry_object_type_to_repo_contents: Dict[FeastObjectType, List[Any]] = ( + FeastObjectType.get_objects_from_repo_contents(desired_repo_contents) + ) for object_type in FEAST_OBJECT_TYPES: ( diff --git a/sdk/python/feast/dqm/profilers/profiler.py b/sdk/python/feast/dqm/profilers/profiler.py index 34496b0cca..03481bdc99 100644 --- a/sdk/python/feast/dqm/profilers/profiler.py +++ b/sdk/python/feast/dqm/profilers/profiler.py @@ -15,13 +15,11 @@ def validate(self, dataset: pd.DataFrame) -> "ValidationReport": ... @abc.abstractmethod - def to_proto(self): - ... + def to_proto(self): ... @classmethod @abc.abstractmethod - def from_proto(cls, proto) -> "Profile": - ... + def from_proto(cls, proto) -> "Profile": ... class Profiler: @@ -34,13 +32,11 @@ def analyze_dataset(self, dataset: pd.DataFrame) -> Profile: ... @abc.abstractmethod - def to_proto(self): - ... + def to_proto(self): ... @classmethod @abc.abstractmethod - def from_proto(cls, proto) -> "Profiler": - ... + def from_proto(cls, proto) -> "Profiler": ... class ValidationReport: diff --git a/sdk/python/feast/embedded_go/online_features_service.py b/sdk/python/feast/embedded_go/online_features_service.py index c6430b5f6d..56427f61e6 100644 --- a/sdk/python/feast/embedded_go/online_features_service.py +++ b/sdk/python/feast/embedded_go/online_features_service.py @@ -65,7 +65,6 @@ def get_online_features( request_data: Dict[str, Union[List[Any], Value_pb2.RepeatedValue]], full_feature_names: bool = False, ): - if feature_service: join_keys_types = self._service.GetEntityTypesMapByFeatureService( feature_service.name diff --git a/sdk/python/feast/feature_logging.py b/sdk/python/feast/feature_logging.py index bd45c09b0a..2843f87121 100644 --- a/sdk/python/feast/feature_logging.py +++ b/sdk/python/feast/feature_logging.py @@ -86,15 +86,15 @@ def get_schema(self, registry: "BaseRegistry") -> pa.Schema: fields[join_key] = FEAST_TYPE_TO_ARROW_TYPE[entity_column.dtype] for feature in projection.features: - fields[ - f"{projection.name_to_use()}__{feature.name}" - ] = FEAST_TYPE_TO_ARROW_TYPE[feature.dtype] - fields[ - f"{projection.name_to_use()}__{feature.name}__timestamp" - ] = PA_TIMESTAMP_TYPE - fields[ - f"{projection.name_to_use()}__{feature.name}__status" - ] = pa.int32() + fields[f"{projection.name_to_use()}__{feature.name}"] = ( + FEAST_TYPE_TO_ARROW_TYPE[feature.dtype] + ) + fields[f"{projection.name_to_use()}__{feature.name}__timestamp"] = ( + PA_TIMESTAMP_TYPE + ) + fields[f"{projection.name_to_use()}__{feature.name}__status"] = ( + pa.int32() + ) # system columns fields[LOG_TIMESTAMP_FIELD] = pa.timestamp("us", tz=UTC) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 83aaafd686..62ce9d6e38 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -818,7 +818,8 @@ def apply( views_to_update = [ ob for ob in objects - if ( + if + ( # BFVs are not handled separately from FVs right now. (isinstance(ob, FeatureView) or isinstance(ob, BatchFeatureView)) and not isinstance(ob, StreamFeatureView) @@ -950,7 +951,9 @@ def apply( validation_references.name, project=self.project, commit=False ) - tables_to_delete: List[FeatureView] = views_to_delete + sfvs_to_delete if not partial else [] # type: ignore + tables_to_delete: List[FeatureView] = ( + views_to_delete + sfvs_to_delete if not partial else [] # type: ignore + ) tables_to_keep: List[FeatureView] = views_to_update + sfvs_to_update # type: ignore self._get_provider().update_infra( @@ -1575,7 +1578,10 @@ def _get_online_features( num_rows = _validate_entity_values(entity_proto_values) _validate_feature_refs(_feature_refs, full_feature_names) - (grouped_refs, grouped_odfv_refs,) = _group_feature_refs( + ( + grouped_refs, + grouped_odfv_refs, + ) = _group_feature_refs( _feature_refs, requested_feature_views, requested_on_demand_feature_views, @@ -1728,9 +1734,9 @@ def _get_entity_maps( ) entity_name_to_join_key_map[entity_name] = join_key for entity_column in feature_view.entity_columns: - entity_type_map[ - entity_column.name - ] = entity_column.dtype.to_value_type() + entity_type_map[entity_column.name] = ( + entity_column.dtype.to_value_type() + ) return ( entity_name_to_join_key_map, @@ -2005,11 +2011,11 @@ def _augment_response_with_on_demand_transforms( if odfv.mode == "python": if initial_response_dict is None: initial_response_dict = initial_response.to_dict() - transformed_features_dict: Dict[ - str, List[Any] - ] = odfv.get_transformed_features( - initial_response_dict, - full_feature_names, + transformed_features_dict: Dict[str, List[Any]] = ( + odfv.get_transformed_features( + initial_response_dict, + full_feature_names, + ) ) elif odfv.mode in {"pandas", "substrait"}: if initial_response_df is None: diff --git a/sdk/python/feast/infra/materialization/aws_lambda/app.py b/sdk/python/feast/infra/materialization/aws_lambda/app.py index 375674adaa..2bf65542e5 100644 --- a/sdk/python/feast/infra/materialization/aws_lambda/app.py +++ b/sdk/python/feast/infra/materialization/aws_lambda/app.py @@ -23,7 +23,6 @@ def handler(event, context): print("Received event: " + json.dumps(event, indent=2), flush=True) try: - config_base64 = event[FEATURE_STORE_YAML_ENV_NAME] config_bytes = base64.b64decode(config_base64) diff --git a/sdk/python/feast/infra/materialization/batch_materialization_engine.py b/sdk/python/feast/infra/materialization/batch_materialization_engine.py index 41ab9f22d4..8e854a508d 100644 --- a/sdk/python/feast/infra/materialization/batch_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/batch_materialization_engine.py @@ -49,24 +49,19 @@ class MaterializationJob(ABC): task: MaterializationTask @abstractmethod - def status(self) -> MaterializationJobStatus: - ... + def status(self) -> MaterializationJobStatus: ... @abstractmethod - def error(self) -> Optional[BaseException]: - ... + def error(self) -> Optional[BaseException]: ... @abstractmethod - def should_be_retried(self) -> bool: - ... + def should_be_retried(self) -> bool: ... @abstractmethod - def job_id(self) -> str: - ... + def job_id(self) -> str: ... @abstractmethod - def url(self) -> Optional[str]: - ... + def url(self) -> Optional[str]: ... class BatchMaterializationEngine(ABC): diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index d82e0920e2..3ad6fe4b55 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -5,9 +5,8 @@ from typing import Callable, List, Literal, Sequence, Union import yaml -from kubernetes import client +from kubernetes import client, utils from kubernetes import config as k8s_config -from kubernetes import utils from kubernetes.client.exceptions import ApiException from kubernetes.utils import FailToCreateError from pydantic import StrictStr diff --git a/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py index 798d3a8e6f..24608baebf 100644 --- a/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py @@ -3,7 +3,6 @@ from typing import Callable, List, Literal, Optional, Sequence, Union, cast import dill -import pandas import pandas as pd import pyarrow from tqdm import tqdm @@ -201,7 +200,6 @@ class _SparkSerializedArtifacts: @classmethod def serialize(cls, feature_view, repo_config): - # serialize to proto feature_view_proto = feature_view.to_proto().SerializeToString() diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 28bec198a5..4a81982dcd 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -169,7 +169,6 @@ def teardown_infra( fvs: Sequence[Union[BatchFeatureView, StreamFeatureView, FeatureView]], entities: Sequence[Entity], ): - stage_path = f'"{self.repo_config.batch_engine.database}"."{self.repo_config.batch_engine.schema_}"."feast_{project}"' with GetSnowflakeConnection(self.repo_config.batch_engine) as conn: query = f"DROP STAGE IF EXISTS {stage_path}" @@ -230,8 +229,9 @@ def _materialize_one( project: str, tqdm_builder: Callable[[int], tqdm], ): - assert isinstance(feature_view, BatchFeatureView) or isinstance( - feature_view, FeatureView + assert ( + isinstance(feature_view, BatchFeatureView) + or isinstance(feature_view, FeatureView) ), "Snowflake can only materialize FeatureView & BatchFeatureView feature view types." entities = [] @@ -350,7 +350,6 @@ def generate_snowflake_materialization_query( feature_batch: list, project: str, ) -> str: - if feature_view.batch_source.created_timestamp_column: fv_created_str = f',"{feature_view.batch_source.created_timestamp_column}"' else: @@ -477,7 +476,6 @@ def materialize_to_external_online_store( feature_view: Union[StreamFeatureView, FeatureView], pbar: tqdm, ) -> None: - feature_names = [feature.name for feature in feature_view.features] with GetSnowflakeConnection(repo_config.batch_engine) as conn: diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 68420c0664..897647bfc2 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -95,9 +95,9 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" - table_create_disposition: Literal[ - "CREATE_NEVER", "CREATE_IF_NEEDED" - ] = "CREATE_IF_NEEDED" + table_create_disposition: Literal["CREATE_NEVER", "CREATE_IF_NEEDED"] = ( + "CREATE_IF_NEEDED" + ) """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED. Custom constraint for table_create_disposition. To understand more, see: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py index ae510171db..43960d87d5 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py @@ -205,7 +205,6 @@ def get_historical_features( @contextlib.contextmanager def query_generator() -> Iterator[str]: - table_name = offline_utils.get_temp_entity_table_name() _upload_entity_df(entity_df, athena_client, config, s3_resource, table_name) @@ -240,7 +239,6 @@ def query_generator() -> Iterator[str]: try: yield query finally: - # Always clean up the temp Athena table aws_utils.execute_athena_query( athena_client, @@ -423,7 +421,6 @@ def persist( @log_exceptions_and_usage def to_athena(self, table_name: str) -> None: - if self.on_demand_feature_views: transformed_df = self.to_df() diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py index 0aca42cd68..509d707935 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py @@ -307,7 +307,6 @@ def __init__( @staticmethod def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage: - return SavedDatasetAthenaStorage( table_ref=AthenaOptions.from_proto(storage_proto.athena_storage).table ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index 6b2238830b..f01144afcc 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -22,7 +22,6 @@ class AthenaDataSourceCreator(DataSourceCreator): - tables: List[str] = [] def __init__(self, project_name: str, *args, **kwargs): @@ -53,7 +52,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - table_name = destination_name s3_target = ( self.offline_store_config.s3_staging_location diff --git a/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py b/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py index 8787d70158..37fc6a4718 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py +++ b/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py @@ -299,9 +299,9 @@ def __init__( ) -> None: super().__init__() self.table = table - self._on_demand_feature_views: List[ - OnDemandFeatureView - ] = on_demand_feature_views + self._on_demand_feature_views: List[OnDemandFeatureView] = ( + on_demand_feature_views + ) self._full_feature_names = full_feature_names self._metadata = metadata diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index 17a8e20f78..1bf10202e1 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -160,7 +160,7 @@ def query_generator() -> Iterator[str]: # Hack for query_context.entity_selections to support uppercase in columns for context in query_context_dict: context["entity_selections"] = [ - f'''"{entity_selection.replace(' AS ', '" AS "')}\"''' + f""""{entity_selection.replace(' AS ', '" AS "')}\"""" for entity_selection in context["entity_selections"] ] @@ -338,9 +338,11 @@ def _get_entity_df_event_timestamp_range( # If the entity_df is a string (SQL query), determine range # from table with _get_conn(config.offline_store) as conn, conn.cursor() as cur: - cur.execute( - f"SELECT MIN({entity_df_event_timestamp_col}) AS min, MAX({entity_df_event_timestamp_col}) AS max FROM ({entity_df}) as tmp_alias" - ), + ( + cur.execute( + f"SELECT MIN({entity_df_event_timestamp_col}) AS min, MAX({entity_df_event_timestamp_col}) AS max FROM ({entity_df}) as tmp_alias" + ), + ) res = cur.fetchone() entity_df_event_timestamp_range = (res[0], res[1]) else: diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py index bc535ed194..bbb3f768fd 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py @@ -117,7 +117,6 @@ def get_table_column_names_and_types( ) def get_table_query_string(self) -> str: - if self._postgres_options._table: return f"{self._postgres_options._table}" else: diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index b1b1c04c7d..43902f33cf 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -389,7 +389,6 @@ def supports_remote_storage_export(self) -> bool: def to_remote_storage(self) -> List[str]: """Currently only works for local and s3-based staging locations""" if self.supports_remote_storage_export(): - sdf: pyspark.sql.DataFrame = self.to_spark_df() if self._config.offline_store.staging_location.startswith("/"): @@ -405,7 +404,6 @@ def to_remote_storage(self) -> List[str]: return _list_files_in_folder(output_uri) elif self._config.offline_store.staging_location.startswith("s3://"): - spark_compatible_s3_staging_location = ( self._config.offline_store.staging_location.replace( "s3://", "s3a://" diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py index 5967b7a863..9e2ea3708d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py @@ -17,6 +17,7 @@ file_format: parquet ``` """ + from datetime import datetime from typing import Any, Dict, Iterator, Optional, Set diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index bd3f9def8f..0dee517eb3 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -46,7 +46,6 @@ def trino_container(): class TrinoSourceCreator(DataSourceCreator): - tables: List[str] = [] def __init__( diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py index e618e8664e..73d40d902e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py @@ -182,7 +182,6 @@ def trino_options(self, trino_options): @staticmethod def from_proto(data_source: DataSourceProto): - assert data_source.HasField("trino_options") return TrinoSource( diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index d922e98c14..1ae9a1558d 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -176,7 +176,6 @@ def get_historical_features( # Create lazy function that is only called from the RetrievalJob object def evaluate_historical_retrieval(): - # Create a copy of entity_df to prevent modifying the original entity_df_with_features = entity_df.copy() @@ -188,25 +187,31 @@ def evaluate_historical_retrieval(): or entity_df_event_timestamp_col_type.tz != pytz.UTC ): # Make sure all event timestamp fields are tz-aware. We default tz-naive fields to UTC - entity_df_with_features[ - entity_df_event_timestamp_col - ] = entity_df_with_features[entity_df_event_timestamp_col].apply( - lambda x: x if x.tzinfo is not None else x.replace(tzinfo=pytz.utc) + entity_df_with_features[entity_df_event_timestamp_col] = ( + entity_df_with_features[ + entity_df_event_timestamp_col + ].apply( + lambda x: x + if x.tzinfo is not None + else x.replace(tzinfo=pytz.utc) + ) ) # Convert event timestamp column to datetime and normalize time zone to UTC # This is necessary to avoid issues with pd.merge_asof if isinstance(entity_df_with_features, dd.DataFrame): - entity_df_with_features[ - entity_df_event_timestamp_col - ] = dd.to_datetime( - entity_df_with_features[entity_df_event_timestamp_col], utc=True + entity_df_with_features[entity_df_event_timestamp_col] = ( + dd.to_datetime( + entity_df_with_features[entity_df_event_timestamp_col], + utc=True, + ) ) else: - entity_df_with_features[ - entity_df_event_timestamp_col - ] = pd.to_datetime( - entity_df_with_features[entity_df_event_timestamp_col], utc=True + entity_df_with_features[entity_df_event_timestamp_col] = ( + pd.to_datetime( + entity_df_with_features[entity_df_event_timestamp_col], + utc=True, + ) ) # Sort event timestamp values diff --git a/sdk/python/feast/infra/offline_stores/redshift_source.py b/sdk/python/feast/infra/offline_stores/redshift_source.py index 52ab50ba00..f8cd53b246 100644 --- a/sdk/python/feast/infra/offline_stores/redshift_source.py +++ b/sdk/python/feast/infra/offline_stores/redshift_source.py @@ -220,9 +220,9 @@ def get_table_column_names_and_types( if config.offline_store.cluster_id: # Provisioned cluster - paginator_kwargs[ - "ClusterIdentifier" - ] = config.offline_store.cluster_id + paginator_kwargs["ClusterIdentifier"] = ( + config.offline_store.cluster_id + ) paginator_kwargs["DbUser"] = config.offline_store.user elif config.offline_store.workgroup: # Redshift serverless diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index cfaca038e7..907e4d4483 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -286,7 +286,6 @@ def get_historical_features( @contextlib.contextmanager def query_generator() -> Iterator[str]: - table_name = offline_utils.get_temp_entity_table_name() _upload_entity_df(entity_df, snowflake_conn, config, table_name) @@ -412,7 +411,6 @@ def __init__( feature_views: Optional[List[FeatureView]] = None, metadata: Optional[RetrievalMetadata] = None, ): - if feature_views is None: feature_views = [] if not isinstance(query, str): @@ -507,7 +505,6 @@ def to_snowflake( return None def to_arrow_batches(self) -> Iterator[pyarrow.Table]: - table_name = "temp_arrow_batches_" + uuid.uuid4().hex self.to_snowflake(table_name=table_name, allow_overwrite=True, temporary=True) @@ -520,7 +517,6 @@ def to_arrow_batches(self) -> Iterator[pyarrow.Table]: return arrow_batches def to_pandas_batches(self) -> Iterator[pd.DataFrame]: - table_name = "temp_pandas_batches_" + uuid.uuid4().hex self.to_snowflake(table_name=table_name, allow_overwrite=True, temporary=True) @@ -624,13 +620,10 @@ def _get_entity_schema( snowflake_conn: SnowflakeConnection, config: RepoConfig, ) -> Dict[str, np.dtype]: - if isinstance(entity_df, pd.DataFrame): - return dict(zip(entity_df.columns, entity_df.dtypes)) else: - query = f"SELECT * FROM ({entity_df}) LIMIT 1" limited_entity_df = execute_snowflake_statement( snowflake_conn, query @@ -645,7 +638,6 @@ def _upload_entity_df( config: RepoConfig, table_name: str, ) -> None: - if isinstance(entity_df, pd.DataFrame): # Write the data from the DataFrame to the table # Known issues with following entity data types: BINARY @@ -669,7 +661,6 @@ def _upload_entity_df( def _fix_entity_selections_identifiers(query_context) -> list: - for i, qc in enumerate(query_context): for j, es in enumerate(qc.entity_selections): query_context[i].entity_selections[j] = f'"{es}"'.replace(" AS ", '" AS "') diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index 9a2c6e09bc..c0b2417099 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -395,7 +395,6 @@ def __init__(self, table_ref: str): @staticmethod def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage: - return SavedDatasetSnowflakeStorage( table_ref=SnowflakeOptions.from_proto(storage_proto.snowflake_storage).table ) diff --git a/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py b/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py index 7ec803a69c..2537ecbf45 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py @@ -17,6 +17,7 @@ """ Hazelcast online store for Feast. """ + import base64 import threading from datetime import datetime, timezone @@ -200,7 +201,6 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - online_store_config = config.online_store if not isinstance(online_store_config, HazelcastOnlineStoreConfig): raise HazelcastInvalidConfig( diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index 4b2d8ae39c..d46b848c12 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -107,9 +107,9 @@ def online_write_batch( ) values_dict = {} for feature_name, val in values.items(): - values_dict[ - HbaseConstants.get_col_from_feature(feature_name) - ] = val.SerializeToString() + values_dict[HbaseConstants.get_col_from_feature(feature_name)] = ( + val.SerializeToString() + ) if isinstance(timestamp, datetime): values_dict[HbaseConstants.DEFAULT_EVENT_TS] = struct.pack( ">L", int(calendar.timegm(timestamp.timetuple())) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index cf07d5fef1..26916a9fcb 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -41,7 +41,6 @@ class MySQLOnlineStore(OnlineStore): _conn: Optional[Connection] = None def _get_conn(self, config: RepoConfig) -> Connection: - online_store_config = config.online_store assert isinstance(online_store_config, MySQLOnlineStoreConfig) @@ -65,7 +64,6 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: - conn = self._get_conn(config) cur = conn.cursor() @@ -178,7 +176,6 @@ def update( # We don't create any special state for the entities in this implementation. for table in tables_to_keep: - table_name = _table_id(project, table) index_name = f"{table_name}_ek" cur.execute( diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index ae96e16c64..149354b472 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -169,7 +169,6 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: - online_config = config.online_store assert isinstance(online_config, DatastoreOnlineStoreConfig) client = self._get_client(online_config) @@ -259,7 +258,6 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - online_config = config.online_store assert isinstance(online_config, DatastoreOnlineStoreConfig) client = self._get_client(online_config) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 4a6aa28889..745a9ed5a5 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -38,9 +38,9 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): """Online store config for local (SQLite-based) store""" - type: Literal[ - "sqlite", "feast.infra.online_stores.sqlite.SqliteOnlineStore" - ] = "sqlite" + type: Literal["sqlite", "feast.infra.online_stores.sqlite.SqliteOnlineStore"] = ( + "sqlite" + ) """ Online store type selector""" path: StrictStr = "data/online.db" @@ -86,7 +86,6 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: - conn = self._get_conn(config) project = config.project diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index c67164103e..ed1fc3ab87 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -489,7 +489,6 @@ def get_validation_reference( def list_validation_references( self, project: str, allow_cache: bool = False ) -> List[ValidationReference]: - """ Retrieve a list of validation references from the registry @@ -550,14 +549,12 @@ def apply_user_metadata( project: str, feature_view: BaseFeatureView, metadata_bytes: Optional[bytes], - ): - ... + ): ... @abstractmethod def get_user_metadata( self, project: str, feature_view: BaseFeatureView - ) -> Optional[bytes]: - ... + ) -> Optional[bytes]: ... @abstractmethod def proto(self) -> RegistryProto: @@ -642,9 +639,9 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: ): if "userDefinedFunction" not in odfv_dict["spec"]: odfv_dict["spec"]["userDefinedFunction"] = {} - odfv_dict["spec"]["userDefinedFunction"][ - "body" - ] = on_demand_feature_view.feature_transformation.udf_string + odfv_dict["spec"]["userDefinedFunction"]["body"] = ( + on_demand_feature_view.feature_transformation.udf_string + ) odfv_dict["spec"]["featureTransformation"]["userDefinedFunction"][ "body" ] = on_demand_feature_view.feature_transformation.udf_string @@ -669,9 +666,9 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: ): sfv_dict = self._message_to_sorted_dict(stream_feature_view.to_proto()) - sfv_dict["spec"]["userDefinedFunction"][ - "body" - ] = stream_feature_view.udf_string + sfv_dict["spec"]["userDefinedFunction"]["body"] = ( + stream_feature_view.udf_string + ) registry_dict["streamFeatureViews"].append(sfv_dict) for saved_dataset in sorted( diff --git a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py index 8548e4dbd8..dd965c4bed 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py +++ b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py @@ -48,7 +48,6 @@ def __init__(self, config: Any, autocommit=True): self.autocommit = autocommit def __enter__(self): - assert self.config.type in [ "snowflake.registry", "snowflake.offline", @@ -512,7 +511,6 @@ def chunk_helper(lst: pd.DataFrame, n: int) -> Iterator[Tuple[int, pd.DataFrame] def parse_private_key_path(key_path: str, private_key_passphrase: str) -> bytes: - with open(key_path, "rb") as key: p_key = serialization.load_pem_private_key( key.read(), diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index f83500cbc9..8a61b11065 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -160,9 +160,9 @@ def __init__( # noqa: C901 elif isinstance(odfv_source, FeatureViewProjection): self.source_feature_view_projections[odfv_source.name] = odfv_source else: - self.source_feature_view_projections[ - odfv_source.name - ] = odfv_source.projection + self.source_feature_view_projections[odfv_source.name] = ( + odfv_source.projection + ) self.feature_transformation = feature_transformation @@ -430,7 +430,6 @@ def get_transformed_features_dict( self, feature_dict: Dict[str, Any], # type: ignore ) -> Dict[str, Any]: - # we need a mapping from full feature name to short and back to do a renaming # The simplest thing to do is to make the full reference, copy the columns with the short reference # and rerun @@ -669,7 +668,6 @@ def mainify(obj) -> None: obj.__module__ = "__main__" def decorator(user_function): - return_annotation = inspect.signature(user_function).return_annotation if ( return_annotation diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 000e000438..0b659b960c 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -201,7 +201,6 @@ def parse_repo(repo_root: Path) -> RepoContents: @log_exceptions_and_usage def plan(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool): - os.chdir(repo_path) project, registry, repo, store = _prepare_registry_and_repo(repo_config, repo_path) diff --git a/sdk/python/feast/templates/athena/feature_repo/test_workflow.py b/sdk/python/feast/templates/athena/feature_repo/test_workflow.py index bf69a4bff0..8d6479da80 100644 --- a/sdk/python/feast/templates/athena/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/athena/feature_repo/test_workflow.py @@ -11,9 +11,7 @@ def test_end_to_end(): - try: - # Before running this test method # 1. Upload the driver_stats.parquet file to your S3 bucket. # (https://github.com/feast-dev/feast-custom-offline-store-demo/tree/main/feature_repo/data) diff --git a/sdk/python/feast/templates/snowflake/bootstrap.py b/sdk/python/feast/templates/snowflake/bootstrap.py index 01f4045fe7..2224dc5359 100644 --- a/sdk/python/feast/templates/snowflake/bootstrap.py +++ b/sdk/python/feast/templates/snowflake/bootstrap.py @@ -55,7 +55,6 @@ def bootstrap(): f'Should I upload example data to Snowflake (overwriting "{project_name}_feast_driver_hourly_stats" table)?', default=True, ): - snowflake_conn = snowflake.connector.connect( account=snowflake_deployment_url, user=snowflake_user, diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 70fbda964d..89a1a9ab41 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -70,9 +70,9 @@ def _get_requested_feature_views_to_features_dict( Set full_feature_names to True to have feature names prefixed by their feature view name.""" feature_views_to_feature_map: Dict["FeatureView", List[str]] = defaultdict(list) - on_demand_feature_views_to_feature_map: Dict[ - "OnDemandFeatureView", List[str] - ] = defaultdict(list) + on_demand_feature_views_to_feature_map: Dict["OnDemandFeatureView", List[str]] = ( + defaultdict(list) + ) for ref in feature_refs: ref_parts = ref.split(":") diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml new file mode 100644 index 0000000000..10ad007fa9 --- /dev/null +++ b/sdk/python/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +exclude = [".git","__pycache__","docs/conf.py","dist","feast/protos","feast/embedded_go/lib","feast/infra/utils/snowflake/snowpark/snowflake_udfs.py"] + +[tool.ruff.lint] +select = ["E","F","W","I"] +ignore = ["E203", "E266", "E501", "E721"] + +[tool.ruff.lint.isort] +known-first-party = ["feast", "feast", "feast_serving_server", "feast_core_server"] +default-section = "third-party" + +[tool.mypy] +files = ["feast","tests"] +ignore_missing_imports = true +exclude = ["feast/embedded_go/lib"] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index ac1994da37..af7a87c11b 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -57,8 +59,6 @@ beautifulsoup4==4.12.3 # via nbconvert bidict==0.23.1 # via ibis-framework -black==22.12.0 - # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.34.69 @@ -105,7 +105,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # black # bowler # dask # feast (setup.py) @@ -193,8 +192,6 @@ firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler -flake8==6.0.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 @@ -354,8 +351,6 @@ isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.13.2 - # via feast (setup.py) jedi==0.19.1 # via ipython jinja2==3.1.3 @@ -447,8 +442,6 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mccabe==0.7.0 - # via flake8 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 @@ -482,9 +475,7 @@ mypy==1.9.0 # feast (setup.py) # sqlalchemy mypy-extensions==1.0.0 - # via - # black - # mypy + # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) nbclient==0.10.0 @@ -559,8 +550,6 @@ parsy==2.1 # via ibis-framework partd==1.4.1 # via dask -pathspec==0.12.1 - # via black pbr==6.0.0 # via mock pexpect==4.9.0 @@ -569,7 +558,6 @@ pip-tools==7.4.1 # via feast (setup.py) platformdirs==3.11.0 # via - # black # jupyter-core # snowflake-connector-python # virtualenv @@ -647,8 +635,6 @@ pyasn1-modules==0.3.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) -pycodestyle==2.10.0 - # via flake8 pycparser==2.21 # via cffi pydantic==2.6.4 @@ -658,8 +644,6 @@ pydantic==2.6.4 # great-expectations pydantic-core==2.16.3 # via pydantic -pyflakes==3.0.1 - # via flake8 pygments==2.17.2 # via # feast (setup.py) @@ -807,6 +791,8 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations +ruff==0.3.4 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.12.0 @@ -889,7 +875,6 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via - # black # build # coverage # jupyterlab diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 367b5dc050..53c7fd1bce 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -18,6 +18,8 @@ anyio==4.3.0 # watchfiles appdirs==1.4.4 # via fissix +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -57,8 +59,6 @@ beautifulsoup4==4.12.3 # via nbconvert bidict==0.23.1 # via ibis-framework -black==22.12.0 - # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.34.69 @@ -105,7 +105,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # black # bowler # dask # feast (setup.py) @@ -193,8 +192,6 @@ firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler -flake8==6.0.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 @@ -362,8 +359,6 @@ isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.13.2 - # via feast (setup.py) jedi==0.19.1 # via ipython jinja2==3.1.3 @@ -455,8 +450,6 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mccabe==0.7.0 - # via flake8 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 @@ -490,9 +483,7 @@ mypy==1.9.0 # feast (setup.py) # sqlalchemy mypy-extensions==1.0.0 - # via - # black - # mypy + # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) nbclient==0.10.0 @@ -567,8 +558,6 @@ parsy==2.1 # via ibis-framework partd==1.4.1 # via dask -pathspec==0.12.1 - # via black pbr==6.0.0 # via mock pexpect==4.9.0 @@ -577,7 +566,6 @@ pip-tools==7.4.1 # via feast (setup.py) platformdirs==3.11.0 # via - # black # jupyter-core # snowflake-connector-python # virtualenv @@ -655,8 +643,6 @@ pyasn1-modules==0.3.0 # via google-auth pybindgen==0.22.1 # via feast (setup.py) -pycodestyle==2.10.0 - # via flake8 pycparser==2.21 # via cffi pydantic==2.6.4 @@ -666,8 +652,6 @@ pydantic==2.6.4 # great-expectations pydantic-core==2.16.3 # via pydantic -pyflakes==3.0.1 - # via flake8 pygments==2.17.2 # via # feast (setup.py) @@ -817,6 +801,8 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml +ruff==0.3.3 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.12.0 @@ -899,7 +885,6 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via - # black # build # coverage # jupyterlab @@ -979,7 +964,6 @@ typing-extensions==4.10.0 # async-lru # azure-core # azure-storage-blob - # black # fastapi # great-expectations # ibis-framework diff --git a/sdk/python/setup.cfg b/sdk/python/setup.cfg deleted file mode 100644 index d934249d69..0000000000 --- a/sdk/python/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[isort] -src_paths = feast,tests -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -skip=feast/protos,feast/embedded_go/lib -known_first_party=feast,feast_serving_server,feast_core_server -default_section=THIRDPARTY - -[flake8] -ignore = E203, E266, E501, W503, C901 -max-line-length = 88 -max-complexity = 20 -select = B,C,E,F,W,T4 -exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib,feast/infra/utils/snowflake/snowpark/snowflake_udfs.py - -[mypy] -files=feast,tests -ignore_missing_imports=true -exclude=feast/embedded_go/lib diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index f745bafa13..d2450bf868 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -440,9 +440,9 @@ def construct_test_environment( aws_registry_path = os.getenv( "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" ) - registry: Union[ - str, RegistryConfig - ] = f"{aws_registry_path}/{project}/registry.db" + registry: Union[str, RegistryConfig] = ( + f"{aws_registry_path}/{project}/registry.db" + ) else: registry = RegistryConfig( path=str(Path(repo_dir_name) / "registry.db"), diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 066497a0bc..4fcd9533e8 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -42,7 +42,6 @@ def create_dataset(self): self.client.update_dataset(self.dataset, ["default_table_expiration_ms"]) def teardown(self): - for table in self.tables: self.client.delete_table(table, not_found_ok=True) @@ -68,7 +67,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - destination_name = self.get_prefixed_table_name(destination_name) self.create_dataset() diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 008bb8d881..c70dae9863 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -43,7 +43,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - destination_name = self.get_prefixed_table_name(destination_name) f = tempfile.NamedTemporaryFile( @@ -98,7 +97,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - destination_name = self.get_prefixed_table_name(destination_name) dataset_path = tempfile.TemporaryDirectory( diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index 5a4e3f1085..60fb8950a9 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -20,7 +20,6 @@ class RedshiftDataSourceCreator(DataSourceCreator): - tables: List[str] = [] def __init__(self, project_name: str, *args, **kwargs): @@ -54,7 +53,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - destination_name = self.get_prefixed_table_name(destination_name) aws_utils.upload_df_to_redshift( diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py index 1481b11a10..237be2ac01 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py @@ -24,7 +24,6 @@ class SnowflakeDataSourceCreator(DataSourceCreator): - tables: List[str] = [] def __init__(self, project_name: str, *args, **kwargs): @@ -53,7 +52,6 @@ def create_data_source( field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", ) -> DataSource: - destination_name = self.get_prefixed_table_name(destination_name) with GetSnowflakeConnection(self.offline_store_config) as conn: diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/hazelcast.py b/sdk/python/tests/integration/feature_repos/universal/online_store/hazelcast.py index 65d74135ae..d50f2b75a3 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/hazelcast.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/hazelcast.py @@ -12,7 +12,6 @@ class HazelcastOnlineStoreCreator(OnlineStoreCreator): - cluster_name: str = "" container: DockerContainer = None diff --git a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py index c7028a09ef..e85c1d7311 100644 --- a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py +++ b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py @@ -57,7 +57,6 @@ def test_spark_materialization_consistency(): ) try: - fs.apply([driver, driver_stats_fv]) print(df) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 3bc2b3fb39..232f035609 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -111,7 +111,6 @@ def minio_registry() -> Registry: ], ) def test_apply_entity_integration(test_registry): - entity = Entity( name="driver_car_id", description="Car driver id", diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index 7c24589c6f..3ce5876bd6 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -144,7 +144,7 @@ def test_feature_get_online_features_types_match( fs.materialize( environment.start_date, environment.end_date - - timedelta(hours=1) # throwing out last record to make sure + - timedelta(hours=1), # throwing out last record to make sure # we can successfully infer type even from all empty values ) diff --git a/sdk/python/tests/unit/cli/test_cli.py b/sdk/python/tests/unit/cli/test_cli.py index d15e1d1616..a286c847dd 100644 --- a/sdk/python/tests/unit/cli/test_cli.py +++ b/sdk/python/tests/unit/cli/test_cli.py @@ -105,7 +105,6 @@ def test_3rd_party_registry_store_with_fs_yaml_override_by_env_var() -> None: @contextmanager def setup_third_party_provider_repo(provider_name: str): with tempfile.TemporaryDirectory() as repo_dir_name: - # Construct an example repo in a temporary dir repo_path = Path(repo_dir_name) @@ -141,7 +140,6 @@ def setup_third_party_registry_store_repo( registry_store: str, fs_yaml_file_name: str = "feature_store.yaml" ): with tempfile.TemporaryDirectory() as repo_dir_name: - # Construct an example repo in a temporary dir repo_path = Path(repo_dir_name) diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index f93237fce5..9d8c4a7ec1 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -167,7 +167,6 @@ def retrieval_job(request, environment): full_feature_names=False, ) elif request.param is MsSqlServerRetrievalJob: - return MsSqlServerRetrievalJob( query="query", engine=MagicMock(), diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index ca4ed6472b..e1839fbd8b 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -12,7 +12,6 @@ def _test_config(config_text, expect_error: Optional[str]): Try loading a repo config and check raised error against a regex. """ with tempfile.TemporaryDirectory() as repo_dir_name: - repo_path = Path(repo_dir_name) repo_config = repo_path / "feature_store.yaml" diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index be97a838bd..e4acef9713 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -142,7 +142,7 @@ def view_with_missing_feature(features_df: pd.DataFrame) -> pd.DataFrame: mode="pandas", ) def python_native_test_invalid_pandas_view( - input_dict: Dict[str, Any] + input_dict: Dict[str, Any], ) -> Dict[str, Any]: output_dict: Dict[str, Any] = { "output": input_dict["some_date"], diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 926c7226fc..f9ab42ae5e 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -276,7 +276,7 @@ def test_online_to_df(): ) provider = store._get_provider() - for (d, c) in zip(driver_ids, customer_ids): + for d, c in zip(driver_ids, customer_ids): """ driver table: lon lat diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index 9b21900e6d..87e5ef0548 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -43,7 +43,6 @@ def test_null_unix_timestamp_list(): ), ) def test_python_values_to_proto_values_bool(values): - protos = python_values_to_proto_values(values, ValueType.BOOL) converted = feast_value_type_to_python_type(protos[0]) diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index d8c769f12c..798e82de9b 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -180,7 +180,6 @@ def make_feature_store_yaml( repo_dir_name: Path, offline_creator: DataSourceCreator, ): - offline_store_config = offline_creator.create_offline_store_config() online_store = test_repo_config.online_store diff --git a/sdk/python/tests/utils/feature_records.py b/sdk/python/tests/utils/feature_records.py index 3f210f9e1c..2c26f3c000 100644 --- a/sdk/python/tests/utils/feature_records.py +++ b/sdk/python/tests/utils/feature_records.py @@ -260,7 +260,7 @@ def get_expected_training_df( if "val_to_add" in expected_df.columns: expected_df[ get_response_feature_name("conv_rate_plus_val_to_add", full_feature_names) - ] = (expected_df[conv_feature_name] + expected_df["val_to_add"]) + ] = expected_df[conv_feature_name] + expected_df["val_to_add"] return expected_df @@ -291,7 +291,6 @@ def assert_feature_service_correctness( expected_df, event_timestamp, ): - job_from_df = store.get_historical_features( entity_df=entity_df, features=store.get_feature_service(feature_service.name), diff --git a/setup.cfg b/setup.cfg index 2781169a71..2a9acf13da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,2 @@ -[isort] -src_paths = feast,tests -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -skip=feast/protos,feast/embedded_go/lib -known_first_party=feast,feast_serving_server,feast_core_server -default_section=THIRDPARTY - -[flake8] -ignore = E203, E266, E501, W503 -max-line-length = 88 -max-complexity = 20 -select = B,C,E,F,W,T4 -exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib - -[mypy] -files=feast,tests -ignore_missing_imports=true -exclude=feast/embedded_go/lib - [bdist_wheel] universal = 1 diff --git a/setup.py b/setup.py index ef3ba1d784..e14e723d0e 100644 --- a/setup.py +++ b/setup.py @@ -156,9 +156,7 @@ "build", "virtualenv==20.23.0", "cryptography>=35.0,<43", - "flake8>=6.0.0,<6.1.0", - "black>=22.6.0,<23", - "isort>=5,<6", + "ruff>=0.3.3", "grpcio-testing>=1.56.2,<2", # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). "httpx>=0.23.3",