Skip to content

Commit

Permalink
Add try except for import
Browse files Browse the repository at this point in the history
Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>
  • Loading branch information
ankatiyar committed Sep 24, 2024
1 parent 7cf254e commit e66ce25
Showing 1 changed file with 13 additions and 4 deletions.
17 changes: 13 additions & 4 deletions kedro-airflow/kedro_airflow/grouping.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,26 @@
from __future__ import annotations

from kedro.io import CatalogProtocol
from typing import Any

from kedro.io import DataCatalog
from kedro.pipeline.node import Node
from kedro.pipeline.pipeline import Pipeline

try:
from kedro.io import CatalogProtocol
except ImportError: # pragma: no cover
pass


def _is_memory_dataset(catalog, dataset_name: str) -> bool:
if dataset_name not in catalog:
return True
return False


def get_memory_datasets(catalog: CatalogProtocol, pipeline: Pipeline) -> set[str]:
def get_memory_datasets(
catalog: CatalogProtocol[Any] | DataCatalog, pipeline: Pipeline
) -> set[str]:
"""Gather all datasets in the pipeline that are of type MemoryDataset, excluding 'parameters'."""
return {
dataset_name
Expand All @@ -21,7 +30,7 @@ def get_memory_datasets(catalog: CatalogProtocol, pipeline: Pipeline) -> set[str


def create_adjacency_list(
catalog: CatalogProtocol, pipeline: Pipeline
catalog: CatalogProtocol[Any] | DataCatalog, pipeline: Pipeline
) -> tuple[dict[str, set], dict[str, set]]:
"""
Builds adjacency list (adj_list) to search connected components - undirected graph,
Expand All @@ -48,7 +57,7 @@ def create_adjacency_list(


def group_memory_nodes(
catalog: CatalogProtocol, pipeline: Pipeline
catalog: CatalogProtocol[Any] | DataCatalog, pipeline: Pipeline
) -> tuple[dict[str, list[Node]], dict[str, list[str]]]:
"""
Nodes that are connected through MemoryDatasets cannot be distributed across
Expand Down

0 comments on commit e66ce25

Please sign in to comment.