Updated to run with omnipy v0.20.1

Import cleanup

Author: sveinugu

poetry.lock

@@ -17,7 +17,7 @@ reader = "^3.4"
 #github3-py = "^3.2.0"
 #pandas = "1.5.3"
 #pytest = "<8"
-omnipy = "^0.16.1"
+omnipy = "0.20.1"
 #omnipy = { git = "https://github.com/fairtracks/omnipy.git", branch = "release_0.16.0" }
 boto3 = "^1.35.23"
 #prefect-aws = "<0.5"

pyproject.toml

@@ -1,7 +1,9 @@
+from docker.utils.json_stream import json_decoder
 from omnipy import (Chain2,
                     Chain3,
                     convert_dataset,
                     Dataset,
+                    HttpUrlDataset,
                     LinearFlowTemplate,
                     MatchItemsModel,
                     Model,
@@ -63,10 +65,10 @@ class BedDataset(Dataset[BedModel]):
 
 
 # Omnipy tasks
-@TaskTemplate
-def fetch_bed_dataset(url_list: StrDataset) -> StrDataset:
+@TaskTemplate()
+def fetch_bed_dataset(url_list: HttpUrlDataset) -> StrDataset:
     bed_raw_dataset = StrDataset()
-    bed_raw_dataset.load(*url_list.values())
+    bed_raw_dataset.load(url_list)
     return bed_raw_dataset
 
 
@@ -78,11 +80,14 @@ def fetch_bed_dataset(url_list: StrDataset) -> StrDataset:
     convert_dataset.refine(
         name='convert_to_dataframe', fixed_params={'dataset_cls': PandasDataset}),
 )
-def import_bed_files_to_pandas(owner: str, repo: str, path: str, file_suffix: str) -> PandasDataset:
+def import_bed_files_to_pandas(owner: str, repo: str, branch: str, path: str,
+                               file_suffix: str) -> PandasDataset:
     ...
 
 
 # Running the flow
 if __name__ == '__main__':
     import_bed_files_to_pandas.run(
         owner='arq5x', repo='bedtools2', branch='master', path='data', file_suffix='bed')
+
+json_decoder

src/omnipy_examples/bed.py

@@ -7,11 +7,7 @@
 
 import github3
 import httpx
-from omnipy import runtime
-from omnipy.compute.flow import FuncFlowTemplate, LinearFlowTemplate
-from omnipy.compute.task import TaskTemplate
-from omnipy.data.dataset import Dataset
-from omnipy.data.model import Model
+from omnipy import Dataset, FuncFlowTemplate, LinearFlowTemplate, Model, runtime, TaskTemplate
 from omnipy_examples.chatgpt_words import (literary_styles,
                                            surrealistic_adjectives,
                                            surrealistic_nouns)

src/omnipy_examples/chatgpt.py

@@ -1,9 +1,9 @@
-from omnipy.compute.flow import LinearFlowTemplate
-from omnipy.modules.general.tasks import import_directory
-from omnipy.modules.json.datasets import JsonDictOfDictsDataset
-from omnipy.modules.raw.tasks import modify_datafile_contents
+from omnipy import (import_directory,
+                    JsonDictOfDictsDataset,
+                    LinearFlowTemplate,
+                    modify_datafile_contents)
 
-# from omnipy.modules.r_stat import r
+# from omnipy.components.r_stat import r
 
 # Regex patterns for parsing
 #     variable_pattern = re.compile(r"  type discrete \[ \d+ \] \{ (.+) \};\s*")
@@ -14,7 +14,7 @@
 #         re.compile(r"probability \( (.+) \| (.+) \) \{\s*"))
 #     conditional_probability_pattern_2 = re.compile(r"  \((.+)\) (.+);\s*")
 
-# @TaskTemplate
+# @TaskTemplate()
 # def import_dag_from_bnlearn(dag_name: str):
 #     r('chooseCRANmirror(ind = 1)')
 #     r('install.binaries("bnlearn")')

src/omnipy_examples/dagsim.py

@@ -1,9 +1,9 @@
-from omnipy.compute.flow import LinearFlowTemplate
-from omnipy.modules.fairtracks.tasks import import_dataset_from_encode
-from omnipy.modules.json.flows import flatten_nested_json
-from omnipy.modules.pandas.models import PandasDataset
-from omnipy.modules.pandas.tasks import convert_dataset_list_of_dicts_to_pandas
-from omnipy.modules.tables.tasks import remove_columns
+from omnipy import (convert_dataset_list_of_dicts_to_pandas,
+                    flatten_nested_json,
+                    LinearFlowTemplate,
+                    PandasDataset,
+                    remove_columns)
+from omnipy.components._fairtracks.tasks import import_dataset_from_encode
 
 # cast_to_dict_on_top = cast_dataset.refine(
 #     name='cast_to_dict_on_top',
@@ -17,7 +17,7 @@
 # runtime.config.prefect.use_cached_results = True
 #
 #
-# @FuncFlowTemplate
+# @FuncFlowTemplate()
 # def import_encode_data_tmpl():
 #     encode_json = import_dataset_from_encode(
 #         endpoints=[

src/omnipy_examples/encode.py

@@ -1,9 +1,9 @@
-from omnipy.compute.flow import LinearFlowTemplate
-from omnipy.modules.general.tasks import import_directory
-from omnipy.modules.json.flows import (flatten_nested_json,
-                                       transpose_dicts_of_lists_of_dicts_2_lists_of_dicts)
-from omnipy.modules.json.tasks import convert_dataset_string_to_json
-from omnipy.modules.pandas.tasks import convert_dataset_list_of_dicts_to_pandas
+from omnipy import (convert_dataset_list_of_dicts_to_pandas,
+                    convert_dataset_string_to_json,
+                    flatten_nested_json,
+                    import_directory,
+                    LinearFlowTemplate,
+                    transpose_dicts_of_lists_of_dicts_2_lists_of_dicts)
 
 
 @LinearFlowTemplate(

src/omnipy_examples/isajson.py

@@ -1,8 +1,11 @@
 import importlib
 from pathlib import Path
 
-from omnipy import PandasDataset, runtime
-from omnipy.api.enums import ConfigPersistOutputsOptions, ConfigRestoreOutputsOptions, EngineChoice
+from omnipy import (ConfigPersistOutputsOptions,
+                    ConfigRestoreOutputsOptions,
+                    EngineChoice,
+                    PandasDataset,
+                    runtime)
 import typer
 
 app = typer.Typer()

src/omnipy_examples/main.py

@@ -1,7 +1,6 @@
 from typing import Callable, Optional, TypeAlias
 
-from omnipy.modules.json.datasets import JsonDataset
-from omnipy.modules.json.models import JsonModel
+from omnipy import JsonDataset, JsonModel
 from pydantic import BaseModel
 
 IsaJsonModel: TypeAlias = JsonModel

src/omnipy_examples/multi_broker.py

@@ -1,15 +1,16 @@
-from omnipy.compute.flow import FuncFlowTemplate, LinearFlowTemplate
-from omnipy.compute.task import TaskTemplate
-from omnipy.modules.json.datasets import JsonDataset, JsonDictOfListsOfDictsDataset
-from omnipy.modules.json.flows import flatten_nested_json
-from omnipy.modules.json.tasks import transpose_dicts_2_lists_old
-from omnipy.modules.pandas.models import PandasDataset
-from omnipy.modules.pandas.tasks import convert_dataset_list_of_dicts_to_pandas
+from omnipy import (convert_dataset_list_of_dicts_to_pandas,
+                    flatten_nested_json,
+                    FuncFlowTemplate,
+                    JsonDictOfListsOfDictsDataset,
+                    LinearFlowTemplate,
+                    PandasDataset,
+                    TaskTemplate,
+                    transpose_dicts_2_lists)
 import pandas as pd
 import requests
 
 
-@TaskTemplate
+@TaskTemplate()
 def import_uniprot() -> JsonDictOfListsOfDictsDataset:
     HEADERS = {'accept': 'application/json'}
     api_url = 'https://rest.uniprot.org/uniprotkb/search?query=human%20cdc7'
@@ -42,14 +43,14 @@ def import_uniprot() -> JsonDictOfListsOfDictsDataset:
     # TODO: When automatic transformation of task/flow inputs/outputs are implemented,
     #       remove to_pandas calls, here and otherwise
     import_uniprot,  # cast_json,
-    transpose_dicts_2_lists_old,
+    transpose_dicts_2_lists,
     flatten_nested_json,
     convert_dataset_list_of_dicts_to_pandas)
 def import_and_flatten_uniprot() -> PandasDataset:
     ...
 
 
-@TaskTemplate
+@TaskTemplate()
 def pandas_magic(pandas: PandasDataset) -> PandasDataset:
     #  Get synonym table and clean foreign key
     df_synonym = pandas['results.genes.synonyms'].contents
@@ -83,7 +84,7 @@ def pandas_magic(pandas: PandasDataset) -> PandasDataset:
     return out_dataset
 
 
-@TaskTemplate
+@TaskTemplate()
 def pandas_magic_alternative(dataset: PandasDataset) -> PandasDataset:
     df_merge_1 = pd.merge(
         dataset['results.genes.geneName'].contents,
@@ -138,7 +139,7 @@ def pandas_magic_alternative(dataset: PandasDataset) -> PandasDataset:
     return out_dataset
 
 
-# @TaskTemplate
+# @TaskTemplate()
 # def pandas_magic_unimplemented(dataset: PandasDataset) -> PandasDataset:
 #     out_dataset = PandasDataset()
 #     out_dataset['my_table'] = extract_mapped_table_from_flattened_dataset(
@@ -155,14 +156,14 @@ def pandas_magic_alternative(dataset: PandasDataset) -> PandasDataset:
 #     return out_dataset
 
 
-@FuncFlowTemplate
+@FuncFlowTemplate()
 def import_and_flatten_uniprot_with_magic() -> PandasDataset:
     uniprot_6_ds = import_and_flatten_uniprot()
     uniprot_7_ds = pandas_magic_alternative(uniprot_6_ds)
     return uniprot_7_ds
 
 
-# @TaskTemplate
+# @TaskTemplate()
 # def join_a_with_b(pandas_ds: PandasDataset,
 #                   table_a_name: str,
 #                   a_ref_column: str,

src/omnipy_examples/uniprot.py

@@ -1,13 +1,13 @@
 import httpx
-from omnipy import JsonModel, Model, StrDataset, TaskTemplate
+from omnipy import HttpUrlDataset, JsonModel, Model, TaskTemplate
 
 
-@TaskTemplate
+@TaskTemplate()
 def get_github_repo_urls(owner: str, repo: str, branch: str, path: str,
-                         file_suffix: str) -> StrDataset:
+                         file_suffix: str) -> HttpUrlDataset:
     api_url = f'https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={branch}'
     url_pre = f'https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}'
 
     json_data = JsonModel(httpx.get(api_url).raise_for_status().json())
     names = Model[list[str]]([f['name'] for f in json_data if f['name'].endswith(file_suffix)])
-    return StrDataset({name: f'{url_pre}/{name}' for name in names})
+    return HttpUrlDataset({name: f'{url_pre}/{name}' for name in names})

src/omnipy_examples/util.py

@@ -1,21 +1,21 @@
 import asyncio
 
-import omnipy.modules.json.util
-import omnipy.modules.pandas.util
-import omnipy.modules.raw.util
+import omnipy.components.json.util
+import omnipy.components.pandas.util
+import omnipy.components.raw.util
 
-omnipy.modules.json.util.ROOT_DIR = '../../input/bif'
-omnipy.modules.pandas.util.ROOT_DIR = '../../input/bif'
-omnipy.modules.raw.util.ROOT_DIR = '../../input/bif'
+omnipy.components.json.util.ROOT_DIR = '../../input/bif'
+omnipy.components.pandas.util.ROOT_DIR = '../../input/bif'
+omnipy.components.raw.util.ROOT_DIR = '../../input/bif'
 
 
-# @TaskTemplate
+# @TaskTemplate()
 async def fetch_from_api(what: str):
     asyncio.sleep(10)
     return what
 
 
-# @FuncFlowTemplate
+# @FuncFlowTemplate()
 async def caller():
     data = fetch_from_api('asdsda')
     data2 = fetch_from_api('asdd')

workshops/2022.12 Oslo Bioinformatics Days/asyncio.py

@@ -1,8 +1,8 @@
 from typing import Dict, List
 
+from omnipy.components.pandas.datasets import PandasDataset
 from omnipy.data.dataset import Dataset, MultiModelDataset
 from omnipy.data.model import Model
-from omnipy.modules.pandas.models import PandasDataset
 from pydantic import BaseModel, validator