diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index 505c158bb1..57a4f975a1 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -109,7 +109,7 @@ import typing import mrc -from morpheus._lib.common import TypeId +from morpheus.common import TypeId from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -435,7 +435,7 @@ pipeline.add_stage(WriteToFileStage(config, filename=results_file, overwrite=Tru Note that we didn't specify the output format. In our example, the result file contains the extension `.jsonlines`. Morpheus will infer the output format based on the extension. At time of writing the extensions that Morpheus will infer are: `.csv`, `.json` & `.jsonlines` -To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus._lib.common.FileTypes`. Current values defined are: +To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus.common.FileTypes`. Current values defined are: * `FileTypes.Auto` * `FileTypes.JSON` * `FileTypes.CSV` diff --git a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md index e169fb3df0..4ff89174ce 100644 --- a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md +++ b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md @@ -225,7 +225,7 @@ The `DFPFileToDataFrameStage` ([examples/digital_fingerprinting/production/morph | `c` | `morpheus.config.Config` | Morpheus config object | | `schema` | `DataFrameInputSchema` | Schema specifying columns to load, along with any necessary renames and data type conversions | | `filter_null` | `bool` | Optional: Whether to filter null rows after loading, by default True. | -| `file_type` | `morpheus._lib.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, and `FileTypes.JSON`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension | +| `file_type` | `morpheus.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, and `FileTypes.JSON`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension | | `parser_kwargs` | `dict` or `None` | Optional: additional keyword arguments to be passed into the `DataFrame` parser, currently this is going to be either [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html) or [`pandas.read_json`](https://pandas.pydata.org/docs/reference/api/pandas.read_json.html) | | `cache_dir` | `str` | Optional: path to cache location, defaults to `./.cache/dfp` | diff --git a/examples/developer_guide/2_1_real_world_phishing/recipient_features_stage.py b/examples/developer_guide/2_1_real_world_phishing/recipient_features_stage.py index 4e068521c9..b4107a829d 100644 --- a/examples/developer_guide/2_1_real_world_phishing/recipient_features_stage.py +++ b/examples/developer_guide/2_1_real_world_phishing/recipient_features_stage.py @@ -17,8 +17,8 @@ import mrc -from morpheus._lib.common import TypeId from morpheus.cli.register_stage import register_stage +from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.messages.message_meta import MessageMeta diff --git a/examples/developer_guide/2_2_rabbitmq/read_simple.py b/examples/developer_guide/2_2_rabbitmq/read_simple.py index 9b0ef5be45..969124c209 100755 --- a/examples/developer_guide/2_2_rabbitmq/read_simple.py +++ b/examples/developer_guide/2_2_rabbitmq/read_simple.py @@ -18,7 +18,7 @@ from rabbitmq_source_stage import RabbitMQSourceStage -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/read_simple.py b/examples/developer_guide/4_rabbitmq_cpp_stage/read_simple.py index 63ca571682..7581cf38b1 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/read_simple.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/read_simple.py @@ -19,7 +19,7 @@ import click from rabbitmq_source_stage import RabbitMQSourceStage -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py index 27c6230cea..6129840107 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py @@ -42,8 +42,8 @@ from benchmarks.dfp_config import get_duo_source_schema from benchmarks.dfp_config import load_json from benchmarks.dfp_config import set_mlflow_tracking_uri -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.general.linear_modules_stage import LinearModulesStage diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_file_to_df.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_file_to_df.py index c5ab74aa27..bf3bd82f38 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_file_to_df.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_file_to_df.py @@ -32,7 +32,7 @@ import cudf -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline.preallocator_mixin import PreallocatorMixin diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_azure_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/dfp_azure_inference.ipynb index 308a71d3bf..130f162f36 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_azure_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/dfp_azure_inference.ipynb @@ -55,8 +55,8 @@ "from dfp.stages.multi_file_source import MultiFileSource\n", "from dfp.utils.regex_utils import iso_date_regex\n", "\n", - "from morpheus._lib.common import FileTypes\n", - "from morpheus._lib.common import FilterSource\n", + "from morpheus.common import FileTypes\n", + "from morpheus.common import FilterSource\n", "from morpheus.cli.utils import get_log_levels\n", "from morpheus.cli.utils import get_package_relative_file\n", "from morpheus.cli.utils import load_labels_file\n", @@ -463,7 +463,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.6" }, "vscode": { "interpreter": { diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_azure_modules_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_azure_modules_pipeline.py index 7855df7eee..91423953af 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_azure_modules_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_azure_modules_pipeline.py @@ -36,9 +36,9 @@ from dfp.utils.schema_utils import Schema from dfp.utils.schema_utils import SchemaBuilder -from morpheus._lib.common import FilterSource from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import parse_log_level +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.pipeline import LinearPipeline from morpheus.stages.general.linear_modules_stage import LinearModulesStage diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py index 45ea4c259a..027272956e 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py @@ -36,12 +36,12 @@ from dfp.stages.multi_file_source import MultiFileSource from dfp.utils.regex_utils import iso_date_regex -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file from morpheus.cli.utils import load_labels_file from morpheus.cli.utils import parse_log_level +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import CppConfig diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_azure_training.ipynb b/examples/digital_fingerprinting/production/morpheus/dfp_azure_training.ipynb index 3f92a64663..5a1ebaf655 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_azure_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/dfp_azure_training.ipynb @@ -55,7 +55,7 @@ "from dfp.stages.multi_file_source import MultiFileSource\n", "from dfp.utils.regex_utils import iso_date_regex\n", "\n", - "from morpheus._lib.common import FileTypes\n", + "from morpheus.common import FileTypes\n", "from morpheus.cli.utils import get_log_levels\n", "from morpheus.cli.utils import get_package_relative_file\n", "from morpheus.cli.utils import load_labels_file\n", diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/dfp_duo_inference.ipynb index 09242d49fa..3a164432e7 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_duo_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/dfp_duo_inference.ipynb @@ -56,8 +56,8 @@ "from dfp.stages.multi_file_source import MultiFileSource\n", "from dfp.utils.regex_utils import iso_date_regex\n", "\n", - "from morpheus._lib.common import FileTypes\n", - "from morpheus._lib.common import FilterSource\n", + "from morpheus.common import FileTypes\n", + "from morpheus.common import FilterSource\n", "from morpheus.cli.utils import get_log_levels\n", "from morpheus.cli.utils import get_package_relative_file\n", "from morpheus.cli.utils import load_labels_file\n", diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_modules_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_duo_modules_pipeline.py index 3ef9489907..f7d26ff341 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_duo_modules_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_duo_modules_pipeline.py @@ -36,9 +36,9 @@ from dfp.utils.schema_utils import Schema from dfp.utils.schema_utils import SchemaBuilder -from morpheus._lib.common import FilterSource from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import parse_log_level +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.pipeline import LinearPipeline from morpheus.stages.general.linear_modules_stage import LinearModulesStage diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py index ec02e63a3a..588f0a448e 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py @@ -36,12 +36,12 @@ from dfp.stages.multi_file_source import MultiFileSource from dfp.utils.regex_utils import iso_date_regex -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file from morpheus.cli.utils import load_labels_file from morpheus.cli.utils import parse_log_level +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import CppConfig diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_training.ipynb b/examples/digital_fingerprinting/production/morpheus/dfp_duo_training.ipynb index 7b92857243..a96e21948c 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_duo_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/dfp_duo_training.ipynb @@ -55,7 +55,7 @@ "from dfp.stages.multi_file_source import MultiFileSource\n", "from dfp.utils.regex_utils import iso_date_regex\n", "\n", - "from morpheus._lib.common import FileTypes\n", + "from morpheus.common import FileTypes\n", "from morpheus.cli.utils import get_log_levels\n", "from morpheus.cli.utils import get_package_relative_file\n", "from morpheus.cli.utils import load_labels_file\n", diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py index 1fa38b746d..4177d2d011 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py @@ -36,11 +36,11 @@ from dfp.stages.multi_file_source import MultiFileSource from dfp.utils.regex_utils import iso_date_regex -from morpheus._lib.common import FileTypes from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file from morpheus.cli.utils import load_labels_file from morpheus.cli.utils import parse_log_level +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import CppConfig diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py index 49774a0083..45b55b48c4 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py @@ -36,11 +36,11 @@ from dfp.stages.multi_file_source import MultiFileSource from dfp.utils.regex_utils import iso_date_regex -from morpheus._lib.common import FileTypes from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file from morpheus.cli.utils import load_labels_file from morpheus.cli.utils import parse_log_level +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import CppConfig diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index c26059305c..e9f3bc9a66 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -15,12 +15,12 @@ import typing import mrc -from common.data_models import FeatureConfig -from common.feature_extractor import FeatureExtractor from mrc.core import operators as ops from dask.distributed import Client +from common.data_models import FeatureConfig +from common.feature_extractor import FeatureExtractor from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes diff --git a/examples/sid_visualization/run.py b/examples/sid_visualization/run.py index dd275698d9..db32b662d5 100644 --- a/examples/sid_visualization/run.py +++ b/examples/sid_visualization/run.py @@ -19,7 +19,7 @@ import click import mrc -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import PipelineModes @@ -54,7 +54,7 @@ class NLPVizFileSource(PreallocatorMixin, SingleOutputSource): iterative: boolean Iterative mode will emit dataframes one at a time. Otherwise a list of dataframes is emitted. Iterative mode is good for interleaving source stages. - file_type : `morpheus._lib.common.FileTypes`, default = 'auto' + file_type : `morpheus.common.FileTypes`, default = 'auto' Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'json', 'csv' repeat: int, default = 1 @@ -167,6 +167,7 @@ def run_pipeline(debug, use_cpp, num_threads, input_file, max_batch_size, model_ config.num_threads = num_threads config.pipeline_batch_size = max_batch_size config.feature_length = 256 + config.model_max_batch_size = 32 config.class_labels = load_labels_file(get_data_file_path("data/labels_nlp.txt")) # Create a linear pipeline object. diff --git a/morpheus/cli/commands.py b/morpheus/cli/commands.py index 96ec85f810..0a25d0b2af 100644 --- a/morpheus/cli/commands.py +++ b/morpheus/cli/commands.py @@ -48,7 +48,7 @@ DEFAULT_CONFIG = Config() -# List all of the options in from morpheus._lib.common.FileTypes without importing the object. This slows down +# List all of the options in from morpheus.common.FileTypes without importing the object. This slows down # autocomplete too much. FILE_TYPE_NAMES = ["auto", "csv", "json"] diff --git a/morpheus/cli/utils.py b/morpheus/cli/utils.py index e0b4b84d86..91b44f7123 100644 --- a/morpheus/cli/utils.py +++ b/morpheus/cli/utils.py @@ -38,7 +38,7 @@ def str_to_file_type(file_type_str: str): # Delay FileTypes since this will import ._lib - from morpheus._lib.common import FileTypes + from morpheus.common import FileTypes file_type_members = {name.lower(): t for (name, t) in FileTypes.__members__.items()} return file_type_members[file_type_str] diff --git a/morpheus/common/__init__.py b/morpheus/common/__init__.py new file mode 100644 index 0000000000..d2bbebcbc3 --- /dev/null +++ b/morpheus/common/__init__.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module for common utilities and classes in the Morpheus library. +""" + +# Export symbols from the morpheus._lib.common module. Users should never be directly importing morpheus._lib +from morpheus._lib.common import determine_file_type +from morpheus._lib.common import FiberQueue +from morpheus._lib.common import FileTypes +from morpheus._lib.common import FilterSource +from morpheus._lib.common import read_file_to_df +from morpheus._lib.common import Tensor +from morpheus._lib.common import tyepid_to_numpy_str +from morpheus._lib.common import TypeId + +__all__ = [ + "determine_file_type", + "FiberQueue", + "FileTypes", + "FilterSource", + "read_file_to_df", + "Tensor", + "tyepid_to_numpy_str", + "TypeId", +] diff --git a/morpheus/io/deserializers.py b/morpheus/io/deserializers.py index ce1f377c65..947215bbe0 100644 --- a/morpheus/io/deserializers.py +++ b/morpheus/io/deserializers.py @@ -19,9 +19,9 @@ import cudf -from morpheus._lib.common import FileTypes -from morpheus._lib.common import determine_file_type -from morpheus._lib.common import read_file_to_df as read_file_to_df_cpp +from morpheus.common import FileTypes +from morpheus.common import determine_file_type +from morpheus.common import read_file_to_df as read_file_to_df_cpp from morpheus.config import CppConfig from morpheus.io.utils import filter_null_data @@ -49,7 +49,7 @@ def read_file_to_df(file_name: str, ---------- file_name : str File to read. - file_type : `morpheus._lib.common.FileTypes` + file_type : `morpheus.common.FileTypes` Type of file. Leave as Auto to determine from the extension. parser_kwargs : dict, optional Any argument to pass onto the parse, by default {}. Ignored when C++ execution is enabled and `df_type="cudf"` diff --git a/morpheus/modules/file_to_df.py b/morpheus/modules/file_to_df.py index 056e72d409..86512400ff 100644 --- a/morpheus/modules/file_to_df.py +++ b/morpheus/modules/file_to_df.py @@ -30,8 +30,8 @@ import cudf -from morpheus._lib.common import FileTypes from morpheus.cli.utils import str_to_file_type +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.utils.column_info import process_dataframe from morpheus.utils.module_ids import FILE_TO_DF diff --git a/morpheus/modules/filter_detections.py b/morpheus/modules/filter_detections.py index 0ca208dd01..9d1b48552c 100644 --- a/morpheus/modules/filter_detections.py +++ b/morpheus/modules/filter_detections.py @@ -22,7 +22,7 @@ import typing_utils from mrc.core import operators as ops -from morpheus._lib.common import FilterSource +from morpheus.common import FilterSource from morpheus.messages import MultiMessage from morpheus.messages.multi_response_message import MultiResponseMessage from morpheus.utils.module_ids import FILTER_DETECTIONS diff --git a/morpheus/modules/write_to_file.py b/morpheus/modules/write_to_file.py index 5f3fb77ab1..af09ed229c 100644 --- a/morpheus/modules/write_to_file.py +++ b/morpheus/modules/write_to_file.py @@ -22,8 +22,8 @@ import cudf -from morpheus._lib.common import FileTypes -from morpheus._lib.common import determine_file_type +from morpheus.common import FileTypes +from morpheus.common import determine_file_type from morpheus.io import serializers from morpheus.messages.message_meta import MessageMeta from morpheus.utils.module_ids import MODULE_NAMESPACE diff --git a/morpheus/pipeline/preallocator_mixin.py b/morpheus/pipeline/preallocator_mixin.py index 3d6be2c7b8..4891887796 100644 --- a/morpheus/pipeline/preallocator_mixin.py +++ b/morpheus/pipeline/preallocator_mixin.py @@ -23,8 +23,8 @@ import cudf -from morpheus._lib.common import TypeId -from morpheus._lib.common import tyepid_to_numpy_str +from morpheus.common import TypeId +from morpheus.common import tyepid_to_numpy_str from morpheus.config import CppConfig from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage diff --git a/morpheus/pipeline/stream_wrapper.py b/morpheus/pipeline/stream_wrapper.py index caad1da6a4..0f25757153 100644 --- a/morpheus/pipeline/stream_wrapper.py +++ b/morpheus/pipeline/stream_wrapper.py @@ -409,7 +409,7 @@ def _create_ports(self, input_count: int, output_count: int): def get_needed_columns(self): """ Stages which need to have columns inserted into the dataframe, should populate the `self._needed_columns` - dictionary with mapping of column names to `morpheus._lib.common.TypeId`. This will ensure that the columns are + dictionary with mapping of column names to `morpheus.common.TypeId`. This will ensure that the columns are allocated and populated with null values. """ return self._needed_columns.copy() diff --git a/morpheus/stages/input/autoencoder_source_stage.py b/morpheus/stages/input/autoencoder_source_stage.py index dabe57aac1..da95354bda 100644 --- a/morpheus/stages/input/autoencoder_source_stage.py +++ b/morpheus/stages/input/autoencoder_source_stage.py @@ -22,7 +22,7 @@ import pandas as pd from mrc.core import operators as ops -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.messages import UserMessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin @@ -55,7 +55,7 @@ class AutoencoderSourceStage(PreallocatorMixin, SingleOutputSource): files. Any new files that are added that match the glob will then be processed. max_files: int, default = -1 Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited. - file_type : `morpheus._lib.common.FileTypes`, default = 'FileTypes.Auto'. + file_type : `morpheus.common.FileTypes`, default = 'FileTypes.Auto'. Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'json', 'csv' repeat: int, default = 1 diff --git a/morpheus/stages/input/azure_source_stage.py b/morpheus/stages/input/azure_source_stage.py index c53e5e816c..f08c518256 100644 --- a/morpheus/stages/input/azure_source_stage.py +++ b/morpheus/stages/input/azure_source_stage.py @@ -47,7 +47,7 @@ class AzureSourceStage(AutoencoderSourceStage): files. Any new files that are added that match the glob will then be processed. max_files: int, default = -1 Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited. - file_type : `morpheus._lib.common.FileTypes`, default = 'FileTypes.Auto'. + file_type : `morpheus.common.FileTypes`, default = 'FileTypes.Auto'. Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'json', 'csv' repeat: int, default = 1 diff --git a/morpheus/stages/input/cloud_trail_source_stage.py b/morpheus/stages/input/cloud_trail_source_stage.py index 7790d822e4..0e72c28c76 100644 --- a/morpheus/stages/input/cloud_trail_source_stage.py +++ b/morpheus/stages/input/cloud_trail_source_stage.py @@ -19,9 +19,9 @@ import numpy as np import pandas as pd -from morpheus._lib.common import FileTypes -from morpheus._lib.common import determine_file_type from morpheus.cli import register_stage +from morpheus.common import FileTypes +from morpheus.common import determine_file_type from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.stages.input.autoencoder_source_stage import AutoencoderSourceStage @@ -64,7 +64,7 @@ def read_file(filename: str, file_type: FileTypes) -> pd.DataFrame: ---------- filename : str Path to a file to read. - file_type : `morpheus._lib.common.FileTypes` + file_type : `morpheus.common.FileTypes` What type of file to read. Leave as Auto to auto detect based on the file extension. Returns diff --git a/morpheus/stages/input/duo_source_stage.py b/morpheus/stages/input/duo_source_stage.py index 8b5481cf55..98ab458bef 100644 --- a/morpheus/stages/input/duo_source_stage.py +++ b/morpheus/stages/input/duo_source_stage.py @@ -47,7 +47,7 @@ class DuoSourceStage(AutoencoderSourceStage): files. Any new files that are added that match the glob will then be processed. max_files: int, default = -1 Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited. - file_type : `morpheus._lib.common.FileTypes`, default = 'FileTypes.Auto'. + file_type : `morpheus.common.FileTypes`, default = 'FileTypes.Auto'. Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'json', 'csv' repeat: int, default = 1 diff --git a/morpheus/stages/input/file_source_stage.py b/morpheus/stages/input/file_source_stage.py index 1ecee0e64e..d803ac0fb2 100644 --- a/morpheus/stages/input/file_source_stage.py +++ b/morpheus/stages/input/file_source_stage.py @@ -20,8 +20,8 @@ import typing_utils from mrc.core import operators as ops -from morpheus._lib.common import FileTypes from morpheus.cli import register_stage +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df @@ -50,7 +50,7 @@ class FileSourceStage(PreallocatorMixin, SingleOutputSource): iterative : boolean, default = False, is_flag = True Iterative mode will emit dataframes one at a time. Otherwise a list of dataframes is emitted. Iterative mode is good for interleaving source stages. - file_type : `morpheus._lib.common.FileTypes`, optional, case_sensitive = False + file_type : `morpheus.common.FileTypes`, optional, case_sensitive = False Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'csv', 'json' and 'jsonlines' repeat : int, default = 1, min = 1 diff --git a/morpheus/stages/output/write_to_file_stage.py b/morpheus/stages/output/write_to_file_stage.py index 3ccd68ca6a..8f5634632f 100644 --- a/morpheus/stages/output/write_to_file_stage.py +++ b/morpheus/stages/output/write_to_file_stage.py @@ -22,9 +22,9 @@ import cudf import morpheus._lib.stages as _stages -from morpheus._lib.common import FileTypes -from morpheus._lib.common import determine_file_type from morpheus.cli.register_stage import register_stage +from morpheus.common import FileTypes +from morpheus.common import determine_file_type from morpheus.config import Config from morpheus.io import serializers from morpheus.messages import MessageMeta @@ -47,7 +47,7 @@ class WriteToFileStage(SinglePortStage): Name of the file to which the messages will be written. overwrite : boolean, default = False, is_flag = True Overwrite file if exists. Will generate an error otherwise. - file_type : `morpheus._lib.common.FileTypes`, optional, case_sensitive = False + file_type : `morpheus.common.FileTypes`, optional, case_sensitive = False Indicates what type of file to write. Specifying 'auto' will determine the file type from the extension. Supported extensions: 'csv', 'json' and 'jsonlines' include_index_col : bool, default = True diff --git a/morpheus/stages/postprocess/add_classifications_stage.py b/morpheus/stages/postprocess/add_classifications_stage.py index 26c0ba0b0d..f63d0a38d7 100644 --- a/morpheus/stages/postprocess/add_classifications_stage.py +++ b/morpheus/stages/postprocess/add_classifications_stage.py @@ -17,8 +17,8 @@ import mrc -from morpheus._lib.common import TypeId from morpheus.cli.register_stage import register_stage +from morpheus.common import TypeId from morpheus.config import Config from morpheus.stages.postprocess.add_scores_stage_base import AddScoresStageBase diff --git a/morpheus/stages/postprocess/add_scores_stage.py b/morpheus/stages/postprocess/add_scores_stage.py index 4eae6f174b..5c66bb2e1b 100644 --- a/morpheus/stages/postprocess/add_scores_stage.py +++ b/morpheus/stages/postprocess/add_scores_stage.py @@ -17,8 +17,8 @@ import mrc -from morpheus._lib.common import TypeId from morpheus.cli.register_stage import register_stage +from morpheus.common import TypeId from morpheus.config import Config from morpheus.stages.postprocess.add_scores_stage_base import AddScoresStageBase @@ -43,7 +43,7 @@ class AddScoresStage(AddScoresStageBase): the Config.class_labels property. prefix : str, default = "" Prefix to add to each label. Allows adding labels different from the `Config.class_labels` property. - probs_type : `morpheus._lib.common.TypeId`, default = "float32" + probs_type : `morpheus.common.TypeId`, default = "float32" Datatype of the scores columns. """ diff --git a/morpheus/stages/postprocess/add_scores_stage_base.py b/morpheus/stages/postprocess/add_scores_stage_base.py index cf180de62c..8f486de762 100644 --- a/morpheus/stages/postprocess/add_scores_stage_base.py +++ b/morpheus/stages/postprocess/add_scores_stage_base.py @@ -20,7 +20,7 @@ import mrc import mrc.core.operators as ops -from morpheus._lib.common import TypeId +from morpheus.common import TypeId from morpheus.config import Config from morpheus.messages import MultiResponseMessage from morpheus.pipeline.single_port_stage import SinglePortStage diff --git a/morpheus/stages/postprocess/filter_detections_stage.py b/morpheus/stages/postprocess/filter_detections_stage.py index e927a0a549..a98614ce8c 100644 --- a/morpheus/stages/postprocess/filter_detections_stage.py +++ b/morpheus/stages/postprocess/filter_detections_stage.py @@ -22,8 +22,8 @@ from mrc.core import operators as ops import morpheus._lib.stages as _stages -from morpheus._lib.common import FilterSource from morpheus.cli.register_stage import register_stage +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage @@ -69,7 +69,7 @@ class FilterDetectionsStage(SinglePortStage): Threshold to classify, default is 0.5. copy : bool Whether or not to perform a copy. - filter_source : `from morpheus._lib.common.FilterSource`, case_sensitive = False + filter_source : `morpheus.common.FilterSource`, case_sensitive = False Indicate if we are operating on is an output tensor or a field in the DataFrame. Choosing `Auto` will default to `TENSOR` when the incoming message contains output tensorts and `DATAFRAME` otherwise. diff --git a/morpheus/stages/postprocess/validation_stage.py b/morpheus/stages/postprocess/validation_stage.py index 731f4f85b1..589cbf5ed7 100644 --- a/morpheus/stages/postprocess/validation_stage.py +++ b/morpheus/stages/postprocess/validation_stage.py @@ -24,8 +24,8 @@ import cudf -from morpheus._lib.common import FileTypes from morpheus.cli.register_stage import register_stage +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MultiMessage diff --git a/morpheus/utils/directory_watcher.py b/morpheus/utils/directory_watcher.py index d456d258ac..c74afb2d9c 100644 --- a/morpheus/utils/directory_watcher.py +++ b/morpheus/utils/directory_watcher.py @@ -26,7 +26,7 @@ from watchdog.utils.dirsnapshot import EmptyDirectorySnapshot from watchdog.utils.patterns import filter_paths -from morpheus._lib.common import FiberQueue +from morpheus.common import FiberQueue from morpheus.utils.producer_consumer_queue import Closed logger = logging.getLogger(__name__) diff --git a/scripts/compare_data_files.py b/scripts/compare_data_files.py index 3cd3b3e67f..fb4084a1bb 100755 --- a/scripts/compare_data_files.py +++ b/scripts/compare_data_files.py @@ -18,7 +18,7 @@ import logging import sys -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.utils.compare_df import compare_df from morpheus.utils.logger import configure_logging diff --git a/tests/test_abp_kafka.py b/tests/test_abp_kafka.py index 7a46c4e38d..d574c46231 100755 --- a/tests/test_abp_kafka.py +++ b/tests/test_abp_kafka.py @@ -23,7 +23,7 @@ import pandas import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import ConfigFIL from morpheus.config import PipelineModes diff --git a/tests/test_cli.py b/tests/test_cli.py index 2a7ffc136c..1717696556 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,9 +24,9 @@ from mlflow.tracking import fluent import morpheus -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource from morpheus.cli import commands +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import CppConfig diff --git a/tests/test_deserialize_stage_pipe.py b/tests/test_deserialize_stage_pipe.py index 69ae18683e..9bed3d2adf 100755 --- a/tests/test_deserialize_stage_pipe.py +++ b/tests/test_deserialize_stage_pipe.py @@ -19,7 +19,7 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta from morpheus.pipeline import LinearPipeline diff --git a/tests/test_dfp_kafka.py b/tests/test_dfp_kafka.py index 300e585ac7..4a3ce82d1a 100755 --- a/tests/test_dfp_kafka.py +++ b/tests/test_dfp_kafka.py @@ -25,8 +25,8 @@ import pandas as pd import pytest -from morpheus._lib.common import FileTypes from morpheus.cli import commands +from morpheus.common import FileTypes from morpheus.config import ConfigAutoEncoder from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df diff --git a/tests/test_file_in_out.py b/tests/test_file_in_out.py index ffbf39355e..03d3583ead 100755 --- a/tests/test_file_in_out.py +++ b/tests/test_file_in_out.py @@ -19,7 +19,7 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.io.serializers import df_to_csv from morpheus.messages import MessageMeta diff --git a/tests/test_filter_detections_stage.py b/tests/test_filter_detections_stage.py index 8f5de6c21c..79636c8878 100755 --- a/tests/test_filter_detections_stage.py +++ b/tests/test_filter_detections_stage.py @@ -20,8 +20,8 @@ import cupy as cp import pytest -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MultiResponseMessage from morpheus.messages import ResponseMemory diff --git a/tests/test_filter_detections_stage_column_pipe.py b/tests/test_filter_detections_stage_column_pipe.py index b651735dea..b859c5cf44 100755 --- a/tests/test_filter_detections_stage_column_pipe.py +++ b/tests/test_filter_detections_stage_column_pipe.py @@ -19,8 +19,8 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes -from morpheus._lib.common import FilterSource +from morpheus.common import FileTypes +from morpheus.common import FilterSource from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage diff --git a/tests/test_kafka_source_stage_pipe.py b/tests/test_kafka_source_stage_pipe.py index 64797ed50c..7af133e539 100644 --- a/tests/test_kafka_source_stage_pipe.py +++ b/tests/test_kafka_source_stage_pipe.py @@ -22,7 +22,7 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline.linear_pipeline import LinearPipeline diff --git a/tests/test_message_meta.py b/tests/test_message_meta.py index 689c0339b1..fa39ca7f20 100644 --- a/tests/test_message_meta.py +++ b/tests/test_message_meta.py @@ -22,7 +22,7 @@ import cudf -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages.message_meta import MessageMeta from utils import TEST_DIRS diff --git a/tests/test_multi_message.py b/tests/test_multi_message.py index a8161c0ab0..4078056eab 100644 --- a/tests/test_multi_message.py +++ b/tests/test_multi_message.py @@ -28,7 +28,7 @@ import cudf -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.response_memory import ResponseMemory diff --git a/tests/test_multi_segment.py b/tests/test_multi_segment.py index bf28980616..6b37c5a4cd 100644 --- a/tests/test_multi_segment.py +++ b/tests/test_multi_segment.py @@ -18,7 +18,7 @@ import numpy as np -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline import LinearPipeline diff --git a/tests/test_nonlinear_pipeline.py b/tests/test_nonlinear_pipeline.py index d6daee739d..0a64807568 100755 --- a/tests/test_nonlinear_pipeline.py +++ b/tests/test_nonlinear_pipeline.py @@ -24,7 +24,7 @@ import cudf -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta diff --git a/tests/test_phishing_kafka.py b/tests/test_phishing_kafka.py index e9c0bb3feb..69b198b80a 100755 --- a/tests/test_phishing_kafka.py +++ b/tests/test_phishing_kafka.py @@ -23,7 +23,7 @@ import pandas import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.io.utils import filter_null_data diff --git a/tests/test_preallocation_pipe.py b/tests/test_preallocation_pipe.py index 303932d471..c01811cbc6 100755 --- a/tests/test_preallocation_pipe.py +++ b/tests/test_preallocation_pipe.py @@ -21,8 +21,8 @@ import cudf -from morpheus._lib.common import TypeId -from morpheus._lib.common import tyepid_to_numpy_str +from morpheus.common import TypeId +from morpheus.common import tyepid_to_numpy_str from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage diff --git a/tests/test_serialize_stage_pipe.py b/tests/test_serialize_stage_pipe.py index 6a31f4e8d6..5791d17b8e 100755 --- a/tests/test_serialize_stage_pipe.py +++ b/tests/test_serialize_stage_pipe.py @@ -19,7 +19,7 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage diff --git a/tests/test_sid_kafka.py b/tests/test_sid_kafka.py index be3f64c21b..d97003cc02 100755 --- a/tests/test_sid_kafka.py +++ b/tests/test_sid_kafka.py @@ -23,7 +23,7 @@ import pandas import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.io.utils import filter_null_data diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index cacce288a4..eeb6e2b1b3 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -21,7 +21,7 @@ import numpy as np import pytest -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.inference_memory import InferenceMemoryAE diff --git a/tests/test_write_to_kafka_stage_pipe.py b/tests/test_write_to_kafka_stage_pipe.py index 80878d79a6..f424852752 100644 --- a/tests/test_write_to_kafka_stage_pipe.py +++ b/tests/test_write_to_kafka_stage_pipe.py @@ -22,7 +22,7 @@ import cudf -from morpheus._lib.common import FileTypes +from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage diff --git a/tests/utils.py b/tests/utils.py index 05992f0540..82490be8d4 100755 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,8 +27,8 @@ import cudf import morpheus -from morpheus._lib.common import FileTypes from morpheus.cli.register_stage import register_stage +from morpheus.common import FileTypes from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.io.serializers import df_to_csv