Cleanup all uses of import morpheus._lib.common (nv-morpheus#787)

This PR removes all uses of `from morpheus._lib.common import XXX` and replaces them with `from morpheus.common import XXX`. Users should not ever import directly from `morpheus._lib`. Any public facing symbols that are C++ only (and thus exist in `morpheus._lib` should be re-exported from the matching python module's `__init__.py`.

The main reason for this is:

- Avoid errors that can arise from unintentionally using the C++ version of a class instead of a Python version.
- Eliminate the possibility of circular imports
- Hide the use of the non-public `morpheus._lib` namespace

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - David Gardner (https://github.com/dagardner-nv)

URL: nv-morpheus#787

docs/source/developer_guide/guides/2_real_world_phishing.md

@@ -109,7 +109,7 @@ import typing
 
 import mrc
 
-from morpheus._lib.common import TypeId
+from morpheus.common import TypeId
 from morpheus.cli.register_stage import register_stage
 from morpheus.config import Config
 from morpheus.config import PipelineModes
@@ -435,7 +435,7 @@ pipeline.add_stage(WriteToFileStage(config, filename=results_file, overwrite=Tru
 
 Note that we didn't specify the output format. In our example, the result file contains the extension `.jsonlines`. Morpheus will infer the output format based on the extension. At time of writing the extensions that Morpheus will infer are: `.csv`, `.json` & `.jsonlines`
 
-To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus._lib.common.FileTypes`. Current values defined are:
+To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus.common.FileTypes`. Current values defined are:
 * `FileTypes.Auto`
 * `FileTypes.JSON`
 * `FileTypes.CSV`

docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md

@@ -225,7 +225,7 @@ The `DFPFileToDataFrameStage` ([examples/digital_fingerprinting/production/morph
 | `c` | `morpheus.config.Config` | Morpheus config object |
 | `schema` | `DataFrameInputSchema` | Schema specifying columns to load, along with any necessary renames and data type conversions  |
 | `filter_null` | `bool` | Optional: Whether to filter null rows after loading, by default True. |
-| `file_type` | `morpheus._lib.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, and `FileTypes.JSON`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension |
+| `file_type` | `morpheus.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, and `FileTypes.JSON`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension |
 | `parser_kwargs` | `dict` or `None` | Optional: additional keyword arguments to be passed into the `DataFrame` parser, currently this is going to be either [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html) or [`pandas.read_json`](https://pandas.pydata.org/docs/reference/api/pandas.read_json.html) |
 | `cache_dir` | `str` | Optional: path to cache location, defaults to `./.cache/dfp` |
 

examples/developer_guide/2_1_real_world_phishing/recipient_features_stage.py

@@ -17,8 +17,8 @@
 
 import mrc
 
-from morpheus._lib.common import TypeId
 from morpheus.cli.register_stage import register_stage
+from morpheus.common import TypeId
 from morpheus.config import Config
 from morpheus.config import PipelineModes
 from morpheus.messages.message_meta import MessageMeta

examples/developer_guide/2_2_rabbitmq/read_simple.py

@@ -18,7 +18,7 @@
 
 from rabbitmq_source_stage import RabbitMQSourceStage
 
-from morpheus._lib.common import FileTypes
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.pipeline import LinearPipeline
 from morpheus.stages.general.monitor_stage import MonitorStage

examples/developer_guide/4_rabbitmq_cpp_stage/read_simple.py

@@ -19,7 +19,7 @@
 import click
 from rabbitmq_source_stage import RabbitMQSourceStage
 
-from morpheus._lib.common import FileTypes
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.pipeline import LinearPipeline

examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py

@@ -42,8 +42,8 @@
 from benchmarks.dfp_config import get_duo_source_schema
 from benchmarks.dfp_config import load_json
 from benchmarks.dfp_config import set_mlflow_tracking_uri
-from morpheus._lib.common import FileTypes
-from morpheus._lib.common import FilterSource
+from morpheus.common import FileTypes
+from morpheus.common import FilterSource
 from morpheus.config import Config
 from morpheus.pipeline.linear_pipeline import LinearPipeline
 from morpheus.stages.general.linear_modules_stage import LinearModulesStage

examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_file_to_df.py

@@ -32,7 +32,7 @@
 
 import cudf
 
-from morpheus._lib.common import FileTypes
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.io.deserializers import read_file_to_df
 from morpheus.pipeline.preallocator_mixin import PreallocatorMixin

examples/digital_fingerprinting/production/morpheus/dfp_azure_inference.ipynb

@@ -55,8 +55,8 @@
     "from dfp.stages.multi_file_source import MultiFileSource\n",
     "from dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
-    "from morpheus._lib.common import FileTypes\n",
-    "from morpheus._lib.common import FilterSource\n",
+    "from morpheus.common import FileTypes\n",
+    "from morpheus.common import FilterSource\n",
     "from morpheus.cli.utils import get_log_levels\n",
     "from morpheus.cli.utils import get_package_relative_file\n",
     "from morpheus.cli.utils import load_labels_file\n",
@@ -463,7 +463,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.10.6"
   },
   "vscode": {
    "interpreter": {

examples/digital_fingerprinting/production/morpheus/dfp_azure_modules_pipeline.py

@@ -36,9 +36,9 @@
 from dfp.utils.schema_utils import Schema
 from dfp.utils.schema_utils import SchemaBuilder
 
-from morpheus._lib.common import FilterSource
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FilterSource
 from morpheus.config import Config
 from morpheus.pipeline import LinearPipeline
 from morpheus.stages.general.linear_modules_stage import LinearModulesStage

examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py

@@ -36,12 +36,12 @@
 from dfp.stages.multi_file_source import MultiFileSource
 from dfp.utils.regex_utils import iso_date_regex
 
-from morpheus._lib.common import FileTypes
-from morpheus._lib.common import FilterSource
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
 from morpheus.cli.utils import load_labels_file
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FileTypes
+from morpheus.common import FilterSource
 from morpheus.config import Config
 from morpheus.config import ConfigAutoEncoder
 from morpheus.config import CppConfig

examples/digital_fingerprinting/production/morpheus/dfp_azure_training.ipynb

@@ -55,7 +55,7 @@
     "from dfp.stages.multi_file_source import MultiFileSource\n",
     "from dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
-    "from morpheus._lib.common import FileTypes\n",
+    "from morpheus.common import FileTypes\n",
     "from morpheus.cli.utils import get_log_levels\n",
     "from morpheus.cli.utils import get_package_relative_file\n",
     "from morpheus.cli.utils import load_labels_file\n",

examples/digital_fingerprinting/production/morpheus/dfp_duo_inference.ipynb

@@ -56,8 +56,8 @@
     "from dfp.stages.multi_file_source import MultiFileSource\n",
     "from dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
-    "from morpheus._lib.common import FileTypes\n",
-    "from morpheus._lib.common import FilterSource\n",
+    "from morpheus.common import FileTypes\n",
+    "from morpheus.common import FilterSource\n",
     "from morpheus.cli.utils import get_log_levels\n",
     "from morpheus.cli.utils import get_package_relative_file\n",
     "from morpheus.cli.utils import load_labels_file\n",

examples/digital_fingerprinting/production/morpheus/dfp_duo_modules_pipeline.py

@@ -36,9 +36,9 @@
 from dfp.utils.schema_utils import Schema
 from dfp.utils.schema_utils import SchemaBuilder
 
-from morpheus._lib.common import FilterSource
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FilterSource
 from morpheus.config import Config
 from morpheus.pipeline import LinearPipeline
 from morpheus.stages.general.linear_modules_stage import LinearModulesStage

examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py

@@ -36,12 +36,12 @@
 from dfp.stages.multi_file_source import MultiFileSource
 from dfp.utils.regex_utils import iso_date_regex
 
-from morpheus._lib.common import FileTypes
-from morpheus._lib.common import FilterSource
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
 from morpheus.cli.utils import load_labels_file
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FileTypes
+from morpheus.common import FilterSource
 from morpheus.config import Config
 from morpheus.config import ConfigAutoEncoder
 from morpheus.config import CppConfig

examples/digital_fingerprinting/production/morpheus/dfp_duo_training.ipynb

@@ -55,7 +55,7 @@
     "from dfp.stages.multi_file_source import MultiFileSource\n",
     "from dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
-    "from morpheus._lib.common import FileTypes\n",
+    "from morpheus.common import FileTypes\n",
     "from morpheus.cli.utils import get_log_levels\n",
     "from morpheus.cli.utils import get_package_relative_file\n",
     "from morpheus.cli.utils import load_labels_file\n",

examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py

@@ -36,11 +36,11 @@
 from dfp.stages.multi_file_source import MultiFileSource
 from dfp.utils.regex_utils import iso_date_regex
 
-from morpheus._lib.common import FileTypes
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
 from morpheus.cli.utils import load_labels_file
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.config import ConfigAutoEncoder
 from morpheus.config import CppConfig

examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py

@@ -36,11 +36,11 @@
 from dfp.stages.multi_file_source import MultiFileSource
 from dfp.utils.regex_utils import iso_date_regex
 
-from morpheus._lib.common import FileTypes
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
 from morpheus.cli.utils import load_labels_file
 from morpheus.cli.utils import parse_log_level
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.config import ConfigAutoEncoder
 from morpheus.config import CppConfig

examples/ransomware_detection/stages/create_features.py

@@ -15,12 +15,12 @@
 import typing
 
 import mrc
-from common.data_models import FeatureConfig
-from common.feature_extractor import FeatureExtractor
 from mrc.core import operators as ops
 
 from dask.distributed import Client
 
+from common.data_models import FeatureConfig
+from common.feature_extractor import FeatureExtractor
 from morpheus.cli.register_stage import register_stage
 from morpheus.config import Config
 from morpheus.config import PipelineModes

examples/sid_visualization/run.py

@@ -19,7 +19,7 @@
 import click
 import mrc
 
-from morpheus._lib.common import FileTypes
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
@@ -54,7 +54,7 @@ class NLPVizFileSource(PreallocatorMixin, SingleOutputSource):
     iterative: boolean
         Iterative mode will emit dataframes one at a time. Otherwise a list of dataframes is emitted. Iterative mode is
         good for interleaving source stages.
-    file_type : `morpheus._lib.common.FileTypes`, default = 'auto'
+    file_type : `morpheus.common.FileTypes`, default = 'auto'
         Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension.
         Supported extensions: 'json', 'csv'
     repeat: int, default = 1
@@ -167,6 +167,7 @@ def run_pipeline(debug, use_cpp, num_threads, input_file, max_batch_size, model_
     config.num_threads = num_threads
     config.pipeline_batch_size = max_batch_size
     config.feature_length = 256
+    config.model_max_batch_size = 32
     config.class_labels = load_labels_file(get_data_file_path("data/labels_nlp.txt"))
 
     # Create a linear pipeline object.

morpheus/cli/commands.py

@@ -48,7 +48,7 @@
 
 DEFAULT_CONFIG = Config()
 
-# List all of the options in from morpheus._lib.common.FileTypes without importing the object. This slows down
+# List all of the options in from morpheus.common.FileTypes without importing the object. This slows down
 # autocomplete too much.
 FILE_TYPE_NAMES = ["auto", "csv", "json"]
 

morpheus/cli/utils.py

@@ -38,7 +38,7 @@
 
 def str_to_file_type(file_type_str: str):
     # Delay FileTypes since this will import ._lib
-    from morpheus._lib.common import FileTypes
+    from morpheus.common import FileTypes
     file_type_members = {name.lower(): t for (name, t) in FileTypes.__members__.items()}
 
     return file_type_members[file_type_str]

morpheus/common/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Module for common utilities and classes in the Morpheus library.
+"""
+
+# Export symbols from the morpheus._lib.common module. Users should never be directly importing morpheus._lib
+from morpheus._lib.common import determine_file_type
+from morpheus._lib.common import FiberQueue
+from morpheus._lib.common import FileTypes
+from morpheus._lib.common import FilterSource
+from morpheus._lib.common import read_file_to_df
+from morpheus._lib.common import Tensor
+from morpheus._lib.common import tyepid_to_numpy_str
+from morpheus._lib.common import TypeId
+
+__all__ = [
+    "determine_file_type",
+    "FiberQueue",
+    "FileTypes",
+    "FilterSource",
+    "read_file_to_df",
+    "Tensor",
+    "tyepid_to_numpy_str",
+    "TypeId",
+]

morpheus/io/deserializers.py

@@ -19,9 +19,9 @@
 
 import cudf
 
-from morpheus._lib.common import FileTypes
-from morpheus._lib.common import determine_file_type
-from morpheus._lib.common import read_file_to_df as read_file_to_df_cpp
+from morpheus.common import FileTypes
+from morpheus.common import determine_file_type
+from morpheus.common import read_file_to_df as read_file_to_df_cpp
 from morpheus.config import CppConfig
 from morpheus.io.utils import filter_null_data
 
@@ -49,7 +49,7 @@ def read_file_to_df(file_name: str,
     ----------
     file_name : str
         File to read.
-    file_type : `morpheus._lib.common.FileTypes`
+    file_type : `morpheus.common.FileTypes`
         Type of file. Leave as Auto to determine from the extension.
     parser_kwargs : dict, optional
         Any argument to pass onto the parse, by default {}. Ignored when C++ execution is enabled and `df_type="cudf"`

morpheus/modules/file_to_df.py

@@ -30,8 +30,8 @@
 
 import cudf
 
-from morpheus._lib.common import FileTypes
 from morpheus.cli.utils import str_to_file_type
+from morpheus.common import FileTypes
 from morpheus.io.deserializers import read_file_to_df
 from morpheus.utils.column_info import process_dataframe
 from morpheus.utils.module_ids import FILE_TO_DF

morpheus/modules/filter_detections.py

@@ -22,7 +22,7 @@
 import typing_utils
 from mrc.core import operators as ops
 
-from morpheus._lib.common import FilterSource
+from morpheus.common import FilterSource
 from morpheus.messages import MultiMessage
 from morpheus.messages.multi_response_message import MultiResponseMessage
 from morpheus.utils.module_ids import FILTER_DETECTIONS

morpheus/modules/write_to_file.py

@@ -22,8 +22,8 @@
 
 import cudf
 
-from morpheus._lib.common import FileTypes
-from morpheus._lib.common import determine_file_type
+from morpheus.common import FileTypes
+from morpheus.common import determine_file_type
 from morpheus.io import serializers
 from morpheus.messages.message_meta import MessageMeta
 from morpheus.utils.module_ids import MODULE_NAMESPACE

morpheus/pipeline/preallocator_mixin.py

@@ -23,8 +23,8 @@
 
 import cudf
 
-from morpheus._lib.common import TypeId
-from morpheus._lib.common import tyepid_to_numpy_str
+from morpheus.common import TypeId
+from morpheus.common import tyepid_to_numpy_str
 from morpheus.config import CppConfig
 from morpheus.messages import MessageMeta
 from morpheus.messages import MultiMessage

morpheus/pipeline/stream_wrapper.py

@@ -409,7 +409,7 @@ def _create_ports(self, input_count: int, output_count: int):
     def get_needed_columns(self):
         """
         Stages which need to have columns inserted into the dataframe, should populate the `self._needed_columns`
-        dictionary with mapping of column names to `morpheus._lib.common.TypeId`. This will ensure that the columns are
+        dictionary with mapping of column names to `morpheus.common.TypeId`. This will ensure that the columns are
         allocated and populated with null values.
         """
         return self._needed_columns.copy()

morpheus/stages/input/autoencoder_source_stage.py

@@ -22,7 +22,7 @@
 import pandas as pd
 from mrc.core import operators as ops
 
-from morpheus._lib.common import FileTypes
+from morpheus.common import FileTypes
 from morpheus.config import Config
 from morpheus.messages import UserMessageMeta
 from morpheus.pipeline.preallocator_mixin import PreallocatorMixin
@@ -55,7 +55,7 @@ class AutoencoderSourceStage(PreallocatorMixin, SingleOutputSource):
         files. Any new files that are added that match the glob will then be processed.
     max_files: int, default = -1
         Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited.
-    file_type : `morpheus._lib.common.FileTypes`, default = 'FileTypes.Auto'.
+    file_type : `morpheus.common.FileTypes`, default = 'FileTypes.Auto'.
         Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension.
         Supported extensions: 'json', 'csv'
     repeat: int, default = 1

morpheus/stages/input/azure_source_stage.py

@@ -47,7 +47,7 @@ class AzureSourceStage(AutoencoderSourceStage):
         files. Any new files that are added that match the glob will then be processed.
     max_files: int, default = -1
         Max number of files to read. Useful for debugging to limit startup time. Default value of -1 is unlimited.
-    file_type : `morpheus._lib.common.FileTypes`, default = 'FileTypes.Auto'.
+    file_type : `morpheus.common.FileTypes`, default = 'FileTypes.Auto'.
         Indicates what type of file to read. Specifying 'auto' will determine the file type from the extension.
         Supported extensions: 'json', 'csv'
     repeat: int, default = 1