Reorganize the python package files (#98)

This refactors all of the python package files to improve the directory structure, separate classes into their own files, improve readability and reduce merge conflicts. The main morpheus package will now be subdivided into the following submodules:

```
morpheus/
   _lib/
   io/
   messages/
   pipelines/
   stages/
      general/
      inference/
      input/
      output/
      postprocess/
      preprocess
   utils/
```

The largest differences are coming from separating the `morpheus/pipeline.py` into separate files and breaking all stages into their own package (resulting in quite a few new files).

Remaining questions:
- [x] How to handle package reloads for `morpheus.messages`
- [ ] What to call utilities? `utils`? or `utilities`?

Closes #98

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - David Gardner (https://github.com/dagardner-nv)
  - Devin Robison (https://github.com/drobison00)
  - Bartley Richardson (https://github.com/BartleyR)

URL: #98

examples/abp_pcap_detection/abp_pcap_preprocessing.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,14 +23,15 @@
 
 import morpheus._lib.stages as neos
 from morpheus.config import Config
-from morpheus.pipeline.messages import InferenceMemoryFIL
-from morpheus.pipeline.messages import MultiInferenceFILMessage
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.preprocessing import PreprocessBaseStage
+from morpheus.messages import InferenceMemoryFIL
+from morpheus.messages import MultiInferenceFILMessage
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiMessage
+from morpheus.stages.preprocess.preprocessing import PreprocessBaseStage
 
 
 class AbpPcapPreprocessingStage(PreprocessBaseStage):
+
     def __init__(self, c: Config):
         super().__init__(c)
 

examples/abp_pcap_detection/run.py

@@ -23,14 +23,14 @@
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
-from morpheus.pipeline.general_stages import AddClassificationsStage
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.inference.inference_triton import TritonInferenceStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.serialize import SerializeStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.pipeline import LinearPipeline
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.pipeline.linear_pipeline import LinearPipeline
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage
+from morpheus.stages.postprocess.serialize_stage import SerializeStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 from morpheus.utils.logging import configure_logging
 
 

examples/gnn_fraud_detection_pipeline/run.py

@@ -16,20 +16,20 @@
 
 import click
 import psutil
-from stages.classification_stage import ClassificationStage
-from stages.graph_construction_stage import FraudGraphConstructionStage
-from stages.graph_sage_stage import GraphSAGEStage
 
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.serialize import SerializeStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.pipeline import LinearPipeline
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.pipeline.linear_pipeline import LinearPipeline
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.postprocess.serialize_stage import SerializeStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 from morpheus.utils.logging import configure_logging
+from stages.classification_stage import ClassificationStage
+from stages.graph_construction_stage import FraudGraphConstructionStage
+from stages.graph_sage_stage import GraphSAGEStage
 
 
 @click.command()

examples/gnn_fraud_detection_pipeline/stages/classification_stage.py

@@ -1,20 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import typing
 
 import neo
 
-import cudf
 import cuml
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 from .graph_sage_stage import GraphSAGEMultiMessage
 
 
 class ClassificationStage(SinglePortStage):
+
     def __init__(self, c: Config, model_xgb_file: str):
         super().__init__(c)
 

examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py

@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import dataclasses
 import typing
 
@@ -9,9 +24,9 @@
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 
 @dataclasses.dataclass
@@ -20,6 +35,7 @@ class FraudGraphMultiMessage(MultiMessage):
 
 
 class FraudGraphConstructionStage(SinglePortStage):
+
     def __init__(self, c: Config, training_file: str):
         super().__init__(c)
         self._training_data = cudf.read_csv(training_file)

examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py

@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import dataclasses
 import typing
 
@@ -8,10 +23,9 @@
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 from .graph_construction_stage import FraudGraphMultiMessage
 
@@ -23,15 +37,14 @@ class GraphSAGEMultiMessage(MultiMessage):
 
 
 class GraphSAGEStage(SinglePortStage):
-    def __init__(
-        self,
-        c: Config,
-        model_hinsage_file: str,
-        batch_size: int = 5,
-        sample_size = [2, 32],
-        record_id:str = "index",
-        target_node:str ="transaction"
-    ):
+
+    def __init__(self,
+                 c: Config,
+                 model_hinsage_file: str,
+                 batch_size: int = 5,
+                 sample_size=[2, 32],
+                 record_id: str = "index",
+                 target_node: str = "transaction"):
         super().__init__(c)
         self._keras_model = tf.keras.models.load_model(model_hinsage_file)
         self._batch_size = batch_size
@@ -57,7 +70,6 @@ def _inductive_step_hinsage(
         generator = HinSAGENodeGenerator(graph, self._batch_size, self._sample_size, head_node_type=self._target_node)
         test_gen_not_shuffled = generator.flow(node_identifiers, shuffle=False)
 
-
         inductive_emb = trained_model.predict(test_gen_not_shuffled)
         inductive_emb = cudf.DataFrame(inductive_emb, index=node_identifiers)
 

examples/log_parsing/inference.py

@@ -20,27 +20,27 @@
 import neo
 import numpy as np
 import tritonclient.grpc as tritonclient
+from neo.core import operators as ops
+from scipy.special import softmax
+
 from messages import MultiPostprocLogParsingMessage
 from messages import MultiResponseLogParsingMessage
 from messages import PostprocMemoryLogParsing
 from messages import ResponseMemoryLogParsing
-from neo.core import operators as ops
-from scipy.special import softmax
-
 from morpheus.config import Config
-from morpheus.pipeline.inference.inference_stage import InferenceStage
-from morpheus.pipeline.inference.inference_stage import InferenceWorker
-from morpheus.pipeline.inference.inference_triton import InputWrapper
-from morpheus.pipeline.inference.inference_triton import TritonInferenceWorker
-from morpheus.pipeline.messages import InferenceMemory
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import InferenceMemory
+from morpheus.messages import MultiInferenceMessage
+from morpheus.pipeline.stream_pair import StreamPair
+from morpheus.stages.inference.inference_stage import InferenceStage
+from morpheus.stages.inference.inference_stage import InferenceWorker
+from morpheus.stages.inference.triton_inference_stage import InputWrapper
+from morpheus.stages.inference.triton_inference_stage import _TritonInferenceWorker
 from morpheus.utils.producer_consumer_queue import ProducerConsumerQueue
 
 logger = logging.getLogger(__name__)
 
 
-class TritonInferenceLogParsing(TritonInferenceWorker):
+class TritonInferenceLogParsing(_TritonInferenceWorker):
     """
     This class extends TritonInference to deal with scenario-specific NLP models inference requests like building
     response.

examples/log_parsing/messages.py

@@ -16,15 +16,15 @@
 
 import cupy as cp
 
-from morpheus.pipeline.messages import DataClassProp
-from morpheus.pipeline.messages import InferenceMemory
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiResponseMessage
-from morpheus.pipeline.messages import ResponseMemory
-from morpheus.pipeline.messages import get_input
-from morpheus.pipeline.messages import get_output
-from morpheus.pipeline.messages import set_input
-from morpheus.pipeline.messages import set_output
+from morpheus.messages import DataClassProp
+from morpheus.messages import InferenceMemory
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiResponseMessage
+from morpheus.messages import ResponseMemory
+from morpheus.messages import get_input
+from morpheus.messages import get_output
+from morpheus.messages import set_input
+from morpheus.messages import set_output
 
 
 @dataclasses.dataclass

examples/log_parsing/postprocessing.py

@@ -19,13 +19,13 @@
 import neo
 import numpy as np
 import pandas as pd
+
 from messages import MultiPostprocLogParsingMessage
 from messages import MultiResponseLogParsingMessage
-
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MessageMeta
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 
 class LogParsingPostProcessingStage(SinglePortStage):

examples/log_parsing/preprocessing.py

@@ -22,11 +22,11 @@
 
 import morpheus._lib.stages as neos
 from morpheus.config import Config
-from morpheus.pipeline.messages import InferenceMemoryNLP
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiInferenceNLPMessage
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.preprocessing import PreprocessBaseStage
+from morpheus.messages import InferenceMemoryNLP
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiInferenceNLPMessage
+from morpheus.messages import MultiMessage
+from morpheus.stages.preprocess.preprocessing import PreprocessBaseStage
 from morpheus.utils.cudf_subword_helper import tokenize_text_series
 
 
@@ -79,7 +79,6 @@ def __init__(self,
         self._do_lower_case = do_lower_case
         self._add_special_tokens = add_special_tokens
 
-
     @property
     def name(self) -> str:
         return "preprocess-logparsing"

examples/log_parsing/run.py

@@ -23,11 +23,11 @@
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
 from morpheus.pipeline import LinearPipeline
-from morpheus.pipeline.general_stages import BufferStage
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.stages.general.general_stages import BufferStage
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 
 
 @click.command()