Reorganize the python package files

examples/abp_pcap_detection/abp_pcap_preprocessing.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,14 +23,15 @@
 
 import morpheus._lib.stages as neos
 from morpheus.config import Config
-from morpheus.pipeline.messages import InferenceMemoryFIL
-from morpheus.pipeline.messages import MultiInferenceFILMessage
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.preprocessing import PreprocessBaseStage
+from morpheus.messages import InferenceMemoryFIL
+from morpheus.messages import MultiInferenceFILMessage
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiMessage
+from morpheus.stages.preprocess.preprocessing import PreprocessBaseStage
 
 
 class AbpPcapPreprocessingStage(PreprocessBaseStage):
+
     def __init__(self, c: Config):
         super().__init__(c)
 

examples/abp_pcap_detection/run.py

@@ -23,14 +23,14 @@
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
-from morpheus.pipeline.general_stages import AddClassificationsStage
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.inference.inference_triton import TritonInferenceStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.serialize import SerializeStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.pipeline import LinearPipeline
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.pipeline.linear_pipeline import LinearPipeline
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage
+from morpheus.stages.postprocess.serialize_stage import SerializeStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 from morpheus.utils.logging import configure_logging
 
 

examples/gnn_fraud_detection_pipeline/run.py

@@ -16,20 +16,20 @@
 
 import click
 import psutil
-from stages.classification_stage import ClassificationStage
-from stages.graph_construction_stage import FraudGraphConstructionStage
-from stages.graph_sage_stage import GraphSAGEStage
 
 from morpheus.config import Config
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.serialize import SerializeStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.pipeline import LinearPipeline
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.pipeline.linear_pipeline import LinearPipeline
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.postprocess.serialize_stage import SerializeStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 from morpheus.utils.logging import configure_logging
+from stages.classification_stage import ClassificationStage
+from stages.graph_construction_stage import FraudGraphConstructionStage
+from stages.graph_sage_stage import GraphSAGEStage
 
 
 @click.command()

examples/gnn_fraud_detection_pipeline/stages/classification_stage.py

@@ -1,20 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import typing
 
 import neo
 
-import cudf
 import cuml
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 from .graph_sage_stage import GraphSAGEMultiMessage
 
 
 class ClassificationStage(SinglePortStage):
+
     def __init__(self, c: Config, model_xgb_file: str):
         super().__init__(c)
 

examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py

@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import dataclasses
 import typing
 
@@ -9,9 +24,9 @@
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 
 @dataclasses.dataclass
@@ -20,6 +35,7 @@ class FraudGraphMultiMessage(MultiMessage):
 
 
 class FraudGraphConstructionStage(SinglePortStage):
+
     def __init__(self, c: Config, training_file: str):
         super().__init__(c)
         self._training_data = cudf.read_csv(training_file)

examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py

@@ -1,3 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import dataclasses
 import typing
 
@@ -8,10 +23,9 @@
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MultiMessage
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 from .graph_construction_stage import FraudGraphMultiMessage
 
@@ -23,15 +37,14 @@ class GraphSAGEMultiMessage(MultiMessage):
 
 
 class GraphSAGEStage(SinglePortStage):
-    def __init__(
-        self,
-        c: Config,
-        model_hinsage_file: str,
-        batch_size: int = 5,
-        sample_size = [2, 32],
-        record_id:str = "index",
-        target_node:str ="transaction"
-    ):
+
+    def __init__(self,
+                 c: Config,
+                 model_hinsage_file: str,
+                 batch_size: int = 5,
+                 sample_size=[2, 32],
+                 record_id: str = "index",
+                 target_node: str = "transaction"):
         super().__init__(c)
         self._keras_model = tf.keras.models.load_model(model_hinsage_file)
         self._batch_size = batch_size
@@ -57,7 +70,6 @@ def _inductive_step_hinsage(
         generator = HinSAGENodeGenerator(graph, self._batch_size, self._sample_size, head_node_type=self._target_node)
         test_gen_not_shuffled = generator.flow(node_identifiers, shuffle=False)
 
-
         inductive_emb = trained_model.predict(test_gen_not_shuffled)
         inductive_emb = cudf.DataFrame(inductive_emb, index=node_identifiers)
 

examples/log_parsing/inference.py

@@ -20,27 +20,27 @@
 import neo
 import numpy as np
 import tritonclient.grpc as tritonclient
+from neo.core import operators as ops
+from scipy.special import softmax
+
 from messages import MultiPostprocLogParsingMessage
 from messages import MultiResponseLogParsingMessage
 from messages import PostprocMemoryLogParsing
 from messages import ResponseMemoryLogParsing
-from neo.core import operators as ops
-from scipy.special import softmax
-
 from morpheus.config import Config
-from morpheus.pipeline.inference.inference_stage import InferenceStage
-from morpheus.pipeline.inference.inference_stage import InferenceWorker
-from morpheus.pipeline.inference.inference_triton import InputWrapper
-from morpheus.pipeline.inference.inference_triton import TritonInferenceWorker
-from morpheus.pipeline.messages import InferenceMemory
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import InferenceMemory
+from morpheus.messages import MultiInferenceMessage
+from morpheus.pipeline.stream_pair import StreamPair
+from morpheus.stages.inference.inference_stage import InferenceStage
+from morpheus.stages.inference.inference_stage import InferenceWorker
+from morpheus.stages.inference.triton_inference_stage import InputWrapper
+from morpheus.stages.inference.triton_inference_stage import _TritonInferenceWorker
 from morpheus.utils.producer_consumer_queue import ProducerConsumerQueue
 
 logger = logging.getLogger(__name__)
 
 
-class TritonInferenceLogParsing(TritonInferenceWorker):
+class TritonInferenceLogParsing(_TritonInferenceWorker):
     """
     This class extends TritonInference to deal with scenario-specific NLP models inference requests like building
     response.

examples/log_parsing/messages.py

@@ -16,15 +16,15 @@
 
 import cupy as cp
 
-from morpheus.pipeline.messages import DataClassProp
-from morpheus.pipeline.messages import InferenceMemory
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiResponseMessage
-from morpheus.pipeline.messages import ResponseMemory
-from morpheus.pipeline.messages import get_input
-from morpheus.pipeline.messages import get_output
-from morpheus.pipeline.messages import set_input
-from morpheus.pipeline.messages import set_output
+from morpheus.messages import DataClassProp
+from morpheus.messages import InferenceMemory
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiResponseMessage
+from morpheus.messages import ResponseMemory
+from morpheus.messages import get_input
+from morpheus.messages import get_output
+from morpheus.messages import set_input
+from morpheus.messages import set_output
 
 
 @dataclasses.dataclass

examples/log_parsing/postprocessing.py

@@ -19,13 +19,13 @@
 import neo
 import numpy as np
 import pandas as pd
+
 from messages import MultiPostprocLogParsingMessage
 from messages import MultiResponseLogParsingMessage
-
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages import MessageMeta
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 
 class LogParsingPostProcessingStage(SinglePortStage):

examples/log_parsing/preprocessing.py

@@ -22,11 +22,11 @@
 
 import morpheus._lib.stages as neos
 from morpheus.config import Config
-from morpheus.pipeline.messages import InferenceMemoryNLP
-from morpheus.pipeline.messages import MultiInferenceMessage
-from morpheus.pipeline.messages import MultiInferenceNLPMessage
-from morpheus.pipeline.messages import MultiMessage
-from morpheus.pipeline.preprocessing import PreprocessBaseStage
+from morpheus.messages import InferenceMemoryNLP
+from morpheus.messages import MultiInferenceMessage
+from morpheus.messages import MultiInferenceNLPMessage
+from morpheus.messages import MultiMessage
+from morpheus.stages.preprocess.preprocessing import PreprocessBaseStage
 from morpheus.utils.cudf_subword_helper import tokenize_text_series
 
 
@@ -79,7 +79,6 @@ def __init__(self,
         self._do_lower_case = do_lower_case
         self._add_special_tokens = add_special_tokens
 
-
     @property
     def name(self) -> str:
         return "preprocess-logparsing"

examples/log_parsing/run.py

@@ -23,11 +23,11 @@
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
 from morpheus.pipeline import LinearPipeline
-from morpheus.pipeline.general_stages import BufferStage
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.preprocessing import DeserializeStage
+from morpheus.stages.general.general_stages import BufferStage
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 
 
 @click.command()