New tests

tests/test_file_in_out_stage_pipe.py

@@ -110,3 +110,28 @@ def test_file_rw_multi_segment_pipe(tmp_path, config, output_type):
     # Somehow 0.7 ends up being 0.7000000000000001
     output_data = np.around(output_data, 2)
     assert output_data.tolist() == input_data.tolist()
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("input_file",
+                         [
+                             os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv"),
+                             os.path.join(TEST_DIRS.tests_data_dir, "filter_probs_w_id_col.csv")
+                         ])
+def test_file_rw_index_pipe(tmp_path, config, input_file):
+    out_file = os.path.join(tmp_path, 'results.csv')
+
+    pipe = LinearPipeline(config)
+    pipe.set_source(FileSourceStage(config, filename=input_file))
+    pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False, include_index_col=False))
+    pipe.run()
+
+    assert_path_exists(out_file)
+
+    validation_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv")
+    validation_data = np.loadtxt(validation_file, delimiter=",", skiprows=1)
+    output_data = np.loadtxt(out_file, delimiter=",", skiprows=1)
+
+    # Somehow 0.7 ends up being 0.7000000000000001
+    output_data = np.around(output_data, 2)
+    assert output_data.tolist() == validation_data.tolist()

tests/test_message_meta.py

@@ -17,10 +17,12 @@
 import operator
 import os
 
+import numpy as np
 import pytest
 
 from morpheus._lib.common import FileTypes
 from morpheus.io.deserializers import read_file_to_df
+from morpheus.io.serializers import df_to_csv
 from morpheus.messages.message_meta import MessageMeta
 from utils import TEST_DIRS
 
@@ -67,3 +69,19 @@ def test_copy_dataframe(config):
 
     assert meta.copy_dataframe()['v2'][3] != 47
     assert meta.df != 47
+
+
+def test_make_from_file(config, tmp_path):
+    input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs_w_id_col.csv")
+    out_file = os.path.join(tmp_path, 'results.csv')
+    meta = MessageMeta.make_from_file(input_file)
+    with meta.mutable_dataframe() as df:
+        assert list(df.columns) == ['v1', 'v2', 'v3', 'v4']
+
+        with open(out_file, 'w') as fh:
+            fh.writelines(df_to_csv(df, include_header=True, include_index_col=True))
+
+    input_data = np.loadtxt(input_file, delimiter=",", skiprows=1)
+    output_data = np.loadtxt(out_file, delimiter=",", skiprows=1)
+    output_data = np.around(output_data, 2)
+    assert output_data.tolist() == input_data.tolist()