Move common function to util

onair/data_handling/csv_parser.py

@@ -39,11 +39,7 @@ def parse_csv_data(self, data_file):
         return all_data
 
     def parse_meta_data_file(self, meta_data_file, ss_breakdown):
-        parsed_meta_data = extract_meta_data(meta_data_file)
-        if ss_breakdown == False:
-            num_elements = len(parsed_meta_data['subsystem_assignments'])
-            parsed_meta_data['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
-        return parsed_meta_data
+        return extract_meta_data_handle_ss_breakdown(meta_data_file, ss_breakdown)
 
 ##### GETTERS ##################################
 

onair/data_handling/parser_util.py

@@ -12,6 +12,13 @@
 from pandas import to_datetime
 import datetime
 
+def extract_meta_data_handle_ss_breakdown(meta_data_file, ss_breakdown):
+    parsed_meta_data = extract_meta_data(meta_data_file)
+    if ss_breakdown == False:
+        num_elements = len(parsed_meta_data['subsystem_assignments'])
+        parsed_meta_data['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
+    return parsed_meta_data
+
 ## Method to extract configuration data and return 3 dictionaries
 def extract_meta_data(meta_data_file):
     assert meta_data_file != ''

onair/data_handling/redis_adapter.py

@@ -52,11 +52,7 @@ def subscribe_message(self, channel):
             listen_thread.start()
 
     def parse_meta_data_file(self, meta_data_file, ss_breakdown):
-        parsed_meta_data = extract_meta_data(meta_data_file)
-        if ss_breakdown == False:
-            num_elements = len(parsed_meta_data['subsystem_assignments'])
-            parsed_meta_data['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
-        return parsed_meta_data
+        return extract_meta_data_handle_ss_breakdown(meta_data_file, ss_breakdown)
 
     def process_data_file(self, data_file):
         print("Redis Adapter ignoring file")

test/onair/data_handling/test_csv_parser.py

@@ -161,82 +161,26 @@ def test_CSV_parse_csv_data_returns_list_of_data_frames_call_to_iterrows_returns
     assert fake_columns_str.contains.call_args_list[0].args == ('^Unnamed', )
     assert result == expected_result
 
-# CSV parse_meta_data tests # TODO: delete me?
-def test_CSV_parse_meta_data_file_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_when_given_ss_breakdown_does_not_resolve_to_False(mocker, setup_teardown):
+# CSV parse_meta_data tests
+def test_CSV_parse_meta_data_file_returns_call_to_extract_meta_data_handle_ss_breakdown(mocker, setup_teardown):
     # Arrange
     arg_configFile = MagicMock()
-    arg_ss_breakdown = True if pytest.gen.randint(0, 1) else MagicMock()
+    arg_ss_breakdown = MagicMock()
 
     expected_result = MagicMock()
 
-    mocker.patch(csv_parser.__name__ + '.extract_meta_data', return_value=expected_result)
+    mocker.patch(csv_parser.__name__ + '.extract_meta_data_handle_ss_breakdown', return_value=expected_result)
     mocker.patch(csv_parser.__name__ + '.len')
 
     # Act
     result = pytest.cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
 
     # Assert
-    assert csv_parser.extract_meta_data.call_count == 1
-    assert csv_parser.extract_meta_data.call_args_list[0].args == (arg_configFile, )
+    assert csv_parser.extract_meta_data_handle_ss_breakdown.call_count == 1
+    assert csv_parser.extract_meta_data_handle_ss_breakdown.call_args_list[0].args == (arg_configFile, arg_ss_breakdown, )
     assert csv_parser.len.call_count == 0
     assert result == expected_result
 
-def test_CSV_parse_meta_data_file_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_with_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_empty_list_when_len_of_call_value_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_is_0_when_given_ss_breakdown_evaluates_to_False(mocker, setup_teardown):
-    # Arrange
-    arg_configFile = MagicMock()
-    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
-
-    forced_return_extract_meta_data = {}
-    forced_return_len = 0
-    fake_empty_processed_filepath = MagicMock()
-    forced_return_extract_meta_data['subsystem_assignments'] = fake_empty_processed_filepath
-
-    expected_result = []
-
-    mocker.patch(csv_parser.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
-    mocker.patch(csv_parser.__name__ + '.len', return_value=forced_return_len)
-
-    # Act
-    result = pytest.cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
-
-    # Assert
-    assert csv_parser.extract_meta_data.call_count == 1
-    assert csv_parser.extract_meta_data.call_args_list[0].args == (arg_configFile, )
-    assert csv_parser.len.call_count == 1
-    assert csv_parser.len.call_args_list[0].args == (fake_empty_processed_filepath, )
-    assert result['subsystem_assignments'] == expected_result
-
-def test_CSV_parse_meta_data_file_returns_call_to_extract_meta_data_given_metadata_file_and_csv_set_to_True_with_dict_def_subsystem_assignments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_single_item_list_str_MISSION_for_each_item_when_given_ss_breakdown_evaluates_to_False(mocker, setup_teardown):
-    # Arrange
-    arg_configFile = MagicMock()
-    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
-
-    forced_return_extract_meta_data = {}
-    forced_return_process_filepath = MagicMock()
-    fake_processed_filepath = []
-    num_fake_processed_filepaths = pytest.gen.randint(1,10) # arbitrary, from 1 to 10 (0 has own test)
-    for i in range(num_fake_processed_filepaths):
-        fake_processed_filepath.append(i)
-    forced_return_extract_meta_data['subsystem_assignments'] = fake_processed_filepath
-    forced_return_len = num_fake_processed_filepaths
-
-    expected_result = []
-    for i in range(num_fake_processed_filepaths):
-        expected_result.append(['MISSION'])
-
-    mocker.patch(csv_parser.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
-    mocker.patch(csv_parser.__name__ + '.len', return_value=forced_return_len)
-
-    # Act
-    result = pytest.cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
-
-    # Assert
-    assert csv_parser.extract_meta_data.call_count == 1
-    assert csv_parser.extract_meta_data.call_args_list[0].args == (arg_configFile, )
-    assert csv_parser.len.call_count == 1
-    assert csv_parser.len.call_args_list[0].args == (fake_processed_filepath, )
-    assert result['subsystem_assignments'] == expected_result
-
 # CSV get_vehicle_metadata tests
 def test_CSV_get_vehicle_metadata_returns_list_of_headers_and_list_of_test_assignments(setup_teardown):
     # Arrange

test/onair/data_handling/test_parser_util.py

@@ -13,6 +13,82 @@
 
 import onair.data_handling.parser_util as parser_util
 
+# extract_meta_data_handle_ss_breakdown
+def test_parser_util_extract_meta_data_handle_ss_breakdown_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_when_given_ss_breakdown_does_not_resolve_to_False(mocker):
+    # Arrange
+    arg_configFile = MagicMock()
+    arg_ss_breakdown = True if pytest.gen.randint(0, 1) else MagicMock()
+
+    expected_result = MagicMock()
+
+    mocker.patch(parser_util.__name__ + '.extract_meta_data', return_value=expected_result)
+    mocker.patch(parser_util.__name__ + '.len')
+
+    # Act
+    result = parser_util.extract_meta_data_handle_ss_breakdown(arg_configFile, arg_ss_breakdown)
+
+    # Assert
+    assert parser_util.extract_meta_data.call_count == 1
+    assert parser_util.extract_meta_data.call_args_list[0].args == (arg_configFile, )
+    assert parser_util.len.call_count == 0
+    assert result == expected_result
+
+def test_parser_util_extract_meta_data_handle_ss_breakdown_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_with_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_empty_list_when_len_of_call_value_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_is_0_when_given_ss_breakdown_evaluates_to_False(mocker):
+    # Arrange
+    arg_configFile = MagicMock()
+    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
+
+    forced_return_extract_meta_data = {}
+    forced_return_len = 0
+    fake_empty_processed_filepath = MagicMock()
+    forced_return_extract_meta_data['subsystem_assignments'] = fake_empty_processed_filepath
+
+    expected_result = []
+
+    mocker.patch(parser_util.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
+    mocker.patch(parser_util.__name__ + '.len', return_value=forced_return_len)
+
+    # Act
+    result = parser_util.extract_meta_data_handle_ss_breakdown(arg_configFile, arg_ss_breakdown)
+
+    # Assert
+    assert parser_util.extract_meta_data.call_count == 1
+    assert parser_util.extract_meta_data.call_args_list[0].args == (arg_configFile, )
+    assert parser_util.len.call_count == 1
+    assert parser_util.len.call_args_list[0].args == (fake_empty_processed_filepath, )
+    assert result['subsystem_assignments'] == expected_result
+
+def test_parser_util_extract_meta_data_handle_ss_breakdown_returns_call_to_extract_meta_data_given_metadata_file_and_csv_set_to_True_with_dict_def_subsystem_assignments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_single_item_list_str_MISSION_for_each_item_when_given_ss_breakdown_evaluates_to_False(mocker):
+    # Arrange
+    arg_configFile = MagicMock()
+    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
+
+    forced_return_extract_meta_data = {}
+    forced_return_process_filepath = MagicMock()
+    fake_processed_filepath = []
+    num_fake_processed_filepaths = pytest.gen.randint(1,10) # arbitrary, from 1 to 10 (0 has own test)
+    for i in range(num_fake_processed_filepaths):
+        fake_processed_filepath.append(i)
+    forced_return_extract_meta_data['subsystem_assignments'] = fake_processed_filepath
+    forced_return_len = num_fake_processed_filepaths
+
+    expected_result = []
+    for i in range(num_fake_processed_filepaths):
+        expected_result.append(['MISSION'])
+
+    mocker.patch(parser_util.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
+    mocker.patch(parser_util.__name__ + '.len', return_value=forced_return_len)
+
+    # Act
+    result = parser_util.extract_meta_data_handle_ss_breakdown(arg_configFile, arg_ss_breakdown)
+
+    # Assert
+    assert parser_util.extract_meta_data.call_count == 1
+    assert parser_util.extract_meta_data.call_args_list[0].args == (arg_configFile, )
+    assert parser_util.len.call_count == 1
+    assert parser_util.len.call_args_list[0].args == (fake_processed_filepath, )
+    assert result['subsystem_assignments'] == expected_result
+
 # extract_meta_data tests
 def test_parser_util_extract_meta_data_raises_error_when_given_blank_meta_data_file():
     # Arrange

test/onair/data_handling/test_redis_adapter.py

@@ -307,85 +307,27 @@ def test_redis_adapter_DataSource_has_data_returns_instance_new_data():
 
     assert result == expected_result
 
-# redis_adapter parse_meta_data tests # TODO: need to be moved to a util function
-def test_redis_adapter_DataSource_parse_meta_data_file_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_when_given_ss_breakdown_does_not_resolve_to_False(mocker):
+# redis_adapter parse_meta_data tests
+def test_redis_adapter_DataSource_parse_meta_data_file_returns_call_to_extract_meta_data_handle_ss_breakdown(mocker):
     # Arrange
     cut = DataSource.__new__(DataSource)
     arg_configFile = MagicMock()
-    arg_ss_breakdown = True if pytest.gen.randint(0, 1) else MagicMock()
+    arg_ss_breakdown = MagicMock()
 
     expected_result = MagicMock()
 
-    mocker.patch(redis_adapter.__name__ + '.extract_meta_data', return_value=expected_result)
+    mocker.patch(redis_adapter.__name__ + '.extract_meta_data_handle_ss_breakdown', return_value=expected_result)
     mocker.patch(redis_adapter.__name__ + '.len')
 
     # Act
-    result = cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
+    result = cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown, )
 
     # Assert
-    assert redis_adapter.extract_meta_data.call_count == 1
-    assert redis_adapter.extract_meta_data.call_args_list[0].args == (arg_configFile, )
+    assert redis_adapter.extract_meta_data_handle_ss_breakdown.call_count == 1
+    assert redis_adapter.extract_meta_data_handle_ss_breakdown.call_args_list[0].args == (arg_configFile, arg_ss_breakdown)
     assert redis_adapter.len.call_count == 0
     assert result == expected_result
 
-def test_redis_adapter_DataSource_parse_meta_data_file_returns_call_to_extract_meta_data_file_given_metadata_file_and_csv_set_to_True_with_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_empty_list_when_len_of_call_value_dict_def_of_subsystem_assigments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_is_0_when_given_ss_breakdown_evaluates_to_False(mocker):
-    # Arrange
-    cut = DataSource.__new__(DataSource)
-    arg_configFile = MagicMock()
-    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
-
-    forced_return_extract_meta_data = {}
-    forced_return_len = 0
-    fake_empty_processed_filepath = MagicMock()
-    forced_return_extract_meta_data['subsystem_assignments'] = fake_empty_processed_filepath
-
-    expected_result = []
-
-    mocker.patch(redis_adapter.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
-    mocker.patch(redis_adapter.__name__ + '.len', return_value=forced_return_len)
-
-    # Act
-    result = cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
-
-    # Assert
-    assert redis_adapter.extract_meta_data.call_count == 1
-    assert redis_adapter.extract_meta_data.call_args_list[0].args == (arg_configFile, )
-    assert redis_adapter.len.call_count == 1
-    assert redis_adapter.len.call_args_list[0].args == (fake_empty_processed_filepath, )
-    assert result['subsystem_assignments'] == expected_result
-
-def test_redis_adapter_DataSource_parse_meta_data_file_returns_call_to_extract_meta_data_given_metadata_file_and_csv_set_to_True_with_dict_def_subsystem_assignments_def_of_call_to_process_filepath_given_configFile_and_kwarg_csv_set_to_True_set_to_single_item_list_str_MISSION_for_each_item_when_given_ss_breakdown_evaluates_to_False(mocker):
-    # Arrange
-    cut = DataSource.__new__(DataSource)
-    arg_configFile = MagicMock()
-    arg_ss_breakdown = False if pytest.gen.randint(0, 1) else 0
-
-    forced_return_extract_meta_data = {}
-    forced_return_process_filepath = MagicMock()
-    fake_processed_filepath = []
-    num_fake_processed_filepaths = pytest.gen.randint(1,10) # arbitrary, from 1 to 10 (0 has own test)
-    for i in range(num_fake_processed_filepaths):
-        fake_processed_filepath.append(i)
-    forced_return_extract_meta_data['subsystem_assignments'] = fake_processed_filepath
-    forced_return_len = num_fake_processed_filepaths
-
-    expected_result = []
-    for i in range(num_fake_processed_filepaths):
-        expected_result.append(['MISSION'])
-
-    mocker.patch(redis_adapter.__name__ + '.extract_meta_data', return_value=forced_return_extract_meta_data)
-    mocker.patch(redis_adapter.__name__ + '.len', return_value=forced_return_len)
-
-    # Act
-    result = cut.parse_meta_data_file(arg_configFile, arg_ss_breakdown)
-
-    # Assert
-    assert redis_adapter.extract_meta_data.call_count == 1
-    assert redis_adapter.extract_meta_data.call_args_list[0].args == (arg_configFile, )
-    assert redis_adapter.len.call_count == 1
-    assert redis_adapter.len.call_args_list[0].args == (fake_processed_filepath, )
-    assert result['subsystem_assignments'] == expected_result
-
 # redis_adapter get_vehicle_metadata tests
 def test_redis_adapter_DataSource_get_vehicle_metadata_returns_list_of_headers_and_list_of_test_assignments():
     # Arrange