Pass datetime format explicitly when parsing

nemosis/data_fetch_methods.py

@@ -778,14 +778,17 @@ def _infer_column_data_types(data):
     def _get_series_type(series):
         if series.dtype == "object":
             try:
-                col_new = _pd.to_datetime(series)
+                col_new = _pd.to_datetime(series, format="%Y/%m/%d %H:%M:%S.%f")
                 return col_new
-            except Exception as e:
+            except ValueError:
                 try:
-                    col_new = _pd.to_numeric(series)
-                    return col_new
-                except Exception as e:
-                    return series
+                    col_new = _pd.to_datetime(series, format="%Y/%m/%d %H:%M:%S")
+                except ValueError:
+                    try:
+                        col_new = _pd.to_numeric(series)
+                        return col_new
+                    except ValueError as e:
+                        return series
         else:
             return series
 

nemosis/filters.py

@@ -40,7 +40,7 @@ def filter_on_timestamp(data, start_time, end_time):
         data["TIMESTAMP"] = pd.to_datetime(
             data["TIMESTAMP"], format="%Y/%m/%d %H:%M:%S"
         )
-    except Exception as e:
+    except ValueError as e:
         logger.error(e)
         # if date format is wrong, str may be too short
         med_str_len = np.median(data["TIMESTAMP"].str.len())

tests/test_errors_and_warnings.py

@@ -1,6 +1,9 @@
 import unittest
 from nemosis import dynamic_data_compiler, cache_compiler, static_table, defaults
 import os
+import sys
+from unittest.mock import patch
+from io import StringIO
 
 
 class TestDynamicDataCompilerRaisesExpectedErrors(unittest.TestCase):
@@ -96,14 +99,14 @@ def test_raise_error_if_select_columns_not_in_data(self):
                 defaults.raw_data_cache,
                 select_columns=["NOTACOLUMN"],
             )
-        self.assertTrue(
+        self.assertIn(
             (
                 f"None of columns ['NOTACOLUMN'] are in D:/nemosis_test_cache\\PUBLIC_DVD_DISPATCHPRICE_201812010000.feather. "
                 "This may be caused by user input if the 'select_columns' "
                 "argument is being used, or by changed AEMO data formats. "
                 "This error can be avoided by using the argument select_columns='all'."
-            )
-            in str(context.exception)
+            ),
+            str(context.exception)
         )
 
     def test_using_select_columns_all_does_not_raise_error(self):
@@ -177,6 +180,20 @@ def test_using_select_columns_all_does_not_raise_error(self):
         ]
         self.assertSequenceEqual(list(price_data.columns), expected_columns)
 
+class TestWarnings(unittest.TestCase):
+    def test_no_parse_warning(self):
+        with patch('sys.stderr', new=StringIO()) as fakeOutput:
+            print('hello world')
+
+            dynamic_data_compiler(
+                start_time='2017/01/01 00:00:00', 
+                end_time='2017/01/01 00:05:00', 
+                table_name='DISPATCHPRICE', 
+                raw_data_location=defaults.raw_data_cache
+            )
+
+            stderr = fakeOutput.getvalue().strip()
+        self.assertNotIn("UserWarning: Could not infer format", stderr)
 
 class TestCacheCompilerRaisesExpectedErrors(unittest.TestCase):
     def test_raise_error_for_incorrect_table_name(self):
@@ -223,6 +240,7 @@ def test_raise_error_if_select_columns_used_without_rebuild_true(self):
         )
 
 
+
 class TestStaticTableRaisesExpectedErrors(unittest.TestCase):
     def test_raise_error_for_incorrect_table_name(self):
         with self.assertRaises(Exception) as context:
@@ -294,14 +312,14 @@ def test_raise_error_if_select_columns_not_in_data(self):
                 defaults.raw_data_cache,
                 select_columns=["NOTACOLUMN"],
             )
-        self.assertTrue(
+        self.assertIn(
             (
                 f"None of columns ['NOTACOLUMN'] are in D:/nemosis_test_cache\\Ancillary Services Market Causer Pays Variables File.csv. "
                 "This may be caused by user input if the 'select_columns' "
                 "argument is being used, or by changed AEMO data formats. "
                 "This error can be avoided by using the argument select_columns='all'."
-            )
-            in str(context.exception)
+            ),
+            str(context.exception)
         )
 
     def test_using_select_columns_all_does_not_raise_error(self):