update tests for bidding files because of AEMO storage window, add ex…

…ample for disable logging

.gitignore

@@ -26,3 +26,4 @@ nemosis/raw_aemo_data/*
 NEMOSIS.exe
 nemosis/smoke_tests.py
 nemosis/check_new_bid_table_functionality.py
+*.pyc

README.md

@@ -23,6 +23,7 @@
       - [Additional columns](#accessing-additional-table-columns)
   - [Data from static tables](#data-from-static-tables)
     - [static_table](#static_table)
+  - [Disable logging](#disable-logging)
 
 -----
 
@@ -225,3 +226,17 @@ from nemosis import static_table
 
 fcas_variables = static_table('VARIABLES_FCAS_4_SECOND', raw_data_cache)
 ```
+### Disable logging
+
+NEMOSIS uses the python logging module to print messages to the console. If desired, this can be disabled after 
+imports, as shown below. This will disable log messages unless they are at least warnings.
+
+```python
+
+import logging
+
+from nemosis import dynamic_data_compiler
+
+logging.getLogger("nemosis").setLevel(logging.WARNING)
+
+```

e.py

@@ -0,0 +1,12 @@
+import logging
+
+from nemosis import dynamic_data_compiler
+
+logging.getLogger("nemosis").setLevel(logging.WARNING)
+
+start_time = '2017/01/01 00:00:00'
+end_time = '2017/01/01 00:05:00'
+table = 'DISPATCHPRICE'
+raw_data_cache = 'D:/nemosis_cache'
+
+price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache)

setup.py

@@ -21,6 +21,7 @@
         "pandas",
         "xlrd",
         "beautifulsoup4",
+        "openpyxl"
     ],
     classifiers=[
         "Programming Language :: Python :: 3",

tests/test_data_fetch_methods.py

@@ -401,7 +401,7 @@ def setUp(self):
             "BIDDAYOFFER_D": ["DUID", "BIDTYPE"],
         }
 
-        # Filter for bids at the start of the 2021-06-01 file and the end of the 2021-05-31, to make sure that we arn't
+        # Filter for bids at the start of the 2021-06-01 file and the end of the 2021-05-31, to make sure that we aren't
         # skipping any of the data file rows.
         self.filter_values = {
             "DUID-BIDTYPE": (
@@ -421,8 +421,8 @@ def restrictive_filter(data):
         return data
 
     def test_dispatch_tables_start_of_month(self):
-        start_time = "2021/06/01 00:00:00"
-        end_time = "2021/06/01 05:15:00"
+        start_time = "2021/09/01 00:00:00"
+        end_time = "2021/09/01 05:15:00"
         for table in self.table_names:
             print(f"Testing {table} returning values at start of month one.")
             dat_col = defaults.primary_date_columns[table]
@@ -437,11 +437,11 @@ def test_dispatch_tables_start_of_month(self):
             expected_last_time = pd.to_datetime(end_time, format="%Y/%m/%d %H:%M:%S")
             if table == "BIDDAYOFFER_D":
                 expected_length = 2 * 4
-                expected_last_time = "2021/06/01 00:00:00"
+                expected_last_time = "2021/09/01 00:00:00"
                 expected_last_time = pd.to_datetime(
                     expected_last_time, format="%Y/%m/%d %H:%M:%S"
                 )
-                expected_first_time = "2021/05/31 00:00:00"
+                expected_first_time = "2021/08/31 00:00:00"
                 expected_first_time = pd.to_datetime(
                     expected_first_time, format="%Y/%m/%d %H:%M:%S"
                 )
@@ -467,8 +467,8 @@ def test_dispatch_tables_start_of_month(self):
     def test_dispatch_tables_start_of_month_previous_market_day_but_not_start_calendar_month(
         self,
     ):
-        start_time = "2021/06/05 03:00:00"
-        end_time = "2021/06/05 03:15:00"
+        start_time = "2021/09/05 03:00:00"
+        end_time = "2021/09/05 03:15:00"
         for table in self.table_names:
             print(f"Testing {table} returning values at start of month two.")
             dat_col = defaults.primary_date_columns[table]
@@ -483,11 +483,11 @@ def test_dispatch_tables_start_of_month_previous_market_day_but_not_start_calend
             expected_last_time = pd.to_datetime(end_time, format="%Y/%m/%d %H:%M:%S")
             if table == "BIDDAYOFFER_D":
                 expected_length = 1 * 4
-                expected_last_time = "2021/06/04 00:00:00"
+                expected_last_time = "2021/09/04 00:00:00"
                 expected_last_time = pd.to_datetime(
                     expected_last_time, format="%Y/%m/%d %H:%M:%S"
                 )
-                expected_first_time = "2021/06/04 00:00:00"
+                expected_first_time = "2021/09/04 00:00:00"
                 expected_first_time = pd.to_datetime(
                     expected_first_time, format="%Y/%m/%d %H:%M:%S"
                 )
@@ -513,8 +513,8 @@ def test_dispatch_tables_start_of_month_previous_market_day_but_not_start_calend
     def test_dispatch_tables_start_of_month_previous_market_day_and_first_market_day_but_not_start_calendar_month(
         self,
     ):
-        start_time = "2021/06/01 03:00:00"
-        end_time = "2021/06/01 05:00:00"
+        start_time = "2021/09/01 03:00:00"
+        end_time = "2021/09/01 05:00:00"
         for table in self.table_names:
             print(f"Testing {table} returning values at start of month two.")
             dat_col = defaults.primary_date_columns[table]
@@ -529,11 +529,11 @@ def test_dispatch_tables_start_of_month_previous_market_day_and_first_market_day
             expected_last_time = pd.to_datetime(end_time, format="%Y/%m/%d %H:%M:%S")
             if table == "BIDDAYOFFER_D":
                 expected_length = 2 * 4
-                expected_last_time = "2021/06/01 00:00:00"
+                expected_last_time = "2021/09/01 00:00:00"
                 expected_last_time = pd.to_datetime(
                     expected_last_time, format="%Y/%m/%d %H:%M:%S"
                 )
-                expected_first_time = "2021/05/31 00:00:00"
+                expected_first_time = "2021/08/31 00:00:00"
                 expected_first_time = pd.to_datetime(
                     expected_first_time, format="%Y/%m/%d %H:%M:%S"
                 )
@@ -559,8 +559,8 @@ def test_dispatch_tables_start_of_month_previous_market_day_and_first_market_day
     def test_dispatch_tables_start_of_month_first_market_day_but_not_start_calendar_month(
         self,
     ):
-        start_time = "2021/06/01 04:00:00"
-        end_time = "2021/06/01 05:00:00"
+        start_time = "2021/09/01 04:00:00"
+        end_time = "2021/09/01 05:00:00"
         for table in self.table_names:
             print(f"Testing {table} returning values at start of month two.")
             dat_col = defaults.primary_date_columns[table]
@@ -575,11 +575,11 @@ def test_dispatch_tables_start_of_month_first_market_day_but_not_start_calendar_
             expected_last_time = pd.to_datetime(end_time, format="%Y/%m/%d %H:%M:%S")
             if table == "BIDDAYOFFER_D":
                 expected_length = 1 * 4
-                expected_last_time = "2021/06/01 00:00:00"
+                expected_last_time = "2021/09/01 00:00:00"
                 expected_last_time = pd.to_datetime(
                     expected_last_time, format="%Y/%m/%d %H:%M:%S"
                 )
-                expected_first_time = "2021/06/01 00:00:00"
+                expected_first_time = "2021/09/01 00:00:00"
                 expected_first_time = pd.to_datetime(
                     expected_first_time, format="%Y/%m/%d %H:%M:%S"
                 )
@@ -603,8 +603,8 @@ def test_dispatch_tables_start_of_month_first_market_day_but_not_start_calendar_
             print("Passed")
 
     def test_dispatch_tables_end_of_month(self):
-        start_time = "2021/06/30 21:00:00"
-        end_time = "2021/07/01 00:00:00"
+        start_time = "2021/09/30 21:00:00"
+        end_time = "2021/10/01 00:00:00"
         for table in self.table_names:
             print("Testing {} returing values at end of month.".format(table))
             dat_col = defaults.primary_date_columns[table]
@@ -642,15 +642,15 @@ def test_dispatch_tables_end_of_month(self):
             print("Passed")
 
     def test_dispatch_tables_straddle_2_months(self):
-        start_time = "2021/06/30 21:00:00"
-        end_time = "2021/07/01 21:00:00"
+        start_time = "2021/09/30 21:00:00"
+        end_time = "2021/10/01 21:00:00"
         for table in self.table_names:
             print(f"Testing {table} returing values from adjacent months.")
             dat_col = defaults.primary_date_columns[table]
             table_type = self.table_types[table]
             filter_cols = self.table_filters[table]
             cols = [dat_col, *filter_cols]
-            expected_length = 288 * 4
+            expected_length = 240 * 4  # This should be 288 but there data missing in file AEMO published.
             expected_number_of_columns = 3
             expected_first_time = pd.to_datetime(
                 start_time, format="%Y/%m/%d %H:%M:%S"
@@ -681,22 +681,22 @@ def test_dispatch_tables_straddle_2_months(self):
             print("Passed")
 
     def test_dispatch_tables_start_of_year(self):
-        start_time = "2021/01/01 00:00:00"
-        end_time = "2021/01/01 01:00:00"
+        start_time = "2022/01/01 00:00:00"
+        end_time = "2022/01/01 01:00:00"
         for table in self.table_names:
             print("Testing {} returing values at start of year.".format(table))
             dat_col = defaults.primary_date_columns[table]
             table_type = self.table_types[table]
             filter_cols = self.table_filters[table]
             cols = [dat_col, *filter_cols]
-            expected_length = 12 * 3
+            expected_length = 12 * 4
             expected_number_of_columns = 3
             expected_first_time = pd.to_datetime(
                 start_time, format="%Y/%m/%d %H:%M:%S"
             ) + timedelta(minutes=5)
             expected_last_time = pd.to_datetime(end_time, format="%Y/%m/%d %H:%M:%S")
             if table == "BIDDAYOFFER_D":
-                expected_length = 1 * 3
+                expected_length = 1 * 4
                 expected_last_time = expected_last_time.replace(
                     hour=0, minute=0
                 ) - timedelta(days=1)