Fixed various bugs in MarketDataRequest

README.md

@@ -115,6 +115,7 @@ individual data providers)
 
 # Release Notes
 
+* 0.1.29 - findatapy (xx Aug 2022)
 * 0.1.28 - findatapy (19 Jul 2022)
 * 0.1.27 - findatapy (20 May 2022)
 * 0.1.26 - findatapy (07 Oct 2021)
@@ -135,6 +136,9 @@ individual data providers)
 
 # Coding log
 
+* 11 Aug 2022
+  * Fixed various bug in MarketDataRequest, when updating constants and 
+  freeform str ticker queries with kwargs
 * 19 Jul 2022
   * Various fixes for data download
 * 20 May 2022

findatapy/conf/base_depos_tickers_list.csv

@@ -397,8 +397,8 @@ base-depos,bloomberg,daily,EUR2M,TOK,close,EUDRB CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR3M,TOK,close,EUDRC CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR4M,TOK,close,EUDRD CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR6M,TOK,close,EUDRF CMPT Index,8:00pm TOK
-base-depos,bloomberg,daily,EUR9M,TOK,close,EUDRI CMPT Index,8:00pm TOK
-base-depos,bloomberg,daily,EUR1Y,TOK,close,EUDR1 CMPT Curncy,8:00pm TOK
+base-depos,bloomberg,daily,EUR9M,TOK,close,EUDRI CMPT Curncy,8:00pm TOK
+base-depos,bloomberg,daily,EUR1Y,TOK,close,EUDR1 CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR2Y,TOK,close,EUDR2 CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR3Y,TOK,close,EUDR3 CMPT Index,8:00pm TOK
 base-depos,bloomberg,daily,EUR5Y,TOK,close,EUDR5 CMPT Index,8:00pm TOK

findatapy/conf/time_series_fields_list.csv

@@ -7,6 +7,7 @@ quandl,adj-close,adj-close
 quandl,close,settle
 quandl,prev-day-open-interest,prev-day-open-interest
 quandl,volume,volume
+yahoo,adj-close,Adj Close
 yahoo,close,Close
 yahoo,open,Open
 yahoo,high,High

findatapy/market/datavendorweb.py

@@ -46,6 +46,7 @@
 import requests
 
 import pandas as pd
+import numpy as np
 
 # support Quandl 3.x.x
 try:
@@ -716,17 +717,19 @@ def load_ticker(self, md_request):
         if data_frame is None or data_frame.index is []:
             return None
 
-        # convert from vendor to findatapy tickers/fields
-        if data_frame is not None:
-            try:
-                if len(md_request.tickers) > 1:
-                    data_frame.columns = ['/'.join(col) for col in
-                                          data_frame.columns.values]
-            except:
-                pass
+        # Convert from vendor to findatapy tickers/fields
+        #if data_frame is not None:
+        #    try:
+        #        if len(md_request.tickers) > 1:
+        #            data_frame.columns = ['/'.join(col) for col in
+        #                                  data_frame.columns.values]
+        #    except:
+        #        pass
+
 
-            returned_tickers = data_frame.columns
         if data_frame is not None:
+            raw_tickers = data_frame.columns.values.tolist()
+
             # tidy up tickers into a format that is more easily translatable
             # we can often get multiple fields returned (even if we don't ask
             # for them!)
@@ -736,17 +739,21 @@ def load_ticker(self, md_request):
             # returned_fields = [x.replace('value', 'close') for x in
             # returned_fields]  # special case for close
 
-            returned_fields = [x.split('/')[0].lower() for x in
-                               returned_tickers]
-
             # returned_tickers = [x.replace('.', '/') for x in returned_tickers]
-            returned_tickers = [x.split('/')[1] for x in returned_tickers]
 
-            fields = self.translate_from_vendor_field(returned_fields,
-                                                      md_request)
+            # Sometimes Yahoo tickers can have "." in them, so need to use
+            # rsplit
+            returned_tickers = [x.rsplit('.', 1)[0] for x in raw_tickers]
+
+            returned_fields = [x.rsplit('.', 1)[1] for x in
+                               raw_tickers]
+
             tickers = self.translate_from_vendor_ticker(returned_tickers,
                                                         md_request)
 
+            fields = self.translate_from_vendor_field(returned_fields,
+                                                      md_request)
+
             ticker_combined = []
 
             for i in range(0, len(fields)):
@@ -767,9 +774,9 @@ def download_daily(self, md_request):
         data_frame = None
 
         ticker_list = ' '.join(md_request.tickers)
-        data_frame = yf.download(ticker_list,
-                                 start=md_request.start_date,
-                                 end=md_request.finish_date)
+        # data_frame = yf.download(ticker_list,
+        #                         start=md_request.start_date,
+        #                         end=md_request.finish_date)
 
         while (trials < 5):
 
@@ -780,17 +787,37 @@ def download_daily(self, md_request):
 
                 break
             except Exception as e:
+                import time
+
                 print(str(e))
                 trials = trials + 1
+                time.sleep(1)
                 logger.info("Attempting... " + str(
                     trials) + " request to download from Yahoo")
 
         if trials == 5:
-            logger.error("Couldn't download from ONS after several attempts!")
+            logger.error("Couldn't download from Yahoo after several attempts!")
+
+        if data_frame is not None:
+            if len(md_request.tickers) == 1:
+                data_frame.columns = [md_request.tickers[0] + "." + x for
+                                      x in data_frame.columns]
+            else:
+                fields = data_frame.columns.levels[0]
+                tickers = data_frame.columns.levels[1]
 
-        if len(md_request.tickers) == 1:
-            data_frame.columns = [x + '/' + md_request.tickers[0] for
-                                  x in data_frame.columns]
+                new_cols = []
+
+                for fi in fields:
+                    for ti in tickers:
+                        new_cols.append(ti + "." + fi)
+
+                data_frame.columns = new_cols
+
+
+        # if len(md_request.tickers) == 1:
+        #    data_frame.columns = [x + '/' + md_request.tickers[0] for
+        #                          x in data_frame.columns]
 
         return data_frame
 
@@ -2660,7 +2687,12 @@ def load_ticker(self, md_request, index_col=0, max_workers=1,
         data_frame_list = []
 
         def download_data_frame(data_source):
-            if data_engine is not None:
+
+            file_types = ['.csv', '.parquet', '.zip', '.gzip', '.h5']
+
+            read_from_disk = np.all([x not in data_source for x in file_types])
+
+            if data_engine is not None and read_from_disk:
 
                 logger.info("Request " + str(
                     md_request.data_source) + " data via " + str(

findatapy/market/market.py

@@ -61,7 +61,7 @@ def __init__(self, market_data_generator=None, md_request=None):
         self.md_request = md_request
 
     def fetch_market(self, md_request=None, md_request_df=None,
-                     md_request_str=None, tickers=None,
+                     md_request_str=None, md_request_dict=None, tickers=None,
                      start_date=None, finish_date=None, best_match_only=False,
                      **kwargs):
         """Fetches market data for specific tickers
@@ -104,6 +104,18 @@ def fetch_market(self, md_request=None, md_request_df=None,
         if self.md_request is not None:
             md_request = self.md_request
 
+        if isinstance(md_request, str):
+            md_request_str = md_request
+            md_request = MarketDataRequest()
+
+        if isinstance(md_request, dict):
+            md_request_dict = md_request
+            md_request = MarketDataRequest()
+
+        if isinstance(md_request, pd.DataFrame):
+            md_request_df = md_request
+            md_request = MarketDataRequest()
+
         # Any kwargs are assumed to be to set MarketDataRequest attributes
         if kwargs != {}:
             md_request = self._kwargs_to_md_request(kwargs, md_request)
@@ -136,7 +148,7 @@ def fetch_market(self, md_request=None, md_request_df=None,
             return self.fetch_market(md_request)
 
         # Or directly as a string
-        if isinstance(md_request, str):
+        if md_request_str is not None:
             md_request = self.create_md_request_from_str(
                 md_request, 
                 start_date=start_date, finish_date=finish_date,
@@ -145,16 +157,18 @@ def fetch_market(self, md_request=None, md_request_df=None,
             return self.fetch_market(md_request)
 
         # Or directly as a dict
-        if isinstance(md_request, dict):
+        if md_request_dict is not None:
             md_request = self.create_md_request_from_dict(
-                md_request, start_date=start_date, finish_date=finish_date)
+                md_request_dict, md_request=md_request,
+                start_date=start_date, finish_date=finish_date)
 
             return self.fetch_market(md_request)
 
         # Or directly as a DataFrame
-        if isinstance(md_request, pd.DataFrame):
+        if md_request_df is not None:
             md_request = self.create_md_request_from_dataframe(
-                md_request, start_date=start_date, finish_date=finish_date,
+                md_request_df, md_request=md_request,
+                start_date=start_date, finish_date=finish_date,
                 **kwargs)
 
             return self.fetch_market(md_request)
@@ -678,16 +692,16 @@ def create_md_request_from_str(self, md_request_str, md_request=None,
                     best_match_only=best_match_only,
                     smart_group=smart_group)
 
-                md_request_df = self.create_md_request_from_dataframe(
+                md_request = self.create_md_request_from_dataframe(
                     md_request_df,
                     md_request=md_request, start_date=start_date,
                     finish_date=finish_date)
 
-                md_request_df = self._kwargs_to_md_request(kwargs,
-                                                           md_request_df)
+                md_request = self._kwargs_to_md_request(kwargs,
+                                                           md_request)
 
                 # if best_match_only:
-                return md_request_df
+                return md_request
 
 
             else:

findatapy/market/marketdatagenerator.py

@@ -75,7 +75,9 @@ def get_data_vendor(self, md_request):
         # Special case for files (csv, h5, parquet or zip)
         if ".csv" in str(data_source) or ".h5" in str(data_source) or \
                 ".parquet" in str(data_source) or ".zip" in str(data_source) \
-                or data_engine is not None:
+                or (data_engine is not None
+                    and md_request.category is not None
+                    and "internet_load" not in md_request.cache_algo):
             from findatapy.market.datavendorweb import DataVendorFlatFile
             data_vendor = DataVendorFlatFile()
         else: