Rebase Dtemkin Morningstar

docs/source/readers/morningstar.rst

@@ -0,0 +1,14 @@
+Morningstar
+-----------
+
+.. py:module:: pandas_datareader.mstar.daily
+
+.. autoclass:: MorningstarDailyReader
+   :members:
+   :inherited-members:
+
+.. py:module:: pandas_datareader.mstar.financials
+
+.. autoclass:: MorningstarDailyReader
+   :members:
+   :inherited-members:

docs/source/whatsnew/v0.6.0.txt

@@ -24,6 +24,9 @@ Highlights include:
   have been removed.  PDR would like to restore these features, and pull
   requests are welcome.
 
+- A new connector for Morningstart Open, High, Low, Close and Volume was
+  introduced (:issue:`467`)
+
 - A new connector for IEX daily price data was introduced (:issue:`465`).
 
 - A new connector for IEX the majority of the IEX API was introduced
@@ -50,6 +53,9 @@ Enhancements
 - A new data connector for data provided by the Bank of Canada was
   introduced. (:issue:`440`)
 
+- A new data connector for stock pricing data provided by Morningstar
+  was introduced. (:issue:`467`)
+
 .. _whatsnew_060.api_breaking:
 
 Backwards incompatible API changes

pandas_datareader/__init__.py

@@ -1,11 +1,13 @@
 from ._version import get_versions
-
-from .data import (get_components_yahoo, get_data_famafrench, get_data_google,
-                   get_data_yahoo, get_data_enigma, get_data_yahoo_actions,
-                   get_quote_google, get_quote_yahoo, get_tops_iex,
-                   get_last_iex, get_markets_iex, get_summary_iex,
-                   get_records_iex, get_recent_iex, get_iex_symbols,
-                   get_iex_book, DataReader, Options)
+from .data import (DataReader, Options, get_components_yahoo,
+                   get_dailysummary_iex, get_data_enigma, get_data_famafrench,
+                   get_data_fred, get_data_google, get_data_moex,
+                   get_data_morningstar, get_data_quandl, get_data_stooq,
+                   get_data_yahoo, get_data_yahoo_actions, get_iex_book,
+                   get_iex_symbols, get_last_iex, get_markets_iex,
+                   get_nasdaq_symbols,
+                   get_quote_google, get_quote_yahoo, get_recent_iex,
+                   get_records_iex, get_summary_iex, get_tops_iex)
 
 __version__ = get_versions()['version']
 del get_versions
@@ -16,4 +18,6 @@
            'get_iex_book', 'get_iex_symbols', 'get_last_iex',
            'get_markets_iex', 'get_recent_iex', 'get_records_iex',
            'get_summary_iex', 'get_tops_iex',
-           'DataReader', 'Options']
+           'get_nasdaq_symbols', 'get_data_quandl', 'get_data_moex',
+           'get_data_fred', 'get_dailysummary_iex', 'get_data_morningstar',
+           'get_data_stooq', 'DataReader', 'Options']

pandas_datareader/data.py

@@ -8,18 +8,19 @@
 from pandas_datareader.edgar import EdgarIndexReader
 from pandas_datareader.enigma import EnigmaReader
 from pandas_datareader.eurostat import EurostatReader
-from pandas_datareader.exceptions import ImmediateDeprecationError, \
-    DEP_ERROR_MSG
+from pandas_datareader.exceptions import DEP_ERROR_MSG, \
+    ImmediateDeprecationError
 from pandas_datareader.famafrench import FamaFrenchReader
 from pandas_datareader.fred import FredReader
 from pandas_datareader.google.daily import GoogleDailyReader
 from pandas_datareader.google.options import Options as GoogleOptions
 from pandas_datareader.google.quotes import GoogleQuotesReader
 from pandas_datareader.iex.daily import IEXDailyReader
 from pandas_datareader.iex.deep import Deep as IEXDeep
-from pandas_datareader.iex.tops import LastReader as IEXLasts
-from pandas_datareader.iex.tops import TopsReader as IEXTops
+from pandas_datareader.iex.tops import LastReader as IEXLasts, \
+    TopsReader as IEXTops
 from pandas_datareader.moex import MoexReader
+from pandas_datareader.mstar.daily import MorningstarDailyReader
 from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
 from pandas_datareader.oecd import OECDReader
 from pandas_datareader.quandl import QuandlReader
@@ -38,6 +39,7 @@
            'get_tops_iex', 'get_summary_iex', 'get_records_iex',
            'get_recent_iex', 'get_markets_iex', 'get_last_iex',
            'get_iex_symbols', 'get_iex_book', 'get_dailysummary_iex',
+           'get_data_morningstar', 'get_data_stooq',
            'get_data_stooq', 'DataReader']
 
 
@@ -97,6 +99,10 @@ def get_last_iex(*args, **kwargs):
     return IEXLasts(*args, **kwargs).read()
 
 
+def get_data_morningstar(*args, **kwargs):
+    return MorningstarDailyReader(*args, **kwargs).read()
+
+
 def get_markets_iex(*args, **kwargs):
     """
     Returns near-real time volume data across markets segregated by tape
@@ -359,6 +365,11 @@ def DataReader(name, data_source=None, start=None, end=None,
         return MoexReader(symbols=name, start=start, end=end,
                           retry_count=retry_count, pause=pause,
                           session=session).read()
+    elif data_source == "morningstar":
+        return MorningstarDailyReader(symbols=name, start=start, end=end,
+                                      retry_count=retry_count, pause=pause,
+                                      session=session, interval="d").read()
+
     else:
         msg = "data_source=%r is not implemented" % data_source
         raise NotImplementedError(msg)

pandas_datareader/mstar/__init__.py

@@ -0,0 +1,4 @@
+"""
+written and developed by Daniel Temkin
+please refer to LICENSE for ownership and reference information
+"""

pandas_datareader/mstar/daily.py

@@ -0,0 +1,223 @@
+import time
+from datetime import datetime, timedelta
+from warnings import warn
+
+import requests
+from pandas import DataFrame
+
+from pandas_datareader._utils import SymbolWarning
+from pandas_datareader.base import _BaseReader
+
+
+class MorningstarDailyReader(_BaseReader):
+    """
+    Read daily data from Morningstar
+
+    Parameters
+    ----------
+    symbols : {str, List[str]}
+        String symbol of like of symbols
+    start : string, (defaults to '1/1/2010')
+        Starting date, timestamp. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
+    end : string, (defaults to today)
+        Ending date, timestamp. Same format as starting date.
+    retry_count : int, default 3
+        Number of times to retry query request.
+    pause : float, default 0.1
+        Time, in seconds, of the pause between retries.
+    session : Session, default None
+        requests.sessions.Session instance to be used
+    freq : {str, None}
+        Frequency to use in select readers
+    incl_splits : bool, optional
+        Include splits in data
+    incl_dividends : bool,, optional
+        Include divdends in data
+    incl_volume : bool, optional
+        Include volume in data
+    currency : str, optional
+        Currency to use for data
+    interval : str, optional
+        Sampling interval to use for downloaded data
+    """
+
+    def __init__(self, symbols, start=None, end=None, retry_count=3,
+                 pause=0.1, timeout=30, session=None, freq=None,
+                 incl_splits=False, incl_dividends=False, incl_volume=True,
+                 currency='usd', interval='d'):
+        super(MorningstarDailyReader, self).__init__(symbols, start, end,
+                                                     retry_count, pause,
+                                                     timeout, session, freq)
+
+        self.incl_splits = incl_splits
+        self.incl_dividends = incl_dividends
+        self.incl_vol = incl_volume
+        self.currency = currency
+        self.interval = interval
+
+        self._symbol_data_cache = []
+
+    def _url_params(self):
+        if self.interval not in ['d', 'wk', 'mo', 'm', 'w']:
+            raise ValueError("Invalid interval: valid values are  'd', 'wk' "
+                             "and 'mo'. 'm' and 'w' have been implemented for "
+                             "backward compatibility")
+        elif self.interval in ['m', 'mo']:
+            self.interval = 'm'
+        elif self.interval in ['w', 'wk']:
+            self.interval = 'w'
+
+        if self.currency != "usd":
+            warn("Caution! There is no explicit check for a valid currency "
+                 "acronym\nIf an error is encountered consider changing this "
+                 "value.")
+
+        p = {"range": "|".join(
+            [self.start.strftime("%Y-%m-%d"), self.end.strftime("%Y-%m-%d")]),
+            "f": self.interval, "curry": self.currency,
+            "dtype": "his", "showVol": "true",
+            "hasF": "true", "isD": "true", "isS": "true",
+            "ProdCode": "DIRECT"}
+
+        return p
+
+    @property
+    def url(self):
+        """API URL"""
+        return "http://globalquote.morningstar.com/globalcomponent/" \
+               "RealtimeHistoricalStockData.ashx"
+
+    def _get_crumb(self, *args):
+        """Not required """
+        pass
+
+    def _dl_mult_symbols(self, symbols):
+        failed = []
+        symbol_data = []
+        for symbol in symbols:
+
+            params = self._url_params()
+            params.update({"ticker": symbol})
+
+            try:
+                resp = requests.get(self.url, params=params)
+            except Exception:
+                if symbol not in failed:
+                    if self.retry_count == 0:
+                        warn("skipping symbol %s: number of retries "
+                             "exceeded." % symbol)
+                        pass
+                    else:
+                        print("adding %s to retry list" % symbol)
+                        failed.append(symbol)
+            else:
+                if resp.status_code == requests.codes.ok:
+                    jsondata = resp.json()
+                    if jsondata is None:
+                        failed.append(symbol)
+                        continue
+                    jsdata = self._restruct_json(symbol=symbol,
+                                                 jsondata=jsondata)
+                    symbol_data.extend(jsdata)
+                else:
+                    raise Exception("Request Error!: %s : %s" % (
+                        resp.status_code, resp.reason))
+
+            time.sleep(self.pause)
+
+        if len(failed) > 0 and self.retry_count > 0:
+            # TODO: This appears to do nothing since
+            # TODO: successful symbols are not added to
+            self._dl_mult_symbols(symbols=failed)
+            self.retry_count -= 1
+        else:
+            self.retry_count = 0
+
+        if not symbol_data:
+            raise ValueError('All symbols were invalid')
+        elif self.retry_count == 0 and len(failed) > 0:
+            warn("The following symbols were excluded do to http "
+                 "request errors: \n %s" % failed, SymbolWarning)
+
+        symbols_df = DataFrame(data=symbol_data)
+        dfx = symbols_df.set_index(["Symbol", "Date"])
+        return dfx
+
+    @staticmethod
+    def _convert_index2date(enddate, indexvals):
+        i = 0
+        while i < len(indexvals):
+            days = indexvals[len(indexvals) - 1] - indexvals[i]
+            d = enddate - timedelta(days=days)
+            i += 1
+            yield d.strftime("%Y-%m-%d")
+
+    def _restruct_json(self, symbol, jsondata):
+        if jsondata is None:
+            return
+        divdata = jsondata["DividendData"]
+
+        pricedata = jsondata["PriceDataList"][0]["Datapoints"]
+        dateidx = jsondata["PriceDataList"][0]["DateIndexs"]
+        volumes = jsondata["VolumeList"]["Datapoints"]
+
+        date_ = self._convert_index2date(enddate=self.end, indexvals=dateidx)
+        barss = []
+        for p in range(len(pricedata)):
+            bar = pricedata[p]
+            d = next(date_)
+            bardict = {
+                "Symbol": symbol, "Date": d, "Open": bar[0], "High": bar[1],
+                "Low": bar[2],
+                "Close": bar[3]
+            }
+            if len(divdata) == 0:
+                pass
+            else:
+                events = []
+                for x in divdata:
+                    delta = (datetime.strptime(x["Date"], "%Y-%m-%d") -
+                             datetime.strptime(d, "%Y-%m-%d"))
+                    if delta.days == 0:
+                        events.append(x)
+                for e in events:
+                    if self.incl_dividends and e["Type"].find("Div") > -1:
+                        val = e["Desc"].replace(e["Type"], "")
+                        bardict.update({"isDividend": val})
+                    elif (self.incl_splits is True and
+                          e["Type"].find("Split") > -1):
+                        val = e["Desc"].replace(e["Type"], "")
+                        bardict.update({"isSplit": val})
+                    else:
+                        pass
+            if self.incl_vol is True:
+                bardict.update({"Volume": int(volumes[p] * 1000000)})
+            else:
+                pass
+
+            barss.append(bardict)
+        return barss
+
+    def read(self):
+        """Read data"""
+        if isinstance(self.symbols, str):
+            symbols = [self.symbols]
+        else:
+            symbols = self.symbols
+
+        is_str = False
+        try:
+            is_str = all(map(lambda v: isinstance(v, str), symbols))
+        except Exception:
+            pass
+
+        if not is_str:
+            raise TypeError("symbols must be iterable or string and not "
+                            "type %s" % type(self.symbols))
+
+        df = self._dl_mult_symbols(symbols=symbols)
+        if len(df.index.levels[0]) == 0:
+            raise ValueError("None of the provided symbols were valid")
+        else:
+            return df