initial codes

v 0.2

.gitignore

@@ -2,6 +2,8 @@
 __pycache__/
 *.py[cod]
 *$py.class
+*.xml
+*.iml
 
 # C extensions
 *.so

quanttrading2/__init__.py

@@ -1,6 +1,12 @@
-"""
-quanttrading2
-~~~~~~
-
-The quanttrading2 package - Python backtest and live trading
-"""
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from .account import *
+from .brokerage import *
+from .data import *
+from .event import *
+from .log import *
+from .order import *
+from .performance import *
+from .position import *
+from .risk import *
+from .strategy import *

quanttrading2/account/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from .account_event import *
+from .account_manager import *

quanttrading2/account/account_event.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from ..event.event import *
+
+
+class AccountEvent(Event):
+    """
+    also serve as account
+    """
+    def __init__(self):
+        self.event_type = EventType.ACCOUNT
+        self.account_id = ''
+        self.preday_balance = 0.0
+        self.balance = 0.0
+        self.available = 0.0
+        self.commission = 0.0
+        self.margin = 0.0
+        self.closed_pnl = 0.0
+        self.open_pnl = 0.0
+        self.brokerage = ''
+        self.api = ''
+        self.timestamp = ''
+
+    def deserialize(self, msg):
+        v = msg.split('|')
+        self.account_id = v[1]
+        self.preday_balance = float(v[2])
+        self.balance = float(v[3])
+        self.available = float(v[4])
+        self.commission = float(v[5])
+        self.margin = float(v[6])
+        self.closed_pnl = float(v[7])
+        self.open_pnl = float(v[8])
+        self.timestamp = v[9]

quanttrading2/account/account_manager.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .account_event import AccountEvent
+
+
+class AccountManager(object):
+    def __init__(self, config_server):
+        self._config_server = config_server
+        self._account_dict = {}            # account id ==> account
+        self.reset()
+
+    def reset(self):
+        self._account_dict.clear()
+        # initialize accounts from server_config.yaml
+        for a in self._config_server['accounts']:
+            account = AccountEvent()
+            account.account_id = a
+            account.brokerage = self._config_server[a]['broker']
+            account.api = self._config_server[a]['api']
+            self._account_dict[a] = account
+
+    def on_account(self, account_event):
+        if account_event.account_id in self._account_dict:
+            self._account_dict[account_event.account_id].preday_balance = account_event.preday_balance
+            self._account_dict[account_event.account_id].balance = account_event.balance
+            self._account_dict[account_event.account_id].available = account_event.available
+            self._account_dict[account_event.account_id].commission = account_event.commission
+            self._account_dict[account_event.account_id].margin = account_event.margin
+            self._account_dict[account_event.account_id].closed_pnl = account_event.closed_pnl
+            self._account_dict[account_event.account_id].open_pnl = account_event.open_pnl
+            self._account_dict[account_event.account_id].timestamp = account_event.timestamp
+        else:
+            self._account_dict[account_event.account_id] = account_event

quanttrading2/backtest_engine.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import numpy as np
+import pandas as pd
+from datetime import datetime, date
+
+from .event.event import EventType
+from .event.backtest_event_engine import BacktestEventEngine
+from .data.backtest_data_feed_quandl import BacktestDataFeedQuandl
+from .data.backtest_data_feed_local_single_symbol import BacktestDataFeedLocalSingleSymbol
+from .data.backtest_data_feed_local_multiple_symbols import BacktestDataFeedLocalMultipleSymbols
+from .data.data_board import DataBoard
+from .brokerage.backtest_brokerage import BacktestBrokerage
+from .position.portfolio_manager import PortfolioManager
+from .performance.performance_manager import PerformanceManager
+from .risk.risk_manager import PassThroughRiskManager
+
+class BacktestEngine(object):
+    """
+    Event driven backtest engine
+    """
+    def __init__(self):
+        self._current_time = None
+        self._initial_cash = 1_000.0
+        self._symbols = None
+        self._benchmark = None
+        self._start_date = None
+        self._end_date = None
+        self._params = None
+        self._strategy_name = None
+        self._datasource = None
+        self._batch_tag = 0
+        self._multiplier = 1
+        self._fvp_file = None
+        self._hist_dir = None
+        self._output_dir = None
+
+    def set_cash(self, cash):
+        self._initial_cash = cash
+
+    def set_symbols(self, symbols):
+        self._symbols = symbols
+
+    def set_benchmark(self, benchmark):
+        self._benchmark = benchmark
+
+    def set_start_date(self, start_date):
+        self._start_date = start_date
+
+    def set_end_date(self, end_date):
+        self._end_date = end_date
+
+    def set_params(self, params):
+        self._parmas = params
+
+    def set_strategy(self, strategy_name):
+        self._strategy_name = strategy_name
+
+    def _setup(self):
+        ## 1. data_feed
+        symbols_all = self._symbols[:]   # copy
+        if self._benchmark is not None:
+            symbols_all.append(self._benchmark)
+        self._symbols = [str(s) for s in self._symbols]
+        symbols_all = set([str(s) for s in symbols_all])  # remove duplicates
+
+        if (datasource.upper() == 'LOCAL'):
+            print('Using local single symbol data feed')
+            self._data_feed = BacktestDataFeedLocalSingleSymbol(
+                hist_dir=self._hist_dir,
+                start_date=start_date, end_date=send_date
+            )
+        elif (datasource.upper() == 'MULTI_LOCAL'):
+            print('Using local multiple symbol data feed')
+            self._data_feed = BacktestDataFeedLocalMultipleSymbols(
+                hist_dir=self._hist_dir,
+                start_date=start_date, end_date=send_date
+            )
+        else:
+            print('Using Quandl data feed')
+            self._data_feed = BacktestDataFeedQuandl(
+                start_date=start_date, end_date=send_date
+            )
+
+        self._data_feed.subscribe_market_data(self._symbols)         # not symbols_all
+
+        ## 2. event engine
+        self._events_engine = BacktestEventEngine(self._data_feed)
+
+        ## 3. brokerage
+        self._data_board = DataBoard(hist_dir=self._hist_dir, syms=symbols_all)
+        self._backtest_brokerage = BacktestBrokerage(
+            self._events_engine, self._data_board
+        )
+
+        ## 4. portfolio_manager
+        self._df_fvp = None
+        if self._fvp_file is not None:
+            self._df_fvp = pd.read_csv(self._hist_dir+self._fvp_file, index_col=0)
+
+        self._portfolio_manager = PortfolioManager(self._initial_cash, self._df_fvp)
+
+        ## 5. performance_manager
+        self._performance_manager = PerformanceManager(self._symbols, self._benchmark, batch_tag, root_multiplier, self._df_fvp)
+
+        ## 6. risk_manager
+        self._risk_manager = PassThroughRiskManager()
+
+        ## 7. load all strategies
+        strategyClass = strategy_list.get(strategy_name, None)
+        if not strategyClass:
+            print(u'can not find strategy：%s' % strategy_name)
+            return
+        else:
+            print(u'backtesting strategy：%s' % strategy_name)
+        self._strategy = strategyClass(self._events_engine, self._data_board)
+        self._strategy.set_symbols(self._symbols)
+        self._strategy.set_capital(self._initial_cash)
+        self._strategy.on_init(params)
+        self._strategy.on_start()
+
+        ## 8. trade recorder
+        #self._trade_recorder = ExampleTradeRecorder(output_dir)
+
+        ## 9. wire up event handlers
+        self._events_engine.register_handler(EventType.TICK, self._tick_event_handler)
+        self._events_engine.register_handler(EventType.BAR, self._bar_event_handler)
+        self._events_engine.register_handler(EventType.ORDER, self._order_event_handler)
+        self._events_engine.register_handler(EventType.FILL, self._fill_event_handler)
+
+    # ------------------------------------ private functions -----------------------------#
+    def _tick_event_handler(self, tick_event):
+        self._current_time = tick_event.timestamp
+
+        # performance update goes before position updates because it updates previous day performance
+        self._performance_manager.update_performance(self._current_time, self._portfolio_manager, self._data_board)
+        self._portfolio_manager.mark_to_market(self._current_time, tick_event.full_symbol, tick_event.price, self._data_board)
+        self._data_board.on_tick(tick_event)
+        self._strategy.on_tick(tick_event)
+
+    def _bar_event_handler(self, bar_event):
+        self._current_time = bar_event.bar_end_time()
+
+        # performance update goes before position updates because it updates previous day
+        self._performance_manager.update_performance(self._current_time, self._portfolio_manager, self._data_board)
+        self._portfolio_manager.mark_to_market(self._current_time, bar_event.full_symbol, bar_event.adj_close_price, self._data_board)
+        self._data_board.on_bar(bar_event)
+        self._strategy.on_bar(bar_event)
+
+    def _order_event_handler(self, order_event):
+        self._backtest_brokerage.place_order(order_event)
+
+    def _fill_event_handler(self, fill_event):
+        self._portfolio_manager.on_fill(fill_event)
+        self._performance_manager.on_fill(fill_event)
+
+    # -------------------------------- end of private functions -----------------------------#
+
+    # -------------------------------------- public functions -------------------------------#
+    def run(self, tear_sheet=True):
+        """
+        Run backtest
+        """
+        self._setup()
+        self._events_engine.run()
+        self._performance_manager.update_final_performance(self._current_time, self._portfolio_manager, self._data_board)
+        self._performance_manager.save_results(self._output_dir)
+
+        return self._performance_manager.caculate_performance(tear_sheet)
+
+    # ------------------------------- end of public functions -----------------------------#
+
+
+if __name__ == '__main__':
+    hist_dir = 'd:/workspace/quantresearch/data/'
+    fvp_file = ''
+
+    df_fvp = pd.read_csv(hist_dir + fvp_file, index_col=0)
+    backtest_engine = BacktestEngine()
+    results, results_dd, monthly_ret_table, ann_ret_df = backtest_engine.run()
+    if results is None:
+        print('Empty Strategy')
+    else:
+        print(results)
+        print(results_dd)
+        print(monthly_ret_table)
+        print(ann_ret_df)

quanttrading2/backtest_optimization_engine.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import pandas as pd
+import numpy as np
+from datetime import datetime, date
+import multiprocessing
+import yaml
+from .backtest_engine import BacktestEngine
+
+def output(content):
+    print(str(datetime.now()) + "\t" + content)
+
+
+def optimize(config, target_name):
+    backtest = BacktestEngine(config)
+    results = backtest.run(tear_sheet=False)
+
+    try:
+        #target_value = results[target_name]
+        target_value = results[0].loc[target_name][0]   # first table in tuple
+    except KeyError:
+        target_value = 0
+    return (config, target_value, results)
+
+if __name__ == '__main__':
+    # ------------------------ Set up config in code ---------------------------#
+    config = {}
+    config['cash'] = 500000.00
+    config['benchmark'] = None
+    config['root_multiplier'] = None
+    config['fvp_file'] = None
+    config['start_date'] = date(2010, 1, 1)
+    config['end_date'] = datetime.today().date()
+    config['end_date'] = date(2017, 5, 1)
+    config['datasource'] = 'local'
+    config['hist_dir'] = 'd:/workspace/privatefund/backtest/hist/'
+    config['batch_tag'] = '0'           # used to tag first backtest; second backtest; etc
+    config['output_dir'] = 'd:/workspace/privatefund/backtest/out/'
+
+    # strategy specific
+    config['strategy'] = 'MovingAverageCrossStrategy'
+    config['symbols'] = ['SPX Index']
+
+    # you can use for loop to construct params list in code
+    params_list = []
+    for sw in [10, 20, 30, 40, 50]:
+        for lw in [10, 20, 30, 40, 50]:
+            if lw <= sw:
+                continue
+            params_list.append({'short_window':sw, 'long_window': lw})
+
+    config['params_list'] = params_list
+
+    params_list = [{'short_window':10, 'long_window': 20},
+                   {'short_window': 10, 'long_window': 30},
+                   {'short_window': 10, 'long_window': 50},
+                   {'short_window': 20, 'long_window': 30},
+                   {'short_window': 20, 'long_window': 50}]
+    # ------------------------ End of set up config in code ---------------------------#
+
+    # ------------------------ Or read from config file -----------------------------------#
+    config = None
+    try:
+        path = os.path.abspath(os.path.dirname(__file__))
+        config_file = os.path.join(path, 'config_backtest_moving_average_cross.yaml')
+        # config_file = os.path.join(path, 'config_backtest_mean_reversion_spread.yaml')
+        with open(os.path.expanduser(config_file)) as fd:
+            config = yaml.load(fd)
+    except IOError:
+        print("config.yaml is missing")
+    # ----------------------- End of reading from config file ------------------------------#
+
+    target_name = 'Sharpe ratio'
+    pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+    res_list = []
+    batch_token = 0
+    for param in config['params_list']:
+        config_local = config.copy()
+        config_local['params'] = param
+        config_local['batch_tag'] = str(batch_token)
+        res_list.append(pool.apply_async(optimize, (config_local, target_name)))
+        batch_token = batch_token + 1
+    pool.close()
+    pool.join()
+
+    res_list = [res.get() for res in res_list]
+    res_list.sort(reverse=True, key=lambda res:res[1])
+
+    output('-' * 50)
+    output(u'optimization results：')
+    for res in res_list:
+        output(u'Params：%s，%s：%s' % (res[0]['params'], target_name, res[1]))
+

quanttrading2/brokerage/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-