(수익, class, 90일 정밀도 미구현) 함수 및 코드 정리.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# adjust 한글 font\n",
    "from matplotlib import font_manager, rc\n",
    "font_name = font_manager.FontProperties(fname='c:/Windows/Fonts/malgun.ttf').get_name()\n",
    "rc('font', family=font_name)\n",
    "\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "# plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get_technical_indicator.py\n",
    "def get_MACD(df, close='Adj Close', short_ma=12, long_ma=26, signal_period=9):\n",
    "    ewm_12 = df[close].ewm(span = short_ma, min_periods = short_ma - 1).mean()\n",
    "    ewm_26 = df[close].ewm(span = long_ma, min_periods = long_ma - 1).mean()\n",
    "    df['MACD'] = ewm_12 - ewm_26\n",
    "    df['MACDSignal'] = df['MACD'].ewm(span = signal_period, min_periods = signal_period-1).mean()\n",
    "    df['MACDDiff'] = df['MACD'] - df['MACDSignal']\n",
    "    return df\n",
    "\n",
    "def get_Stochastic(df, close='Adj Close', high='High', low='Low', n=14):\n",
    "    df['sto_K_{}'.format(n)] = 100 * ((df[close] - df[low].rolling(n).min()) / \n",
    "                                      (df[high].rolling(n).max() - df[low].rolling(n).min()))\n",
    "    df['sto_D_{}'.format(n)] = df['sto_K_{}'.format(n)].rolling(3).mean()\n",
    "    return df\n",
    "\n",
    "def get_RSI(df, close='Adj Close', n=14):\n",
    "    U = np.where(df[close].diff(1)>0, df[close].diff(1), 0)\n",
    "    D = np.where(df[close].diff(1)<0, df[close].diff(1)*(-1), 0)\n",
    "\n",
    "    AU = pd.Series(U).rolling(window=n, min_periods=n).mean()\n",
    "    AD = pd.Series(D).rolling(window=n, min_periods=n).mean()\n",
    "\n",
    "    RSI = AU.div(AD+AU) * 100\n",
    "    df['RSI_{}'.format(n)] = RSI.values\n",
    "    return df\n",
    "\n",
    "def get_RoC(df, close='Adj Close', periods=2):\n",
    "    df['roc_{}'.format(periods)] = df[close].pct_change(periods=periods)\n",
    "    return df\n",
    "\n",
    "def get_CV(df, close='Adj Close', window=10):\n",
    "    df['his_vol_{}'.format(window)] = np.log(df[close] / df[close].shift(1)).rolling(window).std()*(252**0.5)\n",
    "    return df\n",
    "\n",
    "def get_WilliamR(df, close='Adj Close', high='High', low='Low', n=14):\n",
    "    W_R = -100 * ((df[high].rolling(n).max() - df[close]) / (df[high].rolling(n).max() - df[low].rolling(n).min()))\n",
    "    df['WR_{}'.format(n)] = W_R\n",
    "    return df\n",
    "\n",
    "def get_CCI(df, close='Adj Close', high='High', low='Low', window=10):\n",
    "    TP = (df[close] + df[low] + df[high]) / 3\n",
    "    dataX = []\n",
    "    for i in range(len(TP) - window+1):\n",
    "        a = TP.values[i:(i+window)]\n",
    "        dataX.append(a)\n",
    "    dataMean = [i.mean() for i in dataX]\n",
    "    AVEDEV = [abs(x - mean).mean() for x, mean \n",
    "              in zip(dataX, dataMean)]\n",
    "    for i in range(window-1):\n",
    "        AVEDEV.insert(0,np.nan)\n",
    "    AVEDEV = pd.Series(AVEDEV, index = TP.index)\n",
    "    CCI = (TP - TP.rolling(20).mean()) / (0.015 * AVEDEV)\n",
    "    df['CCI_{}'.format(window)] = CCI\n",
    "    return df\n",
    "\n",
    "def get_DI(df, close='Adj Close', window=10):\n",
    "    df['DI_{}'.format(window)] = 100 * ((df[close] - df[close].rolling(window).mean()) / df[close].rolling(window).mean())\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_manager.py\n",
    "# from get_technical_indicator import *\n",
    "\n",
    "# def load_chart_data(fpath):\n",
    "#     chart_data = pd.read_csv(fpath, thousands=',')\n",
    "#     return chart_data\n",
    "\n",
    "def preprocess(chart_data, close='Adj Close', volume='Volume', windows=10):\n",
    "    prep_data = chart_data\n",
    "    for window in windows:\n",
    "        prep_data['close_ma{}'.format(window)] = prep_data[close].rolling(window).mean()\n",
    "        prep_data['volume_ma{}'.format(window)] = (\n",
    "            prep_data[volume].rolling(window).mean())\n",
    "    return prep_data, len(prep_data.columns)\n",
    "\n",
    "def build_feature(input_data, close='Adj Close', high='High', low='Low'):\n",
    "    data = input_data.copy()\n",
    "    for window in range(12, 91):\n",
    "        data = get_CV(data, close=close, window=window)\n",
    "    for periods in range(1, 11):\n",
    "        data = get_RoC(data, close=close, periods=periods)\n",
    "    for n in range(3, 91):\n",
    "        data = get_RSI(data, close=close, n=n)\n",
    "        data = get_Stochastic(data,close=close, high=high, low=low, n=n)\n",
    "        data = get_WilliamR(data, close=close, high=high, low=low, n=n)\n",
    "    data = get_CCI(data, close=close, high=high, low=low, window=20)\n",
    "    for window in [5,10]:\n",
    "        data = get_DI(data, close=close, window=window)\n",
    "    data = get_MACD(data, close=close)\n",
    "    return data\n",
    "\n",
    "def make_target(df, use_fn='f', period=1, window=1, \n",
    "                method='regress', thresh=.02, close='Adj Close'):\n",
    "    y_df = pd.DataFrame(index=df.index)\n",
    "    f = lambda x, i : np.log(x.shift(-1 * i) / x)\n",
    "    g = lambda x, i, j : np.log(x.shift(-1 * i) / x.rolling(j).mean())\n",
    "    if window > 99:\n",
    "        target = 'sh{:02}_Y_{:03}'.format(period, window)\n",
    "    else:\n",
    "        target = 'sh{:02}_Y_{:02}'.format(period, window)\n",
    "    target = target[:6] if use_fn=='f' else target\n",
    "    y_df[target] = \\\n",
    "            f(df[close], period) if use_fn == 'f' else g(df[close], period, window)\n",
    "    if method != 'regress':\n",
    "        y_df[target] = np.where(\n",
    "            y_df[target] > thresh, 1, np.where(\n",
    "                y_df[target] < (-1 * thresh), -1, 0))\n",
    "    return y_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_x_y(df, period, window, n_col=0, close='Adj Close',\n",
    "               use_fn='f', thresh=.02, method='regress'):\n",
    "    y_df = make_target(df, use_fn=use_fn, period=period, window=window, \n",
    "                           thresh=thresh, method=method, close=close)\n",
    "    y = y_df.columns[0]\n",
    "    compact_df = pd.concat((df, y_df), axis=1)\n",
    "#     compact_df = compact_df[compact_df.columns[n_col:]]\n",
    "#     compact_df = compact_df.dropna()\n",
    "    return compact_df, y\n",
    "\n",
    "def train_test_split(compact_df, start, end, date='date'):\n",
    "    train = compact_df[(compact_df[date] >= start) &\n",
    "                   (compact_df[date] <= end)]\n",
    "    test = compact_df[compact_df[date] >= \\\n",
    "                (pd.Timestamp(end) + pd.Timedelta('1 days')).strftime('%Y-%m-%d')]\n",
    "    train, test = train.dropna(), test.dropna()\n",
    "    return train, test\n",
    "\n",
    "def correl_selection(df, y, corr_li):\n",
    "    col_set = {}\n",
    "    for col_name in corr_li:\n",
    "        corr_ = abs(\n",
    "            df[\n",
    "                [col for col in df \n",
    "                 if (col.find(col_name) != -1)|(col.find(y) != -1)]\n",
    "            ].corr()[y]\n",
    "        ).sort_values(ascending=False)\n",
    "        col_set[corr_.index[1]] = corr_.iloc[1]\n",
    "    return col_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "import sklearn.metrics as met\n",
    "import pickle\n",
    "with open('raw_data_20190115.pickle', 'rb') as handle:\n",
    "    raw_data = pickle.load(handle)\n",
    "\n",
    "del_stock = []\n",
    "for stock_name, data in raw_data.items():\n",
    "    if int(str(data.index[0].year) + '{:02d}'.format(data.index[0].month)) > 201301:\n",
    "            print(stock_name)\n",
    "            del_stock.append(stock_name)\n",
    "\n",
    "for stock_name in del_stock:\n",
    "    del raw_data[stock_name]\n",
    "\n",
    "for stock_name, data in raw_data.items():\n",
    "    raw_data[stock_name] = data.reset_index()\n",
    "\n",
    "result_store= {}\n",
    "for stock_name in raw_data.keys():\n",
    "    chart_data = raw_data[stock_name].copy()\n",
    "    windows = [i for i in range(1, 90)]\n",
    "    prep_data, n_col = preprocess(chart_data, windows=windows)\n",
    "    training_data = build_feature(prep_data, \n",
    "                      close='Adj Close', high='High', low='Low')\n",
    "    del prep_data\n",
    "\n",
    "    start, end = '2013-01-01', '2018-01-01' # 훈련시킬 기간\n",
    "\n",
    "    thresh = .02 # Use only method=='Classification'\n",
    "    # periods = [i for i in range(1, 91)]\n",
    "    # windows = [i for i in range(1, 91, 3)]\n",
    "    periods = [1, 5, 10, 20, 30, 40, 50, 60]\n",
    "    windows = [i for i in range(1, 91, 3)]\n",
    "\n",
    "    prepare_y_dict = {}\n",
    "    for period in periods:\n",
    "        pred_n = 'pred_{:02}'.format(period)\n",
    "        print(pred_n + ' : (max precision 갱신 시 message 출력)')\n",
    "        _prepare = {}\n",
    "        max_precision, max_prec_day = 0, 0\n",
    "        compact_df, comp_y = concat_x_y(training_data, period=period, window=1, \n",
    "                                   n_col=n_col, use_fn='f', close='Adj Close',\n",
    "                                   thresh=thresh, method='regress')\n",
    "        train, test = train_test_split(compact_df, start, end, date='Date')\n",
    "        col_set = correl_selection(train, y=comp_y, corr_li=corr_li)\n",
    "        y_valid = train[comp_y].values\n",
    "        comp_y_tuple = (train, test, col_set)\n",
    "        for window in windows:\n",
    "            compact_df, y = concat_x_y(training_data, period=period, window=window, \n",
    "                                       n_col=n_col, use_fn='g', close='Adj Close',\n",
    "                                       thresh=thresh, method='regress')\n",
    "            train, test = train_test_split(compact_df, start, end, date='Date')\n",
    "            col_set = correl_selection(train, y=y, corr_li=corr_li)\n",
    "            n_cv = 5\n",
    "            ix_cv = int(len(train) / n_cv)\n",
    "            precision = 0\n",
    "            for i in range(n_cv - 1):\n",
    "                X_train = train[list(col_set.keys())].iloc[i*ix_cv:(i+1)*ix_cv]\n",
    "                y_train = train[y].iloc[i*ix_cv:(i+1)*ix_cv]\n",
    "                X_valid = train[list(col_set.keys())].iloc[(i+1)*ix_cv:(i+2)*ix_cv]\n",
    "                y_true = y_valid[(i+1)*ix_cv:(i+2)*ix_cv]\n",
    "\n",
    "                xgb_reg = xgb.XGBRegressor(\n",
    "                     learning_rate =0.15,\n",
    "                     n_estimators=100,\n",
    "                     max_depth=5,\n",
    "                     min_child_weight=2,\n",
    "                     gamma=0,\n",
    "                     subsample=0.8,\n",
    "                     colsample_bytree=0.8,\n",
    "                     nthread=4,\n",
    "                     scale_pos_weight=1,\n",
    "                     seed=27,\n",
    "                     eval_metric='map')\n",
    "                xgb_reg.fit(X_train, y_train)\n",
    "\n",
    "                y_pred = xgb_reg.predict(X_valid)\n",
    "                precision += met.precision_score(\n",
    "                                    np.where(y_true>0, 1, 0), \n",
    "                                    np.where(y_pred>0, 1, 0))\n",
    "            precision = precision / (n_cv - 1)\n",
    "            if (precision > max_precision):\n",
    "                max_prec_day = y[-2:]\n",
    "                max_y = y\n",
    "                max_precision = precision\n",
    "                print('\\t{} : {:.2%}'.format(y, max_precision), end='')\n",
    "            _prepare[y] = (train, test, col_set)\n",
    "\n",
    "        print('\\n\\tmax_precision_day : {}\\n\\tmax_precision : {:.2%}'.format(max_prec_day, max_precision))\n",
    "        train, test, col_set = _prepare[y[:-2]+max_prec_day]\n",
    "        prepare_y_dict[pred_n] = {comp_y : comp_y_tuple,\n",
    "                                  max_y : (train, test, col_set, max_prec_day, max_precision)}\n",
    "    result_store[stock_name] = prepare_y_dict"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}