FinRL_PaperTrading_Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FinRL_PaperTrading_Demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AI4Finance-Foundation/FinRL/blob/master/FinRL_PaperTrading_Demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V1ofncK2cYhs"
      },
      "source": [
        "Disclaimer: Nothing herein is financial advice, and NOT a recommendation to trade real money. Many platforms exist for simulated trading (paper trading) which can be used for building and developing the methods discussed. Please use common sense and always first consult a professional before trading or investing."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3mbRu3s1YlD"
      },
      "source": [
        "# Part 1: Install FinRL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0gkmsPgbvNf6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d421a902-2eab-4cd8-f842-bfb5ad3b3fef"
      },
      "source": [
        "## install finrl library\n",
        "!pip install git+https://github.com/AI4Finance-LLC/FinRL-Library.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/AI4Finance-LLC/FinRL-Library.git\n",
            "  Cloning https://github.com/AI4Finance-LLC/FinRL-Library.git to /tmp/pip-req-build-od0ekoy6\n",
            "  Running command git clone -q https://github.com/AI4Finance-LLC/FinRL-Library.git /tmp/pip-req-build-od0ekoy6\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-05_agdff/pyfolio_d9554690934d499785fcab16dd5cac5c\n",
            "  Running command git clone -q https://github.com/quantopian/pyfolio.git /tmp/pip-install-05_agdff/pyfolio_d9554690934d499785fcab16dd5cac5c\n",
            "Collecting elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl\n",
            "  Cloning https://github.com/AI4Finance-Foundation/ElegantRL.git to /tmp/pip-install-05_agdff/elegantrl_7154d79555e74afb85c897a1b9679942\n",
            "  Running command git clone -q https://github.com/AI4Finance-Foundation/ElegantRL.git /tmp/pip-install-05_agdff/elegantrl_7154d79555e74afb85c897a1b9679942\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (1.19.5)\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (1.1.5)\n",
            "Collecting stockstats\n",
            "  Downloading stockstats-0.3.2-py2.py3-none-any.whl (13 kB)\n",
            "Collecting yfinance\n",
            "  Downloading yfinance-0.1.64.tar.gz (26 kB)\n",
            "Collecting elegantrl\n",
            "  Downloading elegantrl-0.3.2-py3-none-any.whl (73 kB)\n",
            "\u001b[K     |████████████████████████████████| 73 kB 1.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (3.2.2)\n",
            "Requirement already satisfied: scikit-learn>=0.21.0 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (0.22.2.post1)\n",
            "Requirement already satisfied: gym>=0.17 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (0.17.3)\n",
            "Collecting stable-baselines3[extra]\n",
            "  Downloading stable_baselines3-1.3.0-py3-none-any.whl (174 kB)\n",
            "\u001b[K     |████████████████████████████████| 174 kB 11.4 MB/s \n",
            "\u001b[?25hCollecting ray[default]\n",
            "  Downloading ray-1.8.0-cp37-cp37m-manylinux2014_x86_64.whl (54.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 54.7 MB 1.4 MB/s \n",
            "\u001b[?25hCollecting lz4\n",
            "  Downloading lz4-3.1.3-cp37-cp37m-manylinux2010_x86_64.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 37.8 MB/s \n",
            "\u001b[?25hCollecting tensorboardX\n",
            "  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n",
            "\u001b[K     |████████████████████████████████| 124 kB 51.5 MB/s \n",
            "\u001b[?25hCollecting gputil\n",
            "  Downloading GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "Collecting trading_calendars\n",
            "  Downloading trading_calendars-2.1.1.tar.gz (108 kB)\n",
            "\u001b[K     |████████████████████████████████| 108 kB 46.0 MB/s \n",
            "\u001b[?25hCollecting alpaca_trade_api\n",
            "  Downloading alpaca_trade_api-1.4.1-py3-none-any.whl (36 kB)\n",
            "Collecting ccxt\n",
            "  Downloading ccxt-1.60.65-py2.py3-none-any.whl (2.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1 MB 34.8 MB/s \n",
            "\u001b[?25hCollecting jqdatasdk\n",
            "  Downloading jqdatasdk-1.8.10-py3-none-any.whl (153 kB)\n",
            "\u001b[K     |████████████████████████████████| 153 kB 46.3 MB/s \n",
            "\u001b[?25hCollecting wrds\n",
            "  Downloading wrds-3.1.1-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (3.6.4)\n",
            "Requirement already satisfied: setuptools>=41.4.0 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (57.4.0)\n",
            "Requirement already satisfied: wheel>=0.33.6 in /usr/local/lib/python3.7/dist-packages (from finrl==0.3.3) (0.37.0)\n",
            "Collecting pre-commit\n",
            "  Downloading pre_commit-2.15.0-py2.py3-none-any.whl (191 kB)\n",
            "\u001b[K     |████████████████████████████████| 191 kB 47.0 MB/s \n",
            "\u001b[?25hCollecting pybullet\n",
            "  Downloading pybullet-3.2.0-cp37-cp37m-manylinux1_x86_64.whl (89.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 89.3 MB 68 kB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl->finrl==0.3.3) (1.9.0+cu111)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl->finrl==0.3.3) (4.1.2.30)\n",
            "Collecting box2d-py\n",
            "  Downloading box2d_py-2.3.8-cp37-cp37m-manylinux1_x86_64.whl (448 kB)\n",
            "\u001b[K     |████████████████████████████████| 448 kB 54.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (5.5.0)\n",
            "Requirement already satisfied: pytz>=2014.10 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (2018.9)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (1.4.1)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.11.2)\n",
            "Collecting empyrical>=0.5.0\n",
            "  Downloading empyrical-0.5.5.tar.gz (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas-datareader>=0.2 in /usr/local/lib/python3.7/dist-packages (from empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.9.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17->finrl==0.3.3) (1.3.0)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17->finrl==0.3.3) (1.5.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (1.0.18)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (4.4.2)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.8.1)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (5.1.0)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (4.8.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.7.5)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.3) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.3) (1.3.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.3) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->finrl==0.3.3) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib->finrl==0.3.3) (1.15.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (4.2.6)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (2.23.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.2.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.17->finrl==0.3.3) (0.16.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pandas-datareader>=0.2->empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (3.0.4)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.0->finrl==0.3.3) (1.0.1)\n",
            "Requirement already satisfied: msgpack==1.0.2 in /usr/local/lib/python3.7/dist-packages (from alpaca_trade_api->finrl==0.3.3) (1.0.2)\n",
            "Collecting PyYAML==5.4.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 42.3 MB/s \n",
            "\u001b[?25hCollecting websockets<10,>=8.0\n",
            "  Downloading websockets-9.1-cp37-cp37m-manylinux2010_x86_64.whl (103 kB)\n",
            "\u001b[K     |████████████████████████████████| 103 kB 49.4 MB/s \n",
            "\u001b[?25hCollecting aiohttp==3.7.4\n",
            "  Downloading aiohttp-3.7.4-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 36.0 MB/s \n",
            "\u001b[?25hCollecting websocket-client<2,>=0.56.0\n",
            "  Downloading websocket_client-1.2.1-py2.py3-none-any.whl (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 1.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp==3.7.4->alpaca_trade_api->finrl==0.3.3) (21.2.0)\n",
            "Collecting async-timeout<4.0,>=3.0\n",
            "  Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 40.3 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 51.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp==3.7.4->alpaca_trade_api->finrl==0.3.3) (3.7.4.3)\n",
            "Collecting aiodns>=1.1.1\n",
            "  Downloading aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)\n",
            "\u001b[K     |████████████████████████████████| 294 kB 52.6 MB/s \n",
            "\u001b[?25hCollecting cryptography>=2.6.1\n",
            "  Downloading cryptography-35.0.0-cp36-abi3-manylinux_2_24_x86_64.whl (3.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.5 MB 44.0 MB/s \n",
            "\u001b[?25hCollecting pycares>=4.0.0\n",
            "  Downloading pycares-4.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (291 kB)\n",
            "\u001b[K     |████████████████████████████████| 291 kB 46.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.7/dist-packages (from cryptography>=2.6.1->ccxt->finrl==0.3.3) (1.14.6)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt->finrl==0.3.3) (2.20)\n",
            "Collecting pymysql>=0.7.6\n",
            "  Downloading PyMySQL-1.0.2-py3-none-any.whl (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 2.2 MB/s \n",
            "\u001b[?25hCollecting thriftpy2>=0.3.9\n",
            "  Downloading thriftpy2-0.4.14.tar.gz (361 kB)\n",
            "\u001b[K     |████████████████████████████████| 361 kB 50.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: SQLAlchemy>=1.2.8 in /usr/local/lib/python3.7/dist-packages (from jqdatasdk->finrl==0.3.3) (1.4.25)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from SQLAlchemy>=1.2.8->jqdatasdk->finrl==0.3.3) (4.8.1)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from SQLAlchemy>=1.2.8->jqdatasdk->finrl==0.3.3) (1.1.2)\n",
            "Collecting ply<4.0,>=3.4\n",
            "  Downloading ply-3.11-py2.py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 6.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->SQLAlchemy>=1.2.8->jqdatasdk->finrl==0.3.3) (3.6.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.3) (0.7.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.7/dist-packages (from pre-commit->finrl==0.3.3) (0.10.2)\n",
            "Collecting cfgv>=2.0.0\n",
            "  Downloading cfgv-3.3.1-py2.py3-none-any.whl (7.3 kB)\n",
            "Collecting identify>=1.0.0\n",
            "  Downloading identify-2.3.4-py2.py3-none-any.whl (98 kB)\n",
            "\u001b[K     |████████████████████████████████| 98 kB 7.6 MB/s \n",
            "\u001b[?25hCollecting nodeenv>=0.11.1\n",
            "  Downloading nodeenv-1.6.0-py2.py3-none-any.whl (21 kB)\n",
            "Collecting virtualenv>=20.0.8\n",
            "  Downloading virtualenv-20.10.0-py2.py3-none-any.whl (5.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.6 MB 37.3 MB/s \n",
            "\u001b[?25hCollecting distlib<1,>=0.3.1\n",
            "  Downloading distlib-0.3.3-py2.py3-none-any.whl (496 kB)\n",
            "\u001b[K     |████████████████████████████████| 496 kB 52.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock<4,>=3.2 in /usr/local/lib/python3.7/dist-packages (from virtualenv>=20.0.8->pre-commit->finrl==0.3.3) (3.3.0)\n",
            "Collecting platformdirs<3,>=2\n",
            "  Downloading platformdirs-2.4.0-py3-none-any.whl (14 kB)\n",
            "Collecting backports.entry-points-selectable>=1.0.4\n",
            "  Downloading backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl (6.2 kB)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.3) (1.4.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.3) (1.10.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.3) (8.10.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->finrl==0.3.3) (0.7.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (7.1.2)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (1.41.0)\n",
            "Collecting redis>=3.5.0\n",
            "  Downloading redis-3.5.3-py2.py3-none-any.whl (72 kB)\n",
            "\u001b[K     |████████████████████████████████| 72 kB 587 kB/s \n",
            "\u001b[?25hRequirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (2.6.0)\n",
            "Requirement already satisfied: protobuf>=3.15.3 in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (3.17.3)\n",
            "Collecting aioredis<2\n",
            "  Downloading aioredis-1.3.1-py3-none-any.whl (65 kB)\n",
            "\u001b[K     |████████████████████████████████| 65 kB 3.7 MB/s \n",
            "\u001b[?25hCollecting gpustat>=1.0.0b1\n",
            "  Downloading gpustat-1.0.0b1.tar.gz (82 kB)\n",
            "\u001b[K     |████████████████████████████████| 82 kB 255 kB/s \n",
            "\u001b[?25hCollecting aiohttp-cors\n",
            "  Downloading aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting py-spy>=0.2.0\n",
            "  Downloading py_spy-0.3.10-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2 MB 33.9 MB/s \n",
            "\u001b[?25hCollecting opencensus\n",
            "  Downloading opencensus-0.8.0-py2.py3-none-any.whl (128 kB)\n",
            "\u001b[K     |████████████████████████████████| 128 kB 47.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (0.12.0)\n",
            "Collecting colorful\n",
            "  Downloading colorful-0.5.4-py2.py3-none-any.whl (201 kB)\n",
            "\u001b[K     |████████████████████████████████| 201 kB 49.7 MB/s \n",
            "\u001b[?25hCollecting hiredis\n",
            "  Downloading hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (85 kB)\n",
            "\u001b[K     |████████████████████████████████| 85 kB 4.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.7/dist-packages (from gpustat>=1.0.0b1->ray[default]->finrl==0.3.3) (7.352.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from gpustat>=1.0.0b1->ray[default]->finrl==0.3.3) (5.4.8)\n",
            "Collecting blessed>=1.17.1\n",
            "  Downloading blessed-1.19.0-py2.py3-none-any.whl (57 kB)\n",
            "\u001b[K     |████████████████████████████████| 57 kB 5.3 MB/s \n",
            "\u001b[?25hCollecting opencensus-context==0.1.2\n",
            "  Downloading opencensus_context-0.1.2-py2.py3-none-any.whl (4.4 kB)\n",
            "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from opencensus->ray[default]->finrl==0.3.3) (1.26.3)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (1.53.0)\n",
            "Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (1.35.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (21.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (4.7.2)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.21.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]->finrl==0.3.3) (0.4.8)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from ray[default]->finrl==0.3.3) (0.8.9)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.3) (2.6.0)\n",
            "Requirement already satisfied: atari-py~=0.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.3) (0.2.9)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]->finrl==0.3.3) (7.1.2)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (1.8.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (0.12.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (0.4.6)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (1.3.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]->finrl==0.3.3) (3.1.1)\n",
            "Collecting int-date>=0.1.7\n",
            "  Downloading int_date-0.1.8-py2.py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.7/dist-packages (from trading_calendars->finrl==0.3.3) (0.11.1)\n",
            "Collecting psycopg2-binary\n",
            "  Downloading psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.4 MB 31.0 MB/s \n",
            "\u001b[?25hCollecting mock\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from yfinance->finrl==0.3.3) (0.0.9)\n",
            "Collecting lxml\n",
            "  Downloading lxml-4.6.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (6.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.3 MB 38.5 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: finrl, elegantrl, pyfolio, empyrical, gputil, thriftpy2, gpustat, trading-calendars, yfinance\n",
            "  Building wheel for finrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for finrl: filename=finrl-0.3.3-py3-none-any.whl size=3883610 sha256=0eb13d4e33ff777cf9485d4f3efbeb85a3610606a3e314d393ff9550421b61b8\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-npthaa53/wheels/17/ff/bd/1bc602a0352762b0b24041b88536d803ae343ed0a711fcf55e\n",
            "  Building wheel for elegantrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for elegantrl: filename=elegantrl-0.3.2-py3-none-any.whl size=77377 sha256=8d82cd965deb7af49f0a92f3ddcffdfd52302719783dee3eb191cce97f45a70e\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-npthaa53/wheels/99/85/5e/86cb3a9f47adfca5e248295e93113e1b298d60883126d62c84\n",
            "  Building wheel for pyfolio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyfolio: filename=pyfolio-0.9.2+75.g4b901f6-py3-none-any.whl size=75775 sha256=140936daefc388baea226a5ca1bb043e6a807835da0a6f63dcbeb721556dfddf\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-npthaa53/wheels/ef/09/e5/2c1bf37c050d22557c080deb1be986d06424627c04aeca19b9\n",
            "  Building wheel for empyrical (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for empyrical: filename=empyrical-0.5.5-py3-none-any.whl size=39777 sha256=31da0f6029beeec2ae1abaea2fb457e1450f15c95d7c990e2b659f7b5c222606\n",
            "  Stored in directory: /root/.cache/pip/wheels/d9/91/4b/654fcff57477efcf149eaca236da2fce991526cbab431bf312\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gputil: filename=GPUtil-1.4.0-py3-none-any.whl size=7411 sha256=f0948a656309d254792e6baf4a22e3b27c38128ada4c528283805eb4db74af82\n",
            "  Stored in directory: /root/.cache/pip/wheels/6e/f8/83/534c52482d6da64622ddbf72cd93c35d2ef2881b78fd08ff0c\n",
            "  Building wheel for thriftpy2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for thriftpy2: filename=thriftpy2-0.4.14-cp37-cp37m-linux_x86_64.whl size=940440 sha256=8bea54d8eda361c14f8ed5b5e458cdcc60b21abde1a0da838223c33c75c04a8f\n",
            "  Stored in directory: /root/.cache/pip/wheels/2a/f5/49/9c0d851aa64b58db72883cf9393cc824d536bdf13f5c83cff4\n",
            "  Building wheel for gpustat (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-1.0.0b1-py3-none-any.whl size=15979 sha256=e307cc40d671d03ed76355559ed950954b5228e8752a77e4b54a703074a0708d\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/16/e2/3e2437fba4c4b6a97a97bd96fce5d14e66cff5c4966fb1cc8c\n",
            "  Building wheel for trading-calendars (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for trading-calendars: filename=trading_calendars-2.1.1-py3-none-any.whl size=140937 sha256=1320a9705e3c340c874aeeaaf590731b6fbe1bdf12f0cfa8a98a94bd70b094b5\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/9c/d1/46a21e1b99e064cba79b85e9f95e6a208ac5ba4c29ae5962ec\n",
            "  Building wheel for yfinance (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for yfinance: filename=yfinance-0.1.64-py2.py3-none-any.whl size=24109 sha256=5d49441af5dbd5ce65827d2d58288cc4019be15e5b120efc0f2b8ddd9f9ab325\n",
            "  Stored in directory: /root/.cache/pip/wheels/86/fe/9b/a4d3d78796b699e37065e5b6c27b75cff448ddb8b24943c288\n",
            "Successfully built finrl elegantrl pyfolio empyrical gputil thriftpy2 gpustat trading-calendars yfinance\n",
            "Installing collected packages: multidict, yarl, lxml, async-timeout, redis, PyYAML, pycares, ply, platformdirs, opencensus-context, hiredis, distlib, blessed, backports.entry-points-selectable, aiohttp, websockets, websocket-client, virtualenv, thriftpy2, tensorboardX, stable-baselines3, ray, pymysql, pybullet, py-spy, psycopg2-binary, opencensus, nodeenv, mock, int-date, identify, gpustat, empyrical, cryptography, colorful, cfgv, box2d-py, aioredis, aiohttp-cors, aiodns, yfinance, wrds, trading-calendars, stockstats, pyfolio, pre-commit, lz4, jqdatasdk, gputil, elegantrl, ccxt, alpaca-trade-api, finrl\n",
            "  Attempting uninstall: lxml\n",
            "    Found existing installation: lxml 4.2.6\n",
            "    Uninstalling lxml-4.2.6:\n",
            "      Successfully uninstalled lxml-4.2.6\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-5.4.1 aiodns-3.0.0 aiohttp-3.7.4 aiohttp-cors-0.7.0 aioredis-1.3.1 alpaca-trade-api-1.4.1 async-timeout-3.0.1 backports.entry-points-selectable-1.1.0 blessed-1.19.0 box2d-py-2.3.8 ccxt-1.60.65 cfgv-3.3.1 colorful-0.5.4 cryptography-35.0.0 distlib-0.3.3 elegantrl-0.3.2 empyrical-0.5.5 finrl-0.3.3 gpustat-1.0.0b1 gputil-1.4.0 hiredis-2.0.0 identify-2.3.4 int-date-0.1.8 jqdatasdk-1.8.10 lxml-4.6.4 lz4-3.1.3 mock-4.0.3 multidict-5.2.0 nodeenv-1.6.0 opencensus-0.8.0 opencensus-context-0.1.2 platformdirs-2.4.0 ply-3.11 pre-commit-2.15.0 psycopg2-binary-2.9.1 py-spy-0.3.10 pybullet-3.2.0 pycares-4.1.2 pyfolio-0.9.2+75.g4b901f6 pymysql-1.0.2 ray-1.8.0 redis-3.5.3 stable-baselines3-1.3.0 stockstats-0.3.2 tensorboardX-2.4 thriftpy2-0.4.14 trading-calendars-2.1.1 virtualenv-20.10.0 websocket-client-1.2.1 websockets-9.1 wrds-3.1.1 yarl-1.6.3 yfinance-0.1.64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--6Kx8I21erH"
      },
      "source": [
        "## Import related modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H7I7zsyYfoLJ",
        "outputId": "73fd67c1-222c-450c-a51d-eea50e3a5eeb"
      },
      "source": [
        "from finrl.train import train\n",
        "from finrl.test import test\n",
        "from finrl.apps.config import DOW_30_TICKER\n",
        "from finrl.apps.config import TECHNICAL_INDICATORS_LIST\n",
        "from finrl.finrl_meta.env_stock_trading.env_stocktrading_np import StockTradingEnv\n",
        "from finrl.finrl_meta.env_stock_trading.env_stock_papertrading import AlpacaPaperTrading\n",
        "from finrl.finrl_meta.data_processor import DataProcessor\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/pos.py:27: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "  'Module \"zipline.assets\" not found; multipliers will not be applied'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pf5aVHAU-xF6"
      },
      "source": [
        "## Import Dow Jones 30 Symbols"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jx25TA_X87F-"
      },
      "source": [
        "ticker_list = DOW_30_TICKER\n",
        "action_dim = len(DOW_30_TICKER)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UIV0kO_y-inG",
        "outputId": "4892aec0-cf63-43d7-c114-ed2fc8b5588c"
      },
      "source": [
        "print(ticker_list)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CSCO', 'CVX', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'KO', 'JPM', 'MCD', 'MMM', 'MRK', 'MSFT', 'NKE', 'PG', 'TRV', 'UNH', 'CRM', 'VZ', 'V', 'WBA', 'WMT', 'DIS', 'DOW']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CnqQ-cC5-rfO",
        "outputId": "e84ce69a-c049-4c2f-e17d-22a13f6b4789"
      },
      "source": [
        "print(TECHNICAL_INDICATORS_LIST)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['macd', 'boll_ub', 'boll_lb', 'rsi_30', 'cci_30', 'dx_30', 'close_30_sma', 'close_60_sma']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZMkcyjZ-25l"
      },
      "source": [
        "## Calculate the DRL state dimension manually for paper trading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLfkTsXK-e90"
      },
      "source": [
        "# amount + (turbulence, turbulence_bool) + (price, shares, cd (holding time)) * stock_dim + tech_dim\n",
        "state_dim = 1 + 2 + 3 * action_dim + len(TECHNICAL_INDICATORS_LIST) * action_dim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QqUkvImG-n66",
        "outputId": "1b9f80c8-ef04-4138-9ab5-6bab406d8d1b"
      },
      "source": [
        "state_dim"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "333"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3rwy7V72-8YY"
      },
      "source": [
        "## Get the API Keys Ready"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Z6qlLXY-fA2"
      },
      "source": [
        "API_KEY = \"YOUR_ALPACA_API_KEY\"\n",
        "API_SECRET = \"YOUR_ALPACA_API_SECRET\"\n",
        "APCA_API_BASE_URL = 'https://paper-api.alpaca.markets'\n",
        "data_url = 'wss://data.alpaca.markets'\n",
        "env = StockTradingEnv"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J25MuZLiGqCP"
      },
      "source": [
        "## Show the data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "puJZWm8NHtSN"
      },
      "source": [
        "### Step 1. Pick a data source"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ZCru8f7GqgL",
        "outputId": "7c227e80-674e-4aba-df1a-be130fd65b69"
      },
      "source": [
        "DP = DataProcessor(data_source = 'alpaca',\n",
        "                  API_KEY = API_KEY, \n",
        "                  API_SECRET = API_SECRET, \n",
        "                  APCA_API_BASE_URL = APCA_API_BASE_URL\n",
        "                  )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alpaca successfully connected\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nvPEW2mYHvkR"
      },
      "source": [
        "### Step 2. Get ticker list, Set start date and end date, specify the data frequency"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NPNxj6c8HIiE",
        "outputId": "72c38958-3beb-44da-94e3-04f342599539"
      },
      "source": [
        "data = DP.download_data(start_date = '2021-10-01', \n",
        "                        end_date = '2021-10-05',\n",
        "                        ticker_list = ticker_list, \n",
        "                        time_interval= '1Min')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data before 2021-10-01T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-02T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-03T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-04T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-05T15:59:00-04:00 is successfully fetched\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i46jGdE0IAel"
      },
      "source": [
        "### Step 3. Data Cleaning & Feature Engineering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x9euUsEPHWFK",
        "outputId": "e513d0d7-43ef-40b4-b0b5-b329bdfb0d1a"
      },
      "source": [
        "data = DP.clean_data(data)\n",
        "data = DP.add_technical_indicator(data, TECHNICAL_INDICATORS_LIST)\n",
        "data = DP.add_vix(data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data clean finished!\n",
            "Succesfully add technical indicators\n",
            "Data before 2021-10-01T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-02T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-03T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-04T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-05T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "GOcPTaAgHdxa",
        "outputId": "0ec040ba-a7cf-48d0-c7a9-f11770da68f1"
      },
      "source": [
        "data.tail(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>volume</th>\n",
              "      <th>tic</th>\n",
              "      <th>macd</th>\n",
              "      <th>boll_ub</th>\n",
              "      <th>boll_lb</th>\n",
              "      <th>rsi_30</th>\n",
              "      <th>cci_30</th>\n",
              "      <th>dx_30</th>\n",
              "      <th>close_30_sma</th>\n",
              "      <th>close_60_sma</th>\n",
              "      <th>VIXY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>35080</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>386.740</td>\n",
              "      <td>386.740</td>\n",
              "      <td>385.860</td>\n",
              "      <td>385.860</td>\n",
              "      <td>2904.0</td>\n",
              "      <td>GS</td>\n",
              "      <td>-0.207385</td>\n",
              "      <td>387.589166</td>\n",
              "      <td>386.345834</td>\n",
              "      <td>33.723238</td>\n",
              "      <td>-357.485069</td>\n",
              "      <td>51.604691</td>\n",
              "      <td>387.049333</td>\n",
              "      <td>387.364167</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35081</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>329.600</td>\n",
              "      <td>329.600</td>\n",
              "      <td>329.310</td>\n",
              "      <td>329.430</td>\n",
              "      <td>1787.0</td>\n",
              "      <td>HD</td>\n",
              "      <td>-0.448482</td>\n",
              "      <td>331.711177</td>\n",
              "      <td>329.320823</td>\n",
              "      <td>26.228665</td>\n",
              "      <td>-169.376805</td>\n",
              "      <td>76.054777</td>\n",
              "      <td>330.815333</td>\n",
              "      <td>331.213250</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35082</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>215.720</td>\n",
              "      <td>215.720</td>\n",
              "      <td>215.590</td>\n",
              "      <td>215.590</td>\n",
              "      <td>1493.0</td>\n",
              "      <td>HON</td>\n",
              "      <td>-0.240973</td>\n",
              "      <td>217.088555</td>\n",
              "      <td>215.553945</td>\n",
              "      <td>18.327939</td>\n",
              "      <td>-191.493943</td>\n",
              "      <td>68.521373</td>\n",
              "      <td>216.415833</td>\n",
              "      <td>216.538000</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35083</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>143.140</td>\n",
              "      <td>143.180</td>\n",
              "      <td>143.100</td>\n",
              "      <td>143.160</td>\n",
              "      <td>4180.0</td>\n",
              "      <td>IBM</td>\n",
              "      <td>-0.145148</td>\n",
              "      <td>143.866517</td>\n",
              "      <td>143.059983</td>\n",
              "      <td>30.924818</td>\n",
              "      <td>-160.386451</td>\n",
              "      <td>48.082544</td>\n",
              "      <td>143.534000</td>\n",
              "      <td>143.722417</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35084</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>53.955</td>\n",
              "      <td>53.975</td>\n",
              "      <td>53.935</td>\n",
              "      <td>53.945</td>\n",
              "      <td>6274.0</td>\n",
              "      <td>INTC</td>\n",
              "      <td>-0.038827</td>\n",
              "      <td>54.166900</td>\n",
              "      <td>53.925100</td>\n",
              "      <td>29.721071</td>\n",
              "      <td>-169.470945</td>\n",
              "      <td>57.246117</td>\n",
              "      <td>54.060000</td>\n",
              "      <td>54.096000</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35085</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>159.720</td>\n",
              "      <td>159.730</td>\n",
              "      <td>159.540</td>\n",
              "      <td>159.580</td>\n",
              "      <td>4075.0</td>\n",
              "      <td>JNJ</td>\n",
              "      <td>-0.073306</td>\n",
              "      <td>160.073831</td>\n",
              "      <td>159.657669</td>\n",
              "      <td>34.877315</td>\n",
              "      <td>-306.971514</td>\n",
              "      <td>34.150919</td>\n",
              "      <td>159.888667</td>\n",
              "      <td>160.026250</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35086</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>168.870</td>\n",
              "      <td>168.880</td>\n",
              "      <td>168.650</td>\n",
              "      <td>168.650</td>\n",
              "      <td>7616.0</td>\n",
              "      <td>JPM</td>\n",
              "      <td>-0.140191</td>\n",
              "      <td>169.514417</td>\n",
              "      <td>168.669583</td>\n",
              "      <td>33.354857</td>\n",
              "      <td>-173.244738</td>\n",
              "      <td>42.521277</td>\n",
              "      <td>169.178000</td>\n",
              "      <td>169.313083</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35087</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>53.085</td>\n",
              "      <td>53.090</td>\n",
              "      <td>53.055</td>\n",
              "      <td>53.080</td>\n",
              "      <td>15679.0</td>\n",
              "      <td>KO</td>\n",
              "      <td>-0.012753</td>\n",
              "      <td>53.198559</td>\n",
              "      <td>53.068441</td>\n",
              "      <td>42.715325</td>\n",
              "      <td>-190.357440</td>\n",
              "      <td>29.750107</td>\n",
              "      <td>53.138333</td>\n",
              "      <td>53.137750</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35088</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>245.330</td>\n",
              "      <td>245.330</td>\n",
              "      <td>245.180</td>\n",
              "      <td>245.180</td>\n",
              "      <td>2704.0</td>\n",
              "      <td>MCD</td>\n",
              "      <td>-0.269918</td>\n",
              "      <td>246.739285</td>\n",
              "      <td>245.066715</td>\n",
              "      <td>21.785286</td>\n",
              "      <td>-160.170797</td>\n",
              "      <td>79.728448</td>\n",
              "      <td>246.058333</td>\n",
              "      <td>246.202083</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35089</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>177.960</td>\n",
              "      <td>177.960</td>\n",
              "      <td>177.790</td>\n",
              "      <td>177.790</td>\n",
              "      <td>1754.0</td>\n",
              "      <td>MMM</td>\n",
              "      <td>-0.076775</td>\n",
              "      <td>178.495952</td>\n",
              "      <td>177.863548</td>\n",
              "      <td>34.300943</td>\n",
              "      <td>-232.273583</td>\n",
              "      <td>44.417489</td>\n",
              "      <td>178.213500</td>\n",
              "      <td>178.172583</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35090</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>81.595</td>\n",
              "      <td>81.610</td>\n",
              "      <td>81.575</td>\n",
              "      <td>81.590</td>\n",
              "      <td>8480.0</td>\n",
              "      <td>MRK</td>\n",
              "      <td>-0.054943</td>\n",
              "      <td>81.956850</td>\n",
              "      <td>81.510150</td>\n",
              "      <td>38.831341</td>\n",
              "      <td>-142.857143</td>\n",
              "      <td>25.207846</td>\n",
              "      <td>81.732167</td>\n",
              "      <td>81.815083</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35091</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>288.850</td>\n",
              "      <td>288.880</td>\n",
              "      <td>288.530</td>\n",
              "      <td>288.730</td>\n",
              "      <td>24244.0</td>\n",
              "      <td>MSFT</td>\n",
              "      <td>-0.146467</td>\n",
              "      <td>289.571293</td>\n",
              "      <td>288.706207</td>\n",
              "      <td>39.004141</td>\n",
              "      <td>-188.379205</td>\n",
              "      <td>42.062003</td>\n",
              "      <td>289.225667</td>\n",
              "      <td>289.492667</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35092</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>149.580</td>\n",
              "      <td>149.590</td>\n",
              "      <td>149.465</td>\n",
              "      <td>149.520</td>\n",
              "      <td>6883.0</td>\n",
              "      <td>NKE</td>\n",
              "      <td>-0.111984</td>\n",
              "      <td>150.142632</td>\n",
              "      <td>149.492368</td>\n",
              "      <td>37.775018</td>\n",
              "      <td>-166.219839</td>\n",
              "      <td>60.708595</td>\n",
              "      <td>149.912500</td>\n",
              "      <td>149.907083</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35093</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>139.430</td>\n",
              "      <td>139.430</td>\n",
              "      <td>139.310</td>\n",
              "      <td>139.310</td>\n",
              "      <td>3316.0</td>\n",
              "      <td>PG</td>\n",
              "      <td>-0.083896</td>\n",
              "      <td>139.923238</td>\n",
              "      <td>139.396262</td>\n",
              "      <td>31.750025</td>\n",
              "      <td>-264.842747</td>\n",
              "      <td>48.807575</td>\n",
              "      <td>139.713667</td>\n",
              "      <td>139.735833</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35094</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>153.570</td>\n",
              "      <td>153.650</td>\n",
              "      <td>153.570</td>\n",
              "      <td>153.650</td>\n",
              "      <td>1320.0</td>\n",
              "      <td>TRV</td>\n",
              "      <td>-0.038149</td>\n",
              "      <td>153.850881</td>\n",
              "      <td>153.514619</td>\n",
              "      <td>46.886586</td>\n",
              "      <td>-74.290919</td>\n",
              "      <td>24.670390</td>\n",
              "      <td>153.720500</td>\n",
              "      <td>153.740417</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35095</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>393.900</td>\n",
              "      <td>393.900</td>\n",
              "      <td>393.200</td>\n",
              "      <td>393.200</td>\n",
              "      <td>2778.0</td>\n",
              "      <td>UNH</td>\n",
              "      <td>-0.337330</td>\n",
              "      <td>395.924222</td>\n",
              "      <td>393.719278</td>\n",
              "      <td>35.160318</td>\n",
              "      <td>-314.024950</td>\n",
              "      <td>44.275058</td>\n",
              "      <td>394.942833</td>\n",
              "      <td>395.146833</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35096</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>224.340</td>\n",
              "      <td>224.380</td>\n",
              "      <td>224.200</td>\n",
              "      <td>224.270</td>\n",
              "      <td>5364.0</td>\n",
              "      <td>V</td>\n",
              "      <td>-0.232614</td>\n",
              "      <td>225.621581</td>\n",
              "      <td>224.186419</td>\n",
              "      <td>27.798707</td>\n",
              "      <td>-173.469388</td>\n",
              "      <td>71.642315</td>\n",
              "      <td>225.030167</td>\n",
              "      <td>225.164333</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35097</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>54.550</td>\n",
              "      <td>54.555</td>\n",
              "      <td>54.500</td>\n",
              "      <td>54.500</td>\n",
              "      <td>13804.0</td>\n",
              "      <td>VZ</td>\n",
              "      <td>-0.026883</td>\n",
              "      <td>54.660802</td>\n",
              "      <td>54.528698</td>\n",
              "      <td>29.257106</td>\n",
              "      <td>-236.286920</td>\n",
              "      <td>59.868748</td>\n",
              "      <td>54.610167</td>\n",
              "      <td>54.652333</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35098</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>47.100</td>\n",
              "      <td>47.100</td>\n",
              "      <td>47.080</td>\n",
              "      <td>47.090</td>\n",
              "      <td>3300.0</td>\n",
              "      <td>WBA</td>\n",
              "      <td>-0.000467</td>\n",
              "      <td>47.216146</td>\n",
              "      <td>47.085354</td>\n",
              "      <td>47.064782</td>\n",
              "      <td>-62.975239</td>\n",
              "      <td>26.794769</td>\n",
              "      <td>47.125000</td>\n",
              "      <td>47.107500</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35099</th>\n",
              "      <td>2021-10-05 15:59:00-04:00</td>\n",
              "      <td>136.690</td>\n",
              "      <td>136.710</td>\n",
              "      <td>136.600</td>\n",
              "      <td>136.620</td>\n",
              "      <td>5695.0</td>\n",
              "      <td>WMT</td>\n",
              "      <td>-0.076694</td>\n",
              "      <td>137.037700</td>\n",
              "      <td>136.663800</td>\n",
              "      <td>31.557994</td>\n",
              "      <td>-218.758646</td>\n",
              "      <td>61.392805</td>\n",
              "      <td>136.904833</td>\n",
              "      <td>136.971083</td>\n",
              "      <td>21.68</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           time     open  ...  close_60_sma   VIXY\n",
              "35080 2021-10-05 15:59:00-04:00  386.740  ...    387.364167  21.68\n",
              "35081 2021-10-05 15:59:00-04:00  329.600  ...    331.213250  21.68\n",
              "35082 2021-10-05 15:59:00-04:00  215.720  ...    216.538000  21.68\n",
              "35083 2021-10-05 15:59:00-04:00  143.140  ...    143.722417  21.68\n",
              "35084 2021-10-05 15:59:00-04:00   53.955  ...     54.096000  21.68\n",
              "35085 2021-10-05 15:59:00-04:00  159.720  ...    160.026250  21.68\n",
              "35086 2021-10-05 15:59:00-04:00  168.870  ...    169.313083  21.68\n",
              "35087 2021-10-05 15:59:00-04:00   53.085  ...     53.137750  21.68\n",
              "35088 2021-10-05 15:59:00-04:00  245.330  ...    246.202083  21.68\n",
              "35089 2021-10-05 15:59:00-04:00  177.960  ...    178.172583  21.68\n",
              "35090 2021-10-05 15:59:00-04:00   81.595  ...     81.815083  21.68\n",
              "35091 2021-10-05 15:59:00-04:00  288.850  ...    289.492667  21.68\n",
              "35092 2021-10-05 15:59:00-04:00  149.580  ...    149.907083  21.68\n",
              "35093 2021-10-05 15:59:00-04:00  139.430  ...    139.735833  21.68\n",
              "35094 2021-10-05 15:59:00-04:00  153.570  ...    153.740417  21.68\n",
              "35095 2021-10-05 15:59:00-04:00  393.900  ...    395.146833  21.68\n",
              "35096 2021-10-05 15:59:00-04:00  224.340  ...    225.164333  21.68\n",
              "35097 2021-10-05 15:59:00-04:00   54.550  ...     54.652333  21.68\n",
              "35098 2021-10-05 15:59:00-04:00   47.100  ...     47.107500  21.68\n",
              "35099 2021-10-05 15:59:00-04:00  136.690  ...    136.971083  21.68\n",
              "\n",
              "[20 rows x 16 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bbu03L_UIMWt"
      },
      "source": [
        "### Step 4. Transform to numpy array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rzj0vjZZHdGM",
        "outputId": "c6e08f9e-034b-4161-f8f3-2e61426a78c0"
      },
      "source": [
        "price_array, tech_array, turbulence_array = DP.df_to_array(data, if_vix='True')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Successfully transformed into array\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "778F_9aVJPNq",
        "outputId": "cdb23360-53da-48f1-c4d2-815297d3b4b1"
      },
      "source": [
        "price_array"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[141.89 , 213.36 , 174.74 , ...,  54.09 ,  47.075, 139.575],\n",
              "       [141.42 , 212.62 , 168.69 , ...,  54.17 ,  47.16 , 139.53 ],\n",
              "       [141.08 , 212.5  , 169.46 , ...,  54.11 ,  47.17 , 139.445],\n",
              "       ...,\n",
              "       [141.26 , 211.87 , 175.06 , ...,  54.595,  47.13 , 136.78 ],\n",
              "       [141.12 , 211.85 , 174.98 , ...,  54.535,  47.11 , 136.68 ],\n",
              "       [140.98 , 211.77 , 174.74 , ...,  54.5  ,  47.09 , 136.62 ]])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eW0UDAXI1nEa"
      },
      "source": [
        "# Part 2: Train the agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lArLOFcJ7VMO"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g1F84mebj4gu"
      },
      "source": [
        "ERL_PARAMS = {\"learning_rate\": 3e-6,\"batch_size\": 2048,\"gamma\":  0.985,\n",
        "        \"seed\":312,\"net_dimension\":512, \"target_step\":5000, \"eval_gap\":30,\n",
        "        \"eval_times\":1} \n",
        "#if you want to use larger datasets (change to longer period), and it raises error, \n",
        "#please try to increase \"target_step\". It should be larger than the episode steps. "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BxcNI2fdNjip",
        "outputId": "5f638ed6-7dbf-4771-a0cc-3617d5e9b875"
      },
      "source": [
        "train(start_date = '2021-10-11', \n",
        "      end_date = '2021-10-15',\n",
        "      ticker_list = ticker_list, \n",
        "      data_source = 'alpaca',\n",
        "      time_interval= '1Min', \n",
        "      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "      drl_lib='elegantrl', \n",
        "      env=env,\n",
        "      model_name='ppo', \n",
        "      API_KEY = API_KEY, \n",
        "      API_SECRET = API_SECRET, \n",
        "      APCA_API_BASE_URL = APCA_API_BASE_URL,\n",
        "      erl_params=ERL_PARAMS,\n",
        "      cwd='./papertrading_erl', #current_working_dir\n",
        "      break_step=1e5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alpaca successfully connected\n",
            "Data before 2021-10-11T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-12T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-13T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-14T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-15T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Succesfully add technical indicators\n",
            "Data before 2021-10-11T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-12T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-13T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-14T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-15T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Successfully transformed into array\n",
            "| Remove cwd: ./papertrading_erl\n",
            "################################################################################\n",
            "ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.\n",
            "0  3.90e+03    1.01 |\n",
            "0  3.90e+03    1.01 |    1.01    0.0   1948     0 |   -0.06   2.54  -0.01  -0.50\n",
            "0  9.75e+03    1.01 |    1.01    0.0   1948     0 |   -0.06   2.42  -0.04  -0.50\n",
            "0  2.92e+04    1.01 |\n",
            "0  2.92e+04    1.01 |    1.01    0.0   1948     0 |   -0.05   1.20  -0.03  -0.50\n",
            "0  3.31e+04    1.01 |\n",
            "0  3.31e+04    1.01 |    1.01    0.0   1948     0 |   -0.05   1.06  -0.05  -0.50\n",
            "0  3.90e+04    1.02 |\n",
            "0  3.90e+04    1.02 |    1.02    0.0   1948     0 |   -0.05   1.43   0.01  -0.50\n",
            "0  4.48e+04    1.02 |    1.01    0.0   1948     0 |   -0.04   0.95  -0.00  -0.50\n",
            "0  6.43e+04    1.02 |    1.01    0.0   1948     0 |   -0.04   0.74   0.01  -0.50\n",
            "0  8.38e+04    1.02 |    1.01    0.0   1948     0 |   -0.04   0.61  -0.02  -0.50\n",
            "0  1.03e+05    1.02 |    1.01    0.0   1948     0 |   -0.03   0.60  -0.02  -0.50\n",
            "| UsedTime:     289 | SavedDir: ./papertrading_erl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g37WugV_1pAS"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SxYoWCDa02TW",
        "outputId": "b542ce55-2479-4168-fb2a-79f0a3da12c3"
      },
      "source": [
        "account_value_erl=test(start_date = '2021-10-18', \n",
        "                      end_date = '2021-10-19',\n",
        "                      ticker_list = ticker_list, \n",
        "                      data_source = 'alpaca',\n",
        "                      time_interval= '1Min', \n",
        "                      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "                      drl_lib='elegantrl', \n",
        "                      env=env, \n",
        "                      model_name='ppo', \n",
        "                      API_KEY = API_KEY, \n",
        "                      API_SECRET = API_SECRET, \n",
        "                      APCA_API_BASE_URL = APCA_API_BASE_URL,\n",
        "                      cwd='./papertrading_erl',\n",
        "                      net_dimension = 512)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alpaca successfully connected\n",
            "Data before 2021-10-18T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-19T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Succesfully add technical indicators\n",
            "Data before 2021-10-18T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-19T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Successfully transformed into array\n",
            "price_array:  780\n",
            "Test Finished!\n",
            "episode_return 1.0075374272908477\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e8aNQ58X7avM"
      },
      "source": [
        "## Use full data to train "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CQ9_Yv41r88"
      },
      "source": [
        "After tuning well, retrain on the training and testing sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cUSgbwt_10V3",
        "outputId": "e7669aa1-b5d2-48b5-8aee-509a2822b7b4"
      },
      "source": [
        "train(start_date = '2021-10-11', \n",
        "      end_date = '2021-10-19',\n",
        "      ticker_list = ticker_list, \n",
        "      data_source = 'alpaca',\n",
        "      time_interval= '1Min', \n",
        "      technical_indicator_list= TECHNICAL_INDICATORS_LIST,\n",
        "      drl_lib='elegantrl', \n",
        "      env=env, \n",
        "      model_name='ppo', \n",
        "      API_KEY = API_KEY, \n",
        "      API_SECRET = API_SECRET, \n",
        "      APCA_API_BASE_URL = APCA_API_BASE_URL,\n",
        "      erl_params=ERL_PARAMS,\n",
        "      cwd='./papertrading_erl_retrain',\n",
        "      break_step=5e4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alpaca successfully connected\n",
            "Data before 2021-10-11T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-12T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-13T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-14T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-15T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-16T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-17T15:59:00-04:00 is successfully fetched\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "sleep 3 seconds and retrying https://data.alpaca.markets/v1/bars/1Min 3 more time(s)...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data before 2021-10-18T15:59:00-04:00 is successfully fetched\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "sleep 3 seconds and retrying https://data.alpaca.markets/v1/bars/1Min 3 more time(s)...\n",
            "sleep 3 seconds and retrying https://data.alpaca.markets/v1/bars/1Min 3 more time(s)...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data before 2021-10-19T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Succesfully add technical indicators\n",
            "Data before 2021-10-11T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-12T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-13T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-14T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-15T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-16T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-17T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-18T15:59:00-04:00 is successfully fetched\n",
            "Data before 2021-10-19T15:59:00-04:00 is successfully fetched\n",
            "Data clean finished!\n",
            "Successfully transformed into array\n",
            "| Remove cwd: ./papertrading_erl_retrain\n",
            "################################################################################\n",
            "ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.\n",
            "0  2.73e+03    1.01 |\n",
            "0  2.73e+03    1.01 |    1.01    0.0   2728     0 |   -0.05   2.83  -0.04  -0.50\n",
            "| UsedTime:      99 | SavedDir: ./papertrading_erl_retrain\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sIQN6Ggt7gXY"
      },
      "source": [
        "# Part 3: Deploy the agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UFoxkigg1zXa"
      },
      "source": [
        "## Setup Alpaca Paper trading environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpkoZpYzfneS"
      },
      "source": [
        "import datetime\n",
        "import threading\n",
        "from finrl.finrl_meta.data_processors.processor_alpaca import AlpacaProcessor\n",
        "import alpaca_trade_api as tradeapi\n",
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import gym\n",
        "\n",
        "class AlpacaPaperTrading():\n",
        "\n",
        "    def __init__(self,ticker_list, time_interval, drl_lib, agent, cwd, net_dim, \n",
        "                 state_dim, action_dim, API_KEY, API_SECRET, \n",
        "                 APCA_API_BASE_URL, tech_indicator_list, turbulence_thresh=30, \n",
        "                 max_stock=1e2, latency = None):\n",
        "        #load agent\n",
        "        self.drl_lib = drl_lib\n",
        "        if agent =='ppo':\n",
        "            if drl_lib == 'elegantrl':              \n",
        "                from elegantrl.agent import AgentPPO\n",
        "                from elegantrl.run import Arguments, init_agent\n",
        "                #load agent\n",
        "                config = {'state_dim':state_dim,\n",
        "                            'action_dim':action_dim,}\n",
        "                args = Arguments(agent=AgentPPO, env=StockEnvEmpty(config))\n",
        "                args.cwd = cwd\n",
        "                args.net_dim = net_dim\n",
        "                # load agent\n",
        "                try:\n",
        "                    agent = init_agent(args, gpu_id = 0)\n",
        "                    self.act = agent.act\n",
        "                    self.device = agent.device\n",
        "                except BaseException:\n",
        "                    raise ValueError(\"Fail to load agent!\")\n",
        "                        \n",
        "            elif drl_lib == 'rllib':\n",
        "                from ray.rllib.agents import ppo\n",
        "                from ray.rllib.agents.ppo.ppo import PPOTrainer\n",
        "                \n",
        "                config = ppo.DEFAULT_CONFIG.copy()\n",
        "                config['env'] = StockEnvEmpty\n",
        "                config[\"log_level\"] = \"WARN\"\n",
        "                config['env_config'] = {'state_dim':state_dim,\n",
        "                            'action_dim':action_dim,}\n",
        "                trainer = PPOTrainer(env=StockEnvEmpty, config=config)\n",
        "                trainer.restore(cwd)\n",
        "                try:\n",
        "                    trainer.restore(cwd)\n",
        "                    self.agent = trainer\n",
        "                    print(\"Restoring from checkpoint path\", cwd)\n",
        "                except:\n",
        "                    raise ValueError('Fail to load agent!')\n",
        "                    \n",
        "            elif drl_lib == 'stable_baselines3':\n",
        "                from stable_baselines3 import PPO\n",
        "                \n",
        "                try:\n",
        "                    #load agent\n",
        "                    self.model = PPO.load(cwd)\n",
        "                    print(\"Successfully load model\", cwd)\n",
        "                except:\n",
        "                    raise ValueError('Fail to load agent!')\n",
        "                    \n",
        "            else:\n",
        "                raise ValueError('The DRL library input is NOT supported yet. Please check your input.')\n",
        "               \n",
        "        else:\n",
        "            raise ValueError('Agent input is NOT supported yet.')\n",
        "            \n",
        "            \n",
        "            \n",
        "        #connect to Alpaca trading API\n",
        "        try:\n",
        "            self.alpaca = tradeapi.REST(API_KEY,API_SECRET,APCA_API_BASE_URL, 'v2')\n",
        "        except:\n",
        "            raise ValueError('Fail to connect Alpaca. Please check account info and internet connection.')\n",
        "        \n",
        "        #read trading time interval\n",
        "        if time_interval == '1s':\n",
        "            self.time_interval = 1\n",
        "        elif time_interval == '5s':\n",
        "            self.time_interval = 5\n",
        "        elif time_interval == '1Min':\n",
        "            self.time_interval = 60\n",
        "        elif time_interval == '5Min':\n",
        "            self.time_interval = 60 * 5\n",
        "        elif time_interval == '15Min':\n",
        "            self.time_interval = 60 * 15\n",
        "        else:\n",
        "            raise ValueError('Time interval input is NOT supported yet.')\n",
        "        \n",
        "        #read trading settings\n",
        "        self.tech_indicator_list = tech_indicator_list\n",
        "        self.turbulence_thresh = turbulence_thresh\n",
        "        self.max_stock = max_stock \n",
        "        \n",
        "        #initialize account\n",
        "        self.stocks = np.asarray([0] * len(ticker_list)) #stocks holding\n",
        "        self.stocks_cd = np.zeros_like(self.stocks) \n",
        "        self.cash = None #cash record \n",
        "        self.stocks_df = pd.DataFrame(self.stocks, columns=['stocks'], index = ticker_list)\n",
        "        self.asset_list = []\n",
        "        self.price = np.asarray([0] * len(ticker_list))\n",
        "        self.stockUniverse = ticker_list\n",
        "        self.turbulence_bool = 0\n",
        "        self.equities = []\n",
        "        \n",
        "    def test_latency(self, test_times = 10): \n",
        "        total_time = 0\n",
        "        for i in range(0, test_times):\n",
        "            time0 = time.time()\n",
        "            self.get_state()\n",
        "            time1 = time.time()\n",
        "            temp_time = time1 - time0\n",
        "            total_time += temp_time\n",
        "        latency = total_time/test_times\n",
        "        print('latency for data processing: ', latency)\n",
        "        return latency\n",
        "        \n",
        "    def run(self):\n",
        "        orders = self.alpaca.list_orders(status=\"open\")\n",
        "        for order in orders:\n",
        "          self.alpaca.cancel_order(order.id)\n",
        "    \n",
        "        # Wait for market to open.\n",
        "        print(\"Waiting for market to open...\")\n",
        "        tAMO = threading.Thread(target=self.awaitMarketOpen)\n",
        "        tAMO.start()\n",
        "        tAMO.join()\n",
        "        print(\"Market opened.\")\n",
        "        while True:\n",
        "\n",
        "          # Figure out when the market will close so we can prepare to sell beforehand.\n",
        "          clock = self.alpaca.get_clock()\n",
        "          closingTime = clock.next_close.replace(tzinfo=datetime.timezone.utc).timestamp()\n",
        "          currTime = clock.timestamp.replace(tzinfo=datetime.timezone.utc).timestamp()\n",
        "          self.timeToClose = closingTime - currTime\n",
        "    \n",
        "          if(self.timeToClose < (60)):\n",
        "            # Close all positions when 1 minutes til market close.\n",
        "            print(\"Market closing soon. Stop trading.\")\n",
        "            break\n",
        "            \n",
        "            '''# Close all positions when 1 minutes til market close.\n",
        "            print(\"Market closing soon.  Closing positions.\")\n",
        "    \n",
        "            positions = self.alpaca.list_positions()\n",
        "            for position in positions:\n",
        "              if(position.side == 'long'):\n",
        "                orderSide = 'sell'\n",
        "              else:\n",
        "                orderSide = 'buy'\n",
        "              qty = abs(int(float(position.qty)))\n",
        "              respSO = []\n",
        "              tSubmitOrder = threading.Thread(target=self.submitOrder(qty, position.symbol, orderSide, respSO))\n",
        "              tSubmitOrder.start()\n",
        "              tSubmitOrder.join()\n",
        "    \n",
        "            # Run script again after market close for next trading day.\n",
        "            print(\"Sleeping until market close (15 minutes).\")\n",
        "            time.sleep(60 * 15)'''\n",
        "            \n",
        "          else:\n",
        "            trade = threading.Thread(target=self.trade)\n",
        "            trade.start()\n",
        "            trade.join()\n",
        "            last_equity = float(self.alpaca.get_account().last_equity)\n",
        "            cur_time = time.time()\n",
        "            self.equities.append([cur_time,last_equity])\n",
        "            time.sleep(self.time_interval)\n",
        "            \n",
        "    def awaitMarketOpen(self):\n",
        "        isOpen = self.alpaca.get_clock().is_open\n",
        "        while(not isOpen):\n",
        "          clock = self.alpaca.get_clock()\n",
        "          openingTime = clock.next_open.replace(tzinfo=datetime.timezone.utc).timestamp()\n",
        "          currTime = clock.timestamp.replace(tzinfo=datetime.timezone.utc).timestamp()\n",
        "          timeToOpen = int((openingTime - currTime) / 60)\n",
        "          print(str(timeToOpen) + \" minutes til market open.\")\n",
        "          time.sleep(60)\n",
        "          isOpen = self.alpaca.get_clock().is_open\n",
        "    \n",
        "    def trade(self):\n",
        "        state = self.get_state()\n",
        "        \n",
        "        if self.drl_lib == 'elegantrl':\n",
        "            with torch.no_grad():\n",
        "                s_tensor = torch.as_tensor((state,), device=self.device)\n",
        "                a_tensor = self.act(s_tensor)  \n",
        "                action = a_tensor.detach().cpu().numpy()[0]  \n",
        "                \n",
        "            action = (action * self.max_stock).astype(int)\n",
        "            \n",
        "        elif self.drl_lib == 'rllib':\n",
        "            action = self.agent.compute_single_action(state)\n",
        "        \n",
        "        elif self.drl_lib == 'stable_baselines3':\n",
        "            action = self.model.predict(state)[0]\n",
        "            \n",
        "        else:\n",
        "            raise ValueError('The DRL library input is NOT supported yet. Please check your input.')\n",
        "        \n",
        "        self.stocks_cd += 1\n",
        "        if self.turbulence_bool == 0:\n",
        "            min_action = 10  # stock_cd\n",
        "            for index in np.where(action < -min_action)[0]:  # sell_index:\n",
        "                sell_num_shares = min(self.stocks[index], -action[index])\n",
        "                qty =  abs(int(sell_num_shares))\n",
        "                respSO = []\n",
        "                tSubmitOrder = threading.Thread(target=self.submitOrder(qty, self.stockUniverse[index], 'sell', respSO))\n",
        "                tSubmitOrder.start()\n",
        "                tSubmitOrder.join()\n",
        "                self.cash = float(self.alpaca.get_account().cash)\n",
        "                self.stocks_cd[index] = 0\n",
        "\n",
        "            for index in np.where(action > min_action)[0]:  # buy_index:\n",
        "                if self.cash < 0:\n",
        "                    tmp_cash = 0\n",
        "                else:\n",
        "                    tmp_cash = self.cash\n",
        "                buy_num_shares = min(tmp_cash // self.price[index], abs(int(action[index])))\n",
        "                qty = abs(int(buy_num_shares))\n",
        "                respSO = []\n",
        "                tSubmitOrder = threading.Thread(target=self.submitOrder(qty, self.stockUniverse[index], 'buy', respSO))\n",
        "                tSubmitOrder.start()\n",
        "                tSubmitOrder.join()\n",
        "                self.cash = float(self.alpaca.get_account().cash)\n",
        "                self.stocks_cd[index] = 0\n",
        "                \n",
        "        else:  # sell all when turbulence\n",
        "            positions = self.alpaca.list_positions()\n",
        "            for position in positions:\n",
        "                if(position.side == 'long'):\n",
        "                    orderSide = 'sell'\n",
        "                else:\n",
        "                    orderSide = 'buy'\n",
        "                qty = abs(int(float(position.qty)))\n",
        "                respSO = []\n",
        "                tSubmitOrder = threading.Thread(target=self.submitOrder(qty, position.symbol, orderSide, respSO))\n",
        "                tSubmitOrder.start()\n",
        "                tSubmitOrder.join()\n",
        "            \n",
        "            self.stocks_cd[:] = 0\n",
        "            \n",
        "    \n",
        "    def get_state(self):\n",
        "        alpaca = AlpacaProcessor(api=self.alpaca)\n",
        "        price, tech, turbulence = alpaca.fetch_latest_data(ticker_list = self.stockUniverse, time_interval='1Min',\n",
        "                                                     tech_indicator_list=self.tech_indicator_list)\n",
        "        turbulence_bool = 1 if turbulence >= self.turbulence_thresh else 0\n",
        "        \n",
        "        turbulence = (self.sigmoid_sign(turbulence, self.turbulence_thresh) * 2 ** -5).astype(np.float32)\n",
        "        \n",
        "        tech = tech * 2 ** -7\n",
        "        positions = self.alpaca.list_positions()\n",
        "        stocks = [0] * len(self.stockUniverse)\n",
        "        for position in positions:\n",
        "            ind = self.stockUniverse.index(position.symbol)\n",
        "            stocks[ind] = ( abs(int(float(position.qty))))\n",
        "        \n",
        "        stocks = np.asarray(stocks, dtype = float)\n",
        "        cash = float(self.alpaca.get_account().cash)\n",
        "        self.cash = cash\n",
        "        self.stocks = stocks\n",
        "        self.turbulence_bool = turbulence_bool \n",
        "        self.price = price\n",
        "        \n",
        "        \n",
        "        \n",
        "        amount = np.array(self.cash * (2 ** -12), dtype=np.float32)\n",
        "        scale = np.array(2 ** -6, dtype=np.float32)\n",
        "        state = np.hstack((amount,\n",
        "                    turbulence,\n",
        "                    self.turbulence_bool,\n",
        "                    price * scale,\n",
        "                    self.stocks * scale,\n",
        "                    self.stocks_cd,\n",
        "                    tech,\n",
        "                    )).astype(np.float32)\n",
        "        print(len(self.stockUniverse))\n",
        "        return state\n",
        "        \n",
        "    def submitOrder(self, qty, stock, side, resp):\n",
        "        if(qty > 0):\n",
        "          try:\n",
        "            self.alpaca.submit_order(stock, qty, side, \"market\", \"day\")\n",
        "            print(\"Market order of | \" + str(qty) + \" \" + stock + \" \" + side + \" | completed.\")\n",
        "            resp.append(True)\n",
        "          except:\n",
        "            print(\"Order of | \" + str(qty) + \" \" + stock + \" \" + side + \" | did not go through.\")\n",
        "            resp.append(False)\n",
        "        else:\n",
        "          print(\"Quantity is 0, order of | \" + str(qty) + \" \" + stock + \" \" + side + \" | not completed.\")\n",
        "          resp.append(True)\n",
        "\n",
        "    @staticmethod\n",
        "    def sigmoid_sign(ary, thresh):\n",
        "        def sigmoid(x):\n",
        "            return 1 / (1 + np.exp(-x * np.e)) - 0.5\n",
        "\n",
        "        return sigmoid(ary / thresh) * thresh\n",
        "    \n",
        "class StockEnvEmpty(gym.Env):\n",
        "    #Empty Env used for loading rllib agent\n",
        "    def __init__(self,config):\n",
        "      state_dim = config['state_dim']\n",
        "      action_dim = config['action_dim']\n",
        "      self.env_num = 1\n",
        "      self.max_step = 10000\n",
        "      self.env_name = 'StockEnvEmpty'\n",
        "      self.state_dim = state_dim  \n",
        "      self.action_dim = action_dim\n",
        "      self.if_discrete = False  \n",
        "      self.target_return = 9999\n",
        "      self.observation_space = gym.spaces.Box(low=-3000, high=3000, shape=(state_dim,), dtype=np.float32)\n",
        "      self.action_space = gym.spaces.Box(low=-1, high=1, shape=(action_dim,), dtype=np.float32)\n",
        "        \n",
        "    def reset(self):\n",
        "        return \n",
        "\n",
        "    def step(self, actions):\n",
        "        return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "os4C4-4H7ns7"
      },
      "source": [
        "## Run Paper trading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7nw0i-0UN3-7",
        "outputId": "264e5160-3fc9-41c4-cc67-04806aa88367"
      },
      "source": [
        "print(DOW_30_TICKER)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AAPL', 'MSFT', 'JPM', 'V', 'RTX', 'PG', 'GS', 'NKE', 'DIS', 'AXP', 'HD', 'INTC', 'WMT', 'IBM', 'MRK', 'UNH', 'KO', 'CAT', 'TRV', 'JNJ', 'CVX', 'MCD', 'VZ', 'CSCO', 'XOM', 'BA', 'MMM', 'PFE', 'WBA', 'DD']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YsSBK9ION1t6",
        "outputId": "ecf31b65-9252-4db3-ee94-8a67ce08c847"
      },
      "source": [
        "state_dim"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "333"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xYtSv6P1N247",
        "outputId": "4b38a9f6-8eea-410e-bbbd-e6b1cb884900"
      },
      "source": [
        "action_dim"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kl9nulnAJtiI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 392
        },
        "outputId": "900cee44-4471-43c7-ad75-b59aad2f6708"
      },
      "source": [
        "paper_trading_erl = AlpacaPaperTrading(ticker_list = DOW_30_TICKER, \n",
        "                                       time_interval = '1Min', \n",
        "                                       drl_lib = 'elegantrl', \n",
        "                                       agent = 'ppo', \n",
        "                                       cwd = './papertrading_erl_retrain', \n",
        "                                       net_dim = 512, \n",
        "                                       state_dim = state_dim, \n",
        "                                       action_dim= action_dim, \n",
        "                                       API_KEY = API_KEY, \n",
        "                                       API_SECRET = API_SECRET, \n",
        "                                       APCA_API_BASE_URL = APCA_API_BASE_URL, \n",
        "                                       tech_indicator_list = TECHNICAL_INDICATORS_LIST, \n",
        "                                       turbulence_thresh=30, \n",
        "                                       max_stock=1e2)\n",
        "paper_trading_erl.run()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Waiting for market to open...\n",
            "1219 minutes til market open.\n",
            "1218 minutes til market open.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-67-38afe5bd4892>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m                                        \u001b[0mturbulence_thresh\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m                                        max_stock=1e2)\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mpaper_trading_erl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-66-1f98d3692ca8>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    126\u001b[0m         \u001b[0mtAMO\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mthreading\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mThread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mawaitMarketOpen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m         \u001b[0mtAMO\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m         \u001b[0mtAMO\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Market opened.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1042\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1043\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1044\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1045\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1046\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1058\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1059\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1060\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1061\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1062\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "srzBZfYEUI1O"
      },
      "source": [
        "# Part 4: Check Portfolio Performance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "chovN1UhTAht"
      },
      "source": [
        "import alpaca_trade_api as tradeapi\n",
        "import exchange_calendars as tc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pytz\n",
        "import yfinance as yf\n",
        "import matplotlib.ticker as ticker\n",
        "import matplotlib.dates as mdates\n",
        "from datetime import datetime as dt\n",
        "from finrl.plot import backtest_stats\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CaofxMNCfAR1"
      },
      "source": [
        "def get_trading_days(start, end):\n",
        "    nyse = tc.get_calendar('NYSE')\n",
        "    df = nyse.sessions_in_range(pd.Timestamp(start,tz=pytz.UTC),\n",
        "                                pd.Timestamp(end,tz=pytz.UTC))\n",
        "    trading_days = []\n",
        "    for day in df:\n",
        "        trading_days.append(str(day)[:10])\n",
        "\n",
        "    return trading_days\n",
        "\n",
        "def alpaca_history(key, secret, url, start, end):\n",
        "    api = tradeapi.REST(key, secret, url, 'v2')\n",
        "    trading_days = get_trading_days(start, end)\n",
        "    df = pd.DataFrame()\n",
        "    for day in trading_days:\n",
        "        df = df.append(api.get_portfolio_history(date_start = day,timeframe='5Min').df.iloc[:78])\n",
        "    equities = df.equity.values\n",
        "    cumu_returns = equities/equities[0]\n",
        "    cumu_returns = cumu_returns[~np.isnan(cumu_returns)]\n",
        "    \n",
        "    return df, cumu_returns\n",
        "\n",
        "def DIA_history(start):\n",
        "    data_df = yf.download(['^DJI'],start=start, interval=\"5m\")\n",
        "    data_df = data_df.iloc[48:]\n",
        "    baseline_returns = data_df['Adj Close'].values/data_df['Adj Close'].values[0]\n",
        "    return data_df, baseline_returns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5CHiZRVpURpx"
      },
      "source": [
        "## Get cumulative return"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_YT7v-LSdfV"
      },
      "source": [
        "df_erl, cumu_erl = alpaca_history(key=API_KEY, \n",
        "                                  secret=API_SECRET, \n",
        "                                  url=APCA_API_BASE_URL, \n",
        "                                  start='2021-10-19', #must be within 1 month\n",
        "                                  end='2021-10-22') #change the date if error occurs\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IMcQjwHOS6Zb",
        "outputId": "aff51b64-6166-4bdd-e3f7-1b71fb482520"
      },
      "source": [
        "df_djia, cumu_djia = DIA_history(start='2021-10-19')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s_6OhFfTTGte",
        "outputId": "9ce71a5b-8707-4979-cf4f-213377a74599"
      },
      "source": [
        "print(df_erl)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                           profit_loss  profit_loss_pct      equity\n",
            "timestamp                                                          \n",
            "2021-10-19 09:30:00-04:00     -9324.06        -0.009005  1026133.65\n",
            "2021-10-19 09:35:00-04:00    -10106.67        -0.009761  1025351.04\n",
            "2021-10-19 09:40:00-04:00     -8463.56        -0.008174  1026994.15\n",
            "2021-10-19 09:45:00-04:00     -9907.60        -0.009568  1025550.11\n",
            "2021-10-19 09:50:00-04:00     -8516.10        -0.008224  1026941.61\n",
            "...                                ...              ...         ...\n",
            "2021-10-22 15:35:00-04:00      1560.63         0.001507  1037018.34\n",
            "2021-10-22 15:40:00-04:00       729.00         0.000704  1036186.71\n",
            "2021-10-22 15:45:00-04:00       546.57         0.000528  1036004.28\n",
            "2021-10-22 15:50:00-04:00       459.65         0.000444  1035917.36\n",
            "2021-10-22 15:55:00-04:00       862.31         0.000833  1036320.02\n",
            "\n",
            "[312 rows x 3 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aURLl8M0TKHf",
        "outputId": "adb38549-a994-453f-9a04-f85f48fd4704"
      },
      "source": [
        "print(df_djia)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                   Open          High  ...     Adj Close    Volume\n",
            "Datetime                                               ...                        \n",
            "2021-10-19 13:30:00-04:00  35428.171875  35434.890625  ...  35434.699219   2301420\n",
            "2021-10-19 13:35:00-04:00  35435.179688  35440.160156  ...  35437.269531   1806707\n",
            "2021-10-19 13:40:00-04:00  35437.480469  35437.710938  ...  35430.851562   1797519\n",
            "2021-10-19 13:45:00-04:00  35430.601562  35432.808594  ...  35423.058594   1917764\n",
            "2021-10-19 13:50:00-04:00  35422.929688  35435.828125  ...  35435.691406   2018081\n",
            "...                                 ...           ...  ...           ...       ...\n",
            "2021-10-22 15:35:00-04:00  35708.800781  35710.929688  ...  35684.738281   3547180\n",
            "2021-10-22 15:40:00-04:00  35684.339844  35692.960938  ...  35675.769531   4243899\n",
            "2021-10-22 15:45:00-04:00  35675.710938  35680.800781  ...  35670.238281   5178920\n",
            "2021-10-22 15:50:00-04:00  35670.808594  35699.308594  ...  35692.921875   9231104\n",
            "2021-10-22 15:55:00-04:00  35692.941406  35693.449219  ...  35675.960938  15298892\n",
            "\n",
            "[264 rows x 6 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "PJXPwmx9Ts5o",
        "outputId": "a4fddc2b-529f-469d-dbc6-4261947a1d84"
      },
      "source": [
        "df_erl.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>profit_loss</th>\n",
              "      <th>profit_loss_pct</th>\n",
              "      <th>equity</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>timestamp</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2021-10-22 15:35:00-04:00</th>\n",
              "      <td>1560.63</td>\n",
              "      <td>0.001507</td>\n",
              "      <td>1037018.34</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021-10-22 15:40:00-04:00</th>\n",
              "      <td>729.00</td>\n",
              "      <td>0.000704</td>\n",
              "      <td>1036186.71</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021-10-22 15:45:00-04:00</th>\n",
              "      <td>546.57</td>\n",
              "      <td>0.000528</td>\n",
              "      <td>1036004.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021-10-22 15:50:00-04:00</th>\n",
              "      <td>459.65</td>\n",
              "      <td>0.000444</td>\n",
              "      <td>1035917.36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021-10-22 15:55:00-04:00</th>\n",
              "      <td>862.31</td>\n",
              "      <td>0.000833</td>\n",
              "      <td>1036320.02</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           profit_loss  profit_loss_pct      equity\n",
              "timestamp                                                          \n",
              "2021-10-22 15:35:00-04:00      1560.63         0.001507  1037018.34\n",
              "2021-10-22 15:40:00-04:00       729.00         0.000704  1036186.71\n",
              "2021-10-22 15:45:00-04:00       546.57         0.000528  1036004.28\n",
              "2021-10-22 15:50:00-04:00       459.65         0.000444  1035917.36\n",
              "2021-10-22 15:55:00-04:00       862.31         0.000833  1036320.02"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o1Iaw90FTNfU",
        "outputId": "cb50261c-34d2-49d8-d2d1-6a14a5995b9a"
      },
      "source": [
        "returns_erl = cumu_erl -1 \n",
        "returns_dia = cumu_djia - 1\n",
        "returns_dia = returns_dia[:returns_erl.shape[0]]\n",
        "print('len of erl return: ', returns_erl.shape[0])\n",
        "print('len of dia return: ', returns_dia.shape[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "len of erl return:  312\n",
            "len of dia return:  264\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Z0LEm7KUZ5W"
      },
      "source": [
        "## plot and save"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Foqk1wIQTQJ3"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.figure(dpi=1000)\n",
        "plt.grid()\n",
        "plt.grid(which='minor', axis='y')\n",
        "plt.title('Stock Trading (Paper trading)', fontsize=20)\n",
        "plt.plot(returns_erl, label = 'ElegantRL Agent', color = 'red')\n",
        "#plt.plot(returns_sb3, label = 'Stable-Baselines3 Agent', color = 'blue' )\n",
        "#plt.plot(returns_rllib, label = 'RLlib Agent', color = 'green')\n",
        "plt.plot(returns_dia, label = 'DJIA', color = 'grey')\n",
        "plt.ylabel('Return', fontsize=16)\n",
        "plt.xlabel('Year 2021', fontsize=16)\n",
        "plt.xticks(size = 14)\n",
        "plt.yticks(size = 14)\n",
        "ax = plt.gca()\n",
        "ax.xaxis.set_major_locator(ticker.MultipleLocator(78))\n",
        "ax.xaxis.set_minor_locator(ticker.MultipleLocator(6))\n",
        "ax.yaxis.set_minor_locator(ticker.MultipleLocator(0.005))\n",
        "ax.yaxis.set_major_formatter(ticker.PercentFormatter(xmax=1, decimals=2))\n",
        "ax.xaxis.set_major_formatter(ticker.FixedFormatter(['','10-19','','10-20',\n",
        "                                                    '','10-21','','10-22']))\n",
        "plt.legend(fontsize=10.5)\n",
        "plt.savefig('papertrading_stock.png')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_LsHVj_TZGL"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}