Support Highfreq Backtest with the Model/Rule/RL Strategy

examples/multi_level_trading/README.md

@@ -0,0 +1,24 @@
+# Multi-level Trading
+
+This worflow is an example for multi-level trading.
+
+## Introduction
+
+Qlib supports backtesting of various strategies, including portfolio management strategies, order split strategies, model-based strategies (such as deep learning models), rule-based strategies, and RL-based strategies.
+
+And, Qlib also supports multi-level trading and backtesting. It means that users can use different strategies to trade at different frequencies.
+
+This example uses a DropoutTopkStrategy (a strategy based on the daily frequency Lightgbm model) in weekly frequency for portfolio generation. And, at the daily frequency level, this example uses SBBStrategyEMA (a rule-based strategy that uses EMA for decision-making) to split orders. 
+
+## Usage
+
+Start backtesting by running the following command:
+```bash
+    python workflow.py backtest
+```
+
+Start collecting data by running the following command:
+```bash
+    python workflow.py collect_data
+```
+

examples/multi_level_trading/workflow.py

@@ -0,0 +1,173 @@
+#  Copyright (c) Microsoft Corporation.
+#  Licensed under the MIT License.
+
+
+import qlib
+import fire
+from qlib.config import REG_CN
+
+from qlib.utils import exists_qlib_data, init_instance_by_config, flatten_dict
+from qlib.workflow import R
+from qlib.workflow.record_temp import SignalRecord, PortAnaRecord
+from qlib.tests.data import GetData
+from qlib.contrib.backtest import collect_data
+
+
+class MultiLevelTradingWorkflow:
+
+    market = "csi300"
+    benchmark = "SH000300"
+
+    data_handler_config = {
+        "start_time": "2008-01-01",
+        "end_time": "2020-08-01",
+        "fit_start_time": "2008-01-01",
+        "fit_end_time": "2014-12-31",
+        "instruments": market,
+    }
+
+    task = {
+        "model": {
+            "class": "LGBModel",
+            "module_path": "qlib.contrib.model.gbdt",
+            "kwargs": {
+                "loss": "mse",
+                "colsample_bytree": 0.8879,
+                "learning_rate": 0.0421,
+                "subsample": 0.8789,
+                "lambda_l1": 205.6999,
+                "lambda_l2": 580.9768,
+                "max_depth": 8,
+                "num_leaves": 210,
+                "num_threads": 20,
+            },
+        },
+        "dataset": {
+            "class": "DatasetH",
+            "module_path": "qlib.data.dataset",
+            "kwargs": {
+                "handler": {
+                    "class": "Alpha158",
+                    "module_path": "qlib.contrib.data.handler",
+                    "kwargs": data_handler_config,
+                },
+                "segments": {
+                    "train": ("2008-01-01", "2014-12-31"),
+                    "valid": ("2015-01-01", "2016-12-31"),
+                    "test": ("2017-01-01", "2020-08-01"),
+                },
+            },
+        },
+    }
+
+    trade_start_time = "2017-01-01"
+    trade_end_time = "2020-08-01"
+
+    port_analysis_config = {
+        "executor": {
+            "class": "SplitExecutor",
+            "module_path": "qlib.contrib.backtest.executor",
+            "kwargs": {
+                "step_bar": "week",
+                "sub_executor": {
+                    "class": "SimulatorExecutor",
+                    "module_path": "qlib.contrib.backtest.executor",
+                    "kwargs": {
+                        "step_bar": "day",
+                        "verbose": True,
+                        "generate_report": True,
+                    },
+                },
+                "sub_strategy": {
+                    "class": "SBBStrategyEMA",
+                    "module_path": "qlib.contrib.strategy.rule_strategy",
+                    "kwargs": {
+                        "freq": "day",
+                        "instruments": market,
+                    },
+                },
+                "track_data": True,
+            },
+        },
+        "backtest": {
+            "start_time": trade_start_time,
+            "end_time": trade_end_time,
+            "account": 100000000,
+            "benchmark": benchmark,
+            "exchange_kwargs": {
+                "freq": "day",
+                "limit_threshold": 0.095,
+                "deal_price": "close",
+                "open_cost": 0.0005,
+                "close_cost": 0.0015,
+                "min_cost": 5,
+            },
+        },
+    }
+
+    def _init_qlib(self):
+        """initialize qlib"""
+        # use yahoo_cn_1min data
+        provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir
+        if not exists_qlib_data(provider_uri):
+            print(f"Qlib data is not found in {provider_uri}")
+            GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
+        qlib.init(provider_uri=provider_uri, region=REG_CN)
+
+    def _train_model(self, model, dataset):
+        with R.start(experiment_name="train"):
+            R.log_params(**flatten_dict(self.task))
+            model.fit(dataset)
+            R.save_objects(**{"params.pkl": model})
+
+            # prediction
+            recorder = R.get_recorder()
+            sr = SignalRecord(model, dataset, recorder)
+            sr.generate()
+
+    def backtest(self):
+        self._init_qlib()
+        model = init_instance_by_config(self.task["model"])
+        dataset = init_instance_by_config(self.task["dataset"])
+        self._train_model(model, dataset)
+        strategy_config = {
+            "class": "TopkDropoutStrategy",
+            "module_path": "qlib.contrib.strategy.model_strategy",
+            "kwargs": {
+                "model": model,
+                "dataset": dataset,
+                "topk": 50,
+                "n_drop": 5,
+            },
+        }
+        self.port_analysis_config["strategy"] = strategy_config
+        with R.start(experiment_name="backtest"):
+
+            recorder = R.get_recorder()
+            par = PortAnaRecord(recorder, self.port_analysis_config, "day")
+            par.generate()
+
+    def collect_data(self):
+        self._init_qlib()
+        model = init_instance_by_config(self.task["model"])
+        dataset = init_instance_by_config(self.task["dataset"])
+        self._train_model(model, dataset)
+        executor_config = self.port_analysis_config["executor"]
+        backtest_config = self.port_analysis_config["backtest"]
+        strategy_config = {
+            "class": "TopkDropoutStrategy",
+            "module_path": "qlib.contrib.strategy.model_strategy",
+            "kwargs": {
+                "model": model,
+                "dataset": dataset,
+                "topk": 50,
+                "n_drop": 5,
+            },
+        }
+        data_generator = collect_data(executor=executor_config, strategy=strategy_config, **backtest_config)
+        for trade_decision in data_generator:
+            print(trade_decision)
+
+
+if __name__ == "__main__":
+    fire.Fire(MultiLevelTradingWorkflow)

qlib/config.py

@@ -148,6 +148,10 @@ def set_conf_from_C(self, config_c):
         "task_url": "mongodb://localhost:27017/",
         "task_db_name": "default_task_db",
     },
+    # Shift minute for highfreq minite data, used in backtest
+    # if min_data_shift == 0, use default market time [9:30, 11:29, 1:30, 2:59]
+    # if min_data_shift != 0, use shifted market time [9:30, 11:29, 1:30, 2:59] - shift*minute
+    "min_data_shift": {0},
 }
 
 MODE_CONF = {