Merge branch 'c9s:main' into main

chendagan · Sep 12, 2023 · 069cf76 · 069cf76
2 parents 9be9c08 + 388b9c3
commit 069cf76
Show file tree

Hide file tree

Showing 757 changed files with 55,972 additions and 36,562 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,3 +3,5 @@
 /linode
 /frontend
 /desktop
+/data
+/output
diff --git a/.gitattributes b/.gitattributes
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -25,6 +25,7 @@ jobs:
         # list of Docker images to use as base name for tags
         images: |
           yoanlin/bbgo
+          quay.io/yoanlin/bbgo
         # generate Docker tags based on the following events/attributes
         tags: |
           type=schedule
@@ -43,6 +44,12 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Login to Quay Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: quay.io
+        username: ${{ secrets.QUAY_USERNAME }}
+        password: ${{ secrets.QUAY_TOKEN }}
     - name: Build and push
       id: docker_build
       uses: docker/build-push-action@v2

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -4,18 +4,20 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches:
+    - "main"
+    - "v*"
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
 
     strategy:
       matrix:
         redis-version:
         - 6.2
         go-version:
-        - 1.17
         - 1.18
     env:
       MYSQL_DATABASE: bbgo
@@ -25,6 +27,9 @@ jobs:
     steps:
 
     - uses: actions/checkout@v2
+      with:
+        # lfs: 'true'
+        ssh-key: ${{ secrets.git_ssh_key }}
 
     - uses: actions/cache@v2
       with:
@@ -65,6 +70,12 @@ jobs:
     - name: Build
       run: go build -v ./cmd/bbgo
 
+    - name: Link Market Data
+      run: |
+        mkdir -p ~/.bbgo/cache
+        ln -s $PWD/data/binance-markets.json ~/.bbgo/cache/binance-markets.json
+        touch data/binance-markets.json
+
     - name: Test
       run: |
         go test -race -coverprofile coverage.txt -covermode atomic ./pkg/...

diff --git a/.gitignore b/.gitignore
@@ -64,3 +64,5 @@ coverage_dum.txt
 
 *_local.yaml
 /.chglog/
+
+/.credentials
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -0,0 +1,2 @@
+pkg/strategy/grid2 @kbearXD @gx578007
+python @narumiruna
diff --git a/Dockerfile b/Dockerfile
@@ -1,22 +1,24 @@
 # First stage container
-FROM golang:1.17.11-alpine3.16 AS builder
-RUN apk add --no-cache git ca-certificates gcc libc-dev pkgconfig
+FROM golang:1.18.10-alpine3.17 AS builder
+RUN apk add --no-cache git ca-certificates gcc musl-dev libc-dev pkgconfig
 # gcc is for github.com/mattn/go-sqlite3
 # ADD . $GOPATH/src/github.com/c9s/bbgo
 
+
 WORKDIR $GOPATH/src/github.com/c9s/bbgo
 ARG GO_MOD_CACHE
 ENV WORKDIR=$GOPATH/src/github.com/c9s/bbgo
 ENV GOPATH_ORIG=$GOPATH
 ENV GOPATH=${GO_MOD_CACHE:+$WORKDIR/$GO_MOD_CACHE}
 ENV GOPATH=${GOPATH:-$GOPATH_ORIG}
 ENV CGO_ENABLED=1
-RUN cd $WORKDIR && go get github.com/mattn/go-sqlite3
+RUN cd $WORKDIR
 ADD . .
+RUN go get github.com/mattn/go-sqlite3
 RUN go build -o $GOPATH_ORIG/bin/bbgo ./cmd/bbgo
 
 # Second stage container
-FROM alpine:3.16
+FROM alpine:3.17
 
 # Create the default user 'bbgo' and assign to env 'USER'
 ENV USER=bbgo

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # BBGO
 
-A crypto trading bot framework written in Go. The name bbgo comes from the BB8 bot in the Star Wars movie.
+A modern crypto trading bot framework written in Go.
 
 ## Current Status
 
@@ -90,13 +90,48 @@ the implementation.
 
 ![bbgo backtest report](assets/screenshots/backtest-report.jpg)
 
+## Built-in Strategies
+
+| strategy    | description                                                                                                                             | type       | backtest support |
+|-------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------|------------------|
+| grid        | the first generation grid strategy, it provides more flexibility, but you need to prepare inventories                                   | maker      |                  |
+| grid2       | the second generation grid strategy, it can convert your quote asset into a grid, supports base+quote mode                              | maker      |                  |
+| bollgrid    | strategy implements a basic grid strategy with the built-in bollinger indicator                                                         | maker      |                  | 
+| xmaker      | cross exchange market making strategy, it hedges your inventory risk on the other side                                                  | maker      | no               |
+| xnav        | this strategy helps you record the current net asset value                                                                              | tool       | no               |
+| xalign      | this strategy aligns your balance position automatically                                                                                | tool       | no               |
+| xfunding    | a funding rate fee strategy                                                                                                             | funding    | no               |
+| autoborrow  | this strategy uses margin to borrow assets, to help you keep the minimal balance                                                        | tool       | no               |
+| pivotshort  | this strategy finds the pivot low and entry the trade when the price breaks the previous low                                            | long/short |                  |
+| schedule    | this strategy buy/sell with a fixed quantity periodically, you can use this as a single DCA, or to refill the fee asset like BNB.       | tool       |
+| irr         | this strategy opens the position based on the predicated return rate                                                                    | long/short |                  |
+| bollmaker   | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker      |                  |
+| wall        | this strategy creates wall (large amount order) on the order book                                                                       | maker      | no               |
+| scmaker     | this market making strategy is desgiend for stable coin markets, like USDC/USDT                                                         | maker      |                  |
+| drift       |                                                                                                                                         | long/short |                  |
+| rsicross    | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator       | long/short |                  |
+| marketcap   | this strategy implements a strategy that rebalances the portfolio based on the market capitalization                                    | rebalance  | no               |
+| supertrend  | this strategy uses DEMA and Supertrend indicator to open the long/short position                                                        | long/short |                  |
+| trendtrader | this strategy opens long/short position based on the trendline breakout                                                                 | long/short |                  |
+| elliottwave |                                                                                                                                         | long/short |                  |
+| ewoDgtrd    |                                                                                                                                         | long/short |                  |
+| fixedmaker  |                                                                                                                                         | maker      |                  |
+| factoryzoo  |                                                                                                                                         | long/short |                  |
+| fmaker      |                                                                                                                                         | maker      |                  |
+| linregmaker | a linear regression based market maker                                                                                                  | maker      |                  |
+| convert     | convert strategy is a tool that helps you convert specific asset to a target asset                                                      | tool       | no               |
+
+
+
+
 ## Supported Exchanges
 
 - Binance Spot Exchange (and binance.us)
-- FTX Spot Exchange
 - OKEx Spot Exchange
 - Kucoin Spot Exchange
 - MAX Spot Exchange (located in Taiwan)
+- Bitget Exchange (In Progress)
+- Bybit Exchange (In Progress)
 
 ## Documentation and General Topics
 
@@ -108,7 +143,6 @@ Get your exchange API key and secret after you register the accounts (you can ch
 
 - MAX: <https://max.maicoin.com/signup?r=c7982718>
 - Binance: <https://accounts.binance.com/en/register?ref=38192708>
-- FTX: <https://ftx.com/#a=7710474>
 - OKEx: <https://www.okex.com/join/2412712?src=from:ios-share>
 - Kucoin: <https://www.kucoin.com/ucenter/signup?rcode=r3KX2D4>
 
@@ -178,12 +212,6 @@ BINANCE_US=0
 MAX_API_KEY=
 MAX_API_SECRET=
 
-# for FTX exchange, if you have one
-FTX_API_KEY=
-FTX_API_SECRET=
-# specify it if credentials are for subaccount
-FTX_SUBACCOUNT=
-
 # for OKEx exchange, if you have one
 OKEX_API_KEY=
 OKEX_API_SECRET=
@@ -194,6 +222,10 @@ KUCOIN_API_KEY=
 KUCOIN_API_SECRET=
 KUCOIN_API_PASSPHRASE=
 KUCOIN_API_KEY_VERSION=2
+
+# for Bybit exchange, if you have one
+BYBIT_API_KEY=
+BYBIT_API_SECRET=
 ```
 
 Prepare your dotenv file `.env.local` and BBGO yaml config file `bbgo.yaml`.
@@ -240,6 +272,24 @@ bbgo pnl --exchange binance --asset BTC --since "2019-01-01"
 
 ## Advanced Configuration
 
+### Synchronize System Time With Binance
+
+BBGO provides the script for UNIX systems/subsystems to synchronize date with Binance. `jq` and `bc` are required to be
+installed in previous.
+To install the dependencies in Ubuntu, try the following commands:
+
+```bash
+sudo apt install -y bc jq
+```
+
+And to synchronize the date, try:
+
+```bash
+sudo ./scripts/sync_time.sh
+```
+
+You could also add the script to crontab so that the system time could get synchronized with Binance regularly.
+
 ### Testnet (Paper Trading)
 
 Currently only supports binance testnet. To run bbgo in testnet, apply new API keys
@@ -337,8 +387,6 @@ Check out the strategy directory [strategy](pkg/strategy) for all built-in strat
 
 - `pricealert` strategy demonstrates how to use the notification system [pricealert](pkg/strategy/pricealert). See
   [document](./doc/strategy/pricealert.md).
-- `xpuremaker` strategy demonstrates how to maintain the orderbook and submit maker
-  orders [xpuremaker](pkg/strategy/xpuremaker)
 - `buyandhold` strategy demonstrates how to subscribe kline events and submit market
   order [buyandhold](pkg/strategy/pricedrop)
 - `bollgrid` strategy implements a basic grid strategy with the built-in bollinger
@@ -351,8 +399,12 @@ Check out the strategy directory [strategy](pkg/strategy) for all built-in strat
 - `support` strategy uses K-lines with high volume as support [support](pkg/strategy/support). See
   [document](./doc/strategy/support.md).
 - `flashcrash` strategy implements a strategy that catches the flashcrash [flashcrash](pkg/strategy/flashcrash)
-- `marketcap` strategy implements a strategy that rebalances the portfolio based on the
-  market capitalization [marketcap](pkg/strategy/marketcap). See [document](./doc/strategy/marketcap.md).
+- `marketcap` strategy implements a strategy that rebalances the portfolio based on the market
+  capitalization [marketcap](pkg/strategy/marketcap). See [document](./doc/strategy/marketcap.md).
+- `pivotshort` - shorting focused strategy.
+- `irr` - return rate strategy.
+- `drift` - drift strategy.
+- `grid2` - the second-generation grid strategy.
 
 To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example if
 you want to run
@@ -432,6 +484,7 @@ See also:
 - <https://github.com/narumiruna/bbgo-marketcap>
 - <https://github.com/austin362667/shadow>
 - <https://github.com/jnlin/bbgo-strategy-infinite-grid>
+- <https://github.com/yubing744/trading-gpt>
 
 ## Command Usages
 
@@ -445,7 +498,6 @@ bbgo submit-order --session=okex --symbol=OKBUSDT --side=buy --price=10.0 --quan
 
 ```sh
 bbgo list-orders open --session=okex --symbol=OKBUSDT
-bbgo list-orders open --session=ftx --symbol=FTTUSDT
 bbgo list-orders open --session=max --symbol=MAXUSDT
 bbgo list-orders open --session=binance --symbol=BNBUSDT
 ```

diff --git a/apps/backtest-report/README.md b/apps/backtest-report/README.md
@@ -19,7 +19,7 @@ Create a symlink to your back-test report output directory:
 Generate some back-test reports:
 
 ```
-(cd ../.. && go run ./cmd/bbgo backtest --config bollmaker_ethusdt.yaml  --debug --session binance  --output output --subdir)
+(cd ../.. && go run ./cmd/bbgo backtest --config bollmaker_ethusdt.yaml --debug --output output --subdir)
 ```
 
 Start the development server:

diff --git a/charts/bbgo/Chart.yaml b/charts/bbgo/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.3.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/bbgo/templates/_helpers.tpl b/charts/bbgo/templates/_helpers.tpl
@@ -36,6 +36,9 @@ Common labels
 {{- define "bbgo.labels" -}}
 helm.sh/chart: {{ include "bbgo.chart" . }}
 {{ include "bbgo.selectorLabels" . }}
+{{- if .Values.additionalLabels }}
+{{ toYaml .Values.additionalLabels }}
+{{- end }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}

diff --git a/charts/bbgo/templates/deployment.yaml b/charts/bbgo/templates/deployment.yaml
@@ -2,6 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "bbgo.fullname" . }}
+  {{- with .Values.deploymentAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   labels:
     {{- include "bbgo.labels" . | nindent 4 }}
 spec:
@@ -22,6 +26,7 @@ spec:
     {{- end }}
       labels:
         {{- include "bbgo.selectorLabels" . | nindent 8 }}
+        {{- include "bbgo.labels" . | nindent 8 }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -49,6 +54,10 @@ spec:
           {{- if .Values.webserver.enabled }}
           - "--enable-webserver"
           {{- end }}
+          {{- if .Values.logFormatter.enabled }}
+          - "--log-formatter"
+          - {{ .Values.logFormatter.format | quote }}
+          {{- end }}
           {{- if .Values.grpc.enabled }}
           - "--enable-grpc"
           - "--grpc-bind"

diff --git a/charts/bbgo/values.yaml b/charts/bbgo/values.yaml
@@ -37,6 +37,10 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+additionalLabels: {}
+
+deploymentAnnotations: {}
+
 podAnnotations: {}
 
 podSecurityContext: {}
@@ -74,6 +78,10 @@ metrics:
   enabled: false
   port: 9090
 
+logFormatter:
+  enabled: false
+  format: json
+
 grpc:
   enabled: false
   port: 50051

diff --git a/cmd/bbgo-lorca/main.go b/cmd/bbgo-lorca/main.go
@@ -101,7 +101,7 @@ func main() {
 			return
 		}
 
-		if err := trader.LoadState(); err != nil {
+		if err := trader.LoadState(ctx); err != nil {
 			log.WithError(err).Error("failed to load strategy states")
 			return
 		}

diff --git a/cmd/bbgo-webview/main.go b/cmd/bbgo-webview/main.go
@@ -109,7 +109,7 @@ func main() {
 				return
 			}
 
-			if err := trader.LoadState(); err != nil {
+			if err := trader.LoadState(ctx); err != nil {
 				log.WithError(err).Error("failed to load strategy states")
 				return
 			}

diff --git a/config/autoborrow.yaml b/config/autoborrow.yaml
@@ -7,18 +7,26 @@ exchangeStrategies:
 
     # minMarginRatio for triggering auto borrow
     # we trigger auto borrow only when the margin ratio is above the number
-    minMarginLevel: 1.5
+    minMarginLevel: 3.0
 
     # maxMarginRatio for stop auto-repay
     # if the margin ratio is high enough, we don't have the urge to repay
-    maxMarginLevel: 10.0
+    maxMarginLevel: 20.0
+
+    marginLevelAlertInterval: 5m
+    marginLevelAlertMinMargin: 2.0
+    marginLevelAlertSlackMentions:
+    - '<@USER_ID>'
+    - '<!subteam^TEAM_ID>'
 
     assets:
     - asset: ETH
       low: 3.0
       maxQuantityPerBorrow: 1.0
       maxTotalBorrow: 10.0
+
     - asset: USDT
       low: 1000.0
       maxQuantityPerBorrow: 100.0
       maxTotalBorrow: 10.0
+