service: implement margin service for syncing margin related data

hevan0018 · May 31, 2022 · bf92e28 · bf92e28
1 parent 7601f08
commit bf92e28
Show file tree

Hide file tree

Showing 14 changed files with 392 additions and 36 deletions.
diff --git a/go.mod b/go.mod
@@ -52,6 +52,7 @@ require (
 )
 
 require (
+	github.com/Masterminds/squirrel v1.5.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@@ -75,6 +76,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lestrrat-go/strftime v1.0.0 // indirect
 	github.com/lib/pq v1.10.5 // indirect

diff --git a/go.sum b/go.sum
@@ -39,6 +39,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc=
+github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/adshao/go-binance/v2 v2.3.5 h1:WVYZecm0w8l14YoWlnKZj6xxZT2AKMTHpMQSqIX1xxA=
 github.com/adshao/go-binance/v2 v2.3.5/go.mod h1:8Pg/FGTLyAhq8QXA0IkoReKyRpoxJcK3LVujKDAZV/c=
@@ -303,6 +305,10 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
 github.com/leekchan/accounting v0.0.0-20191218023648-17a4ce5f94d4 h1:KZzDAtJ7ZLm0zSWVhN/zgyB8Ksx5H+P9irwbTcJ9FwI=
 github.com/leekchan/accounting v0.0.0-20191218023648-17a4ce5f94d4/go.mod h1:3timm6YPhY3YDaGxl0q3eaflX0eoSx3FXn7ckHe4tO0=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=

diff --git a/migrations/mysql/20220531013542_margin_interests.sql b/migrations/mysql/20220531013542_margin_interests.sql
@@ -3,8 +3,6 @@ CREATE TABLE `margin_interests`
 (
     `gid`             BIGINT UNSIGNED          NOT NULL AUTO_INCREMENT,
 
-    `transaction_id`  BIGINT UNSIGNED          NOT NULL,
-
     `exchange`        VARCHAR(24)              NOT NULL DEFAULT '',
 
     `asset`           VARCHAR(24)              NOT NULL DEFAULT '',

diff --git a/migrations/mysql/20220531015005_margin_liquidations.sql b/migrations/mysql/20220531015005_margin_liquidations.sql
@@ -23,7 +23,7 @@ CREATE TABLE `margin_liquidations`
 
     `time_in_force`     VARCHAR(5)              NOT NULL DEFAULT '',
 
-    `updated_time`      DATETIME(3)             NOT NULL,
+    `time`              DATETIME(3)             NOT NULL,
 
     PRIMARY KEY (`gid`),
     UNIQUE KEY (`order_id`, `exchange`)

diff --git a/migrations/sqlite3/20220531013541_margin_interests.sql b/migrations/sqlite3/20220531013541_margin_interests.sql
@@ -3,8 +3,6 @@ CREATE TABLE `margin_interests`
 (
     `gid`             INTEGER PRIMARY KEY AUTOINCREMENT,
 
-    `transaction_id`  INTEGER         NOT NULL,
-
     `exchange`        VARCHAR(24)     NOT NULL DEFAULT '',
 
     `asset`           VARCHAR(24)     NOT NULL DEFAULT '',

diff --git a/migrations/sqlite3/20220531015005_margin_liquidations.sql b/migrations/sqlite3/20220531015005_margin_liquidations.sql
@@ -3,13 +3,13 @@ CREATE TABLE `margin_liquidations`
 (
     `gid`               INTEGER PRIMARY KEY AUTOINCREMENT,
 
-    `exchange`          VARCHAR(24)             NOT NULL DEFAULT '',
+    `exchange`          VARCHAR(24)    NOT NULL DEFAULT '',
 
-    `symbol`            VARCHAR(24)             NOT NULL DEFAULT '',
+    `symbol`            VARCHAR(24)    NOT NULL DEFAULT '',
 
-    `order_id`          INTEGER         NOT NULL,
+    `order_id`          INTEGER        NOT NULL,
 
-    `is_isolated`       BOOL                    NOT NULL DEFAULT false,
+    `is_isolated`       BOOL           NOT NULL DEFAULT false,
 
     `average_price`     DECIMAL(16, 8) NOT NULL,
 
@@ -19,11 +19,11 @@ CREATE TABLE `margin_liquidations`
 
     `executed_quantity` DECIMAL(16, 8) NOT NULL,
 
-    `side`              VARCHAR(5)              NOT NULL DEFAULT '',
+    `side`              VARCHAR(5)     NOT NULL DEFAULT '',
 
-    `time_in_force`     VARCHAR(5)              NOT NULL DEFAULT '',
+    `time_in_force`     VARCHAR(5)     NOT NULL DEFAULT '',
 
-    `updated_time`      DATETIME(3)             NOT NULL
+    `time`      DATETIME(3)    NOT NULL
 );
 
 -- +down

diff --git a/pkg/exchange/batch/margin_liquidation.go b/pkg/exchange/batch/margin_liquidation.go
@@ -0,0 +1,36 @@
+package batch
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+type MarginLiquidationBatchQuery struct {
+	types.MarginHistory
+}
+
+func (e *MarginLiquidationBatchQuery) Query(ctx context.Context, startTime, endTime time.Time) (c chan types.MarginLiquidation, errC chan error) {
+	query := &AsyncTimeRangedBatchQuery{
+		Type:    types.MarginLiquidation{},
+		Limiter: rate.NewLimiter(rate.Every(5*time.Second), 2),
+		Q: func(startTime, endTime time.Time) (interface{}, error) {
+			return e.QueryLiquidationHistory(ctx, &startTime, &endTime)
+		},
+		T: func(obj interface{}) time.Time {
+			return time.Time(obj.(types.MarginLiquidation).UpdatedTime)
+		},
+		ID: func(obj interface{}) string {
+			liquidation := obj.(types.MarginLiquidation)
+			return strconv.FormatUint(liquidation.OrderID, 10)
+		},
+	}
+
+	c = make(chan types.MarginLiquidation, 100)
+	errC = query.Query(ctx, c, startTime, endTime)
+	return c, errC
+}
diff --git a/pkg/exchange/batch/margin_loan.go b/pkg/exchange/batch/margin_loan.go
@@ -22,7 +22,7 @@ func (e *MarginLoanBatchQuery) Query(ctx context.Context, asset string, startTim
 			return e.QueryLoanHistory(ctx, asset, &startTime, &endTime)
 		},
 		T: func(obj interface{}) time.Time {
-			return time.Time(obj.(types.MarginRepay).Time)
+			return time.Time(obj.(types.MarginLoan).Time)
 		},
 		ID: func(obj interface{}) string {
 			loan := obj.(types.MarginLoan)

diff --git a/pkg/exchange/binance/binanceapi/client_test.go b/pkg/exchange/binance/binanceapi/client_test.go
@@ -4,32 +4,15 @@ import (
 	"context"
 	"log"
 	"net/http/httputil"
-	"os"
-	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-)
-
-func maskSecret(s string) string {
-	re := regexp.MustCompile(`\b(\w{4})\w+\b`)
-	s = re.ReplaceAllString(s, "$1******")
-	return s
-}
 
-func integrationTestConfigured(t *testing.T, prefix string) (key, secret string, ok bool) {
-	var hasKey, hasSecret bool
-	key, hasKey = os.LookupEnv(prefix + "_API_KEY")
-	secret, hasSecret = os.LookupEnv(prefix + "_API_SECRET")
-	ok = hasKey && hasSecret && os.Getenv("TEST_"+prefix) == "1"
-	if ok {
-		t.Logf(prefix+" api integration test enabled, key = %s, secret = %s", maskSecret(key), maskSecret(secret))
-	}
-	return key, secret, ok
-}
+	"github.com/c9s/bbgo/pkg/testutil"
+)
 
 func getTestClientOrSkip(t *testing.T) *RestClient {
-	key, secret, ok := integrationTestConfigured(t, "BINANCE")
+	key, secret, ok := testutil.IntegrationTestConfigured(t, "BINANCE")
 	if !ok {
 		t.SkipNow()
 		return nil
@@ -119,7 +102,7 @@ func TestClient_NewGetMarginInterestRateHistoryRequest(t *testing.T) {
 }
 
 func TestClient_privateCall(t *testing.T) {
-	key, secret, ok := integrationTestConfigured(t, "BINANCE")
+	key, secret, ok := testutil.IntegrationTestConfigured(t, "BINANCE")
 	if !ok {
 		t.SkipNow()
 	}