From 2fda4477bd5eca33381f49cd7098d5d04a10b99e Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 9 Aug 2023 15:20:33 +0800 Subject: [PATCH 001/422] bitget: implement QueryMarkets --- pkg/exchange/bitget/bitgetapi/client_test.go | 7 ++ .../bitget/bitgetapi/get_symbols_request.go | 4 +- pkg/exchange/bitget/convert.go | 7 ++ pkg/exchange/bitget/exchange.go | 81 +++++++++++++++++++ pkg/types/market.go | 6 +- 5 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 pkg/exchange/bitget/convert.go diff --git a/pkg/exchange/bitget/bitgetapi/client_test.go b/pkg/exchange/bitget/bitgetapi/client_test.go index 90cd69980a..298036b576 100644 --- a/pkg/exchange/bitget/bitgetapi/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/client_test.go @@ -38,6 +38,13 @@ func TestClient(t *testing.T) { t.Logf("tickers: %+v", tickers) }) + t.Run("GetSymbolsRequest", func(t *testing.T) { + req := client.NewGetSymbolsRequest() + symbols, err := req.Do(ctx) + assert.NoError(t, err) + t.Logf("symbols: %+v", symbols) + }) + t.Run("GetTickerRequest", func(t *testing.T) { req := client.NewGetTickerRequest() req.Symbol("BTCUSDT_SPBL") diff --git a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go index 648175d9ca..c3c4e64a96 100644 --- a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go +++ b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go @@ -18,8 +18,8 @@ type Symbol struct { MaxTradeAmount fixedpoint.Value `json:"maxTradeAmount"` TakerFeeRate fixedpoint.Value `json:"takerFeeRate"` MakerFeeRate fixedpoint.Value `json:"makerFeeRate"` - PriceScale fixedpoint.Value `json:"priceScale"` - QuantityScale fixedpoint.Value `json:"quantityScale"` + PriceScale int `json:"priceScale"` + QuantityScale int `json:"quantityScale"` MinTradeUSDT fixedpoint.Value `json:"minTradeUSDT"` Status string `json:"status"` BuyLimitPriceRatio fixedpoint.Value `json:"buyLimitPriceRatio"` diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go new file mode 100644 index 0000000000..30eb8e1b4b --- /dev/null +++ b/pkg/exchange/bitget/convert.go @@ -0,0 +1,7 @@ +package bitget + +import "strings" + +func toGlobalSymbol(s string) string { + return strings.ToUpper(s) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 0bf7d11f50..50b95a04e3 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -1,9 +1,13 @@ package bitget import ( + "context" + "math" + "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -43,3 +47,80 @@ func (e *Exchange) Name() types.ExchangeName { func (e *Exchange) PlatformFeeCurrency() string { return PlatformToken } + +func (e *Exchange) NewStream() types.Stream { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { + // TODO implement me + req := e.client.NewGetSymbolsRequest() + symbols, err := req.Do(ctx) + if err != nil { + return nil, err + } + + markets := types.MarketMap{} + for _, s := range symbols { + symbol := toGlobalSymbol(s.Symbol) + markets[symbol] = types.Market{ + Symbol: symbol, + LocalSymbol: s.Symbol, + PricePrecision: s.PriceScale, + VolumePrecision: s.QuantityScale, + QuoteCurrency: s.QuoteCoin, + BaseCurrency: s.BaseCoin, + MinNotional: s.MinTradeUSDT, + MinAmount: s.MinTradeUSDT, + MinQuantity: s.MinTradeAmount, + MaxQuantity: s.MaxTradeAmount, + StepSize: fixedpoint.NewFromFloat(math.Pow10(-s.QuantityScale)), + TickSize: fixedpoint.NewFromFloat(math.Pow10(-s.PriceScale)), + MinPrice: 0, + MaxPrice: 0, + } + } + + return markets, nil +} + +func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryTickers(ctx context.Context, symbol ...string) (map[string]types.Ticker, error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { + // TODO implement me + panic("implement me") +} + +func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { + // TODO implement me + panic("implement me") +} diff --git a/pkg/types/market.go b/pkg/types/market.go index 07f2cc0da3..8829fed73d 100644 --- a/pkg/types/market.go +++ b/pkg/types/market.go @@ -47,11 +47,11 @@ type Market struct { // 1.0 / math.Pow10(m.BaseUnitPrecision) StepSize fixedpoint.Value `json:"stepSize,omitempty"` - MinPrice fixedpoint.Value `json:"minPrice,omitempty"` - MaxPrice fixedpoint.Value `json:"maxPrice,omitempty"` - // TickSize is the step size of price TickSize fixedpoint.Value `json:"tickSize,omitempty"` + + MinPrice fixedpoint.Value `json:"minPrice,omitempty"` + MaxPrice fixedpoint.Value `json:"maxPrice,omitempty"` } func (m Market) IsDustQuantity(quantity, price fixedpoint.Value) bool { From 40762cad355f3c371b583e6884aef0379cfff614 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 9 Aug 2023 15:25:38 +0800 Subject: [PATCH 002/422] bitget: implement QueryTicker --- pkg/exchange/bitget/exchange.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 50b95a04e3..b9efe48ded 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -63,9 +63,9 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { markets := types.MarketMap{} for _, s := range symbols { - symbol := toGlobalSymbol(s.Symbol) + symbol := toGlobalSymbol(s.SymbolName) markets[symbol] = types.Market{ - Symbol: symbol, + Symbol: s.SymbolName, LocalSymbol: s.Symbol, PricePrecision: s.PriceScale, VolumePrecision: s.QuantityScale, @@ -86,8 +86,23 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { } func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { - // TODO implement me - panic("implement me") + req := e.client.NewGetTickerRequest() + req.Symbol(symbol) + ticker, err := req.Do(ctx) + if err != nil { + return nil, err + } + + return &types.Ticker{ + Time: ticker.Ts.Time(), + Volume: ticker.BaseVol, + Last: ticker.Close, + Open: ticker.OpenUtc0, + High: ticker.High24H, + Low: ticker.Low24H, + Buy: ticker.BuyOne, + Sell: ticker.SellOne, + }, nil } func (e *Exchange) QueryTickers(ctx context.Context, symbol ...string) (map[string]types.Ticker, error) { From 4a64701e1672b65affe2456783497f09a3b1455f Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 9 Aug 2023 15:30:28 +0800 Subject: [PATCH 003/422] bitget: implement QueryAccount --- pkg/exchange/bitget/convert.go | 20 +++++++++++++++++++- pkg/exchange/bitget/exchange.go | 17 +++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 30eb8e1b4b..0836b4dd07 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,7 +1,25 @@ package bitget -import "strings" +import ( + "strings" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) func toGlobalSymbol(s string) string { return strings.ToUpper(s) } + +func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { + return types.Balance{ + Currency: asset.CoinName, + Available: asset.Available, + Locked: asset.Lock.Add(asset.Frozen), + Borrowed: fixedpoint.Zero, + Interest: fixedpoint.Zero, + NetAsset: fixedpoint.Zero, + MaxWithdrawAmount: fixedpoint.Zero, + } +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index b9efe48ded..cae7d3a0ce 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -116,8 +116,21 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { - // TODO implement me - panic("implement me") + req := e.client.NewGetAccountAssetsRequest() + resp, err := req.Do(ctx) + if err != nil { + return nil, err + } + + bals := types.BalanceMap{} + for _, asset := range resp { + b := toGlobalBalance(asset) + bals[asset.CoinName] = b + } + + account := types.NewAccount() + account.UpdateBalances(bals) + return account, nil } func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { From 5f54c303fbbe9d389264323cb1c5e954ace84527 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 15 Aug 2023 16:40:15 +0800 Subject: [PATCH 004/422] bitget: fix fixedpoint.Value init value --- pkg/exchange/bitget/exchange.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index cae7d3a0ce..0d17c1b006 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -77,8 +77,8 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { MaxQuantity: s.MaxTradeAmount, StepSize: fixedpoint.NewFromFloat(math.Pow10(-s.QuantityScale)), TickSize: fixedpoint.NewFromFloat(math.Pow10(-s.PriceScale)), - MinPrice: 0, - MaxPrice: 0, + MinPrice: fixedpoint.Zero, + MaxPrice: fixedpoint.Zero, } } From 874b6471914dafcc9de58bf137216614d7d3bbdd Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 15 Aug 2023 17:54:24 +0800 Subject: [PATCH 005/422] cmd: document tradeStats binding --- pkg/cmd/backtest.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 6dbfb8d1e0..7ed612ee31 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -308,6 +308,8 @@ var BacktestCmd = &cobra.Command{ var reportDir = outputDirectory var sessionTradeStats = make(map[string]map[string]*types.TradeStats) + // for each exchange session, iterate the positions and + // allocate trade collector to calculate the tradeStats var tradeCollectorList []*core.TradeCollector for _, exSource := range exchangeSources { sessionName := exSource.Session.Name @@ -335,6 +337,7 @@ var BacktestCmd = &cobra.Command{ } sessionTradeStats[sessionName] = tradeStatsMap } + kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { if k.Interval == types.Interval1d && k.Closed { for _, collector := range tradeCollectorList { From db376f8483a1504cd36d1bc051c2994b16cda342 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 5 Sep 2023 18:28:10 +0800 Subject: [PATCH 006/422] FEATURE: use quote quantity if there is QuoteQuantity in trade --- pkg/exchange/max/convert.go | 2 +- pkg/exchange/max/maxapi/userdata.go | 1 + pkg/strategy/grid2/strategy.go | 24 ++++++------- pkg/strategy/grid2/strategy_test.go | 56 +++++++++++++++-------------- pkg/strategy/grid2/trade.go | 14 ++++++++ 5 files changed, 56 insertions(+), 41 deletions(-) diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index 0fb6c7462f..ef7b8a117b 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -300,7 +300,7 @@ func convertWebSocketTrade(t max.TradeUpdate) (*types.Trade, error) { Fee: t.Fee, FeeCurrency: toGlobalCurrency(t.FeeCurrency), FeeDiscounted: t.FeeDiscounted, - QuoteQuantity: t.Price.Mul(t.Volume), + QuoteQuantity: t.Funds, Time: types.Time(t.Timestamp.Time()), }, nil } diff --git a/pkg/exchange/max/maxapi/userdata.go b/pkg/exchange/max/maxapi/userdata.go index 551d3d6852..fe7ae8441d 100644 --- a/pkg/exchange/max/maxapi/userdata.go +++ b/pkg/exchange/max/maxapi/userdata.go @@ -98,6 +98,7 @@ type TradeUpdate struct { Side string `json:"sd"` Price fixedpoint.Value `json:"p"` Volume fixedpoint.Value `json:"v"` + Funds fixedpoint.Value `json:"fn"` Market string `json:"M"` Fee fixedpoint.Value `json:"f"` diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index c532ed1adc..82751f4b4d 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -378,9 +378,9 @@ func (s *Strategy) verifyOrderTrades(o types.Order, trades []types.Trade) bool { return true } -// aggregateOrderFee collects the base fee quantity from the given order +// aggregateOrderQuoteAmountAndBaseFee collects the base fee quantity from the given order // it falls back to query the trades via the RESTful API when the websocket trades are not all received. -func (s *Strategy) aggregateOrderFee(o types.Order) (fixedpoint.Value, string) { +func (s *Strategy) aggregateOrderQuoteAmountAndFee(o types.Order) (fixedpoint.Value, fixedpoint.Value, string) { // try to get the received trades (websocket trades) orderTrades := s.historicalTrades.GetOrderTrades(o) if len(orderTrades) > 0 { @@ -396,16 +396,17 @@ func (s *Strategy) aggregateOrderFee(o types.Order) (fixedpoint.Value, string) { // if one of the trades is missing, we need to query the trades from the RESTful API if s.verifyOrderTrades(o, orderTrades) { // if trades are verified + quoteAmount := aggregateTradesQuoteQuantity(orderTrades) fees := collectTradeFee(orderTrades) if fee, ok := fees[feeCurrency]; ok { - return fee, feeCurrency + return quoteAmount, fee, feeCurrency } - return fixedpoint.Zero, feeCurrency + return quoteAmount, fixedpoint.Zero, feeCurrency } // if we don't support orderQueryService, then we should just skip if s.orderQueryService == nil { - return fixedpoint.Zero, feeCurrency + return fixedpoint.Zero, fixedpoint.Zero, feeCurrency } s.logger.Warnf("GRID: missing #%d order trades or missing trade fee, pulling order trades from API", o.OrderID) @@ -423,13 +424,14 @@ func (s *Strategy) aggregateOrderFee(o types.Order) (fixedpoint.Value, string) { } } + quoteAmount := aggregateTradesQuoteQuantity(orderTrades) // still try to aggregate the trades quantity if we can: fees := collectTradeFee(orderTrades) if fee, ok := fees[feeCurrency]; ok { - return fee, feeCurrency + return quoteAmount, fee, feeCurrency } - return fixedpoint.Zero, feeCurrency + return quoteAmount, fixedpoint.Zero, feeCurrency } func (s *Strategy) processFilledOrder(o types.Order) { @@ -446,7 +448,6 @@ func (s *Strategy) processFilledOrder(o types.Order) { } newQuantity := executedQuantity - executedPrice := o.Price if o.ExecutedQuantity.Compare(o.Quantity) != 0 { s.logger.Warnf("order #%d is filled, but order executed quantity %s != order quantity %s, something is wrong", o.OrderID, o.ExecutedQuantity, o.Quantity) @@ -458,16 +459,11 @@ func (s *Strategy) processFilledOrder(o types.Order) { } */ - // will be used for calculating quantity - orderExecutedQuoteAmount := executedQuantity.Mul(executedPrice) - // round down order executed quote amount to avoid insufficient balance - orderExecutedQuoteAmount = orderExecutedQuoteAmount.Round(s.Market.PricePrecision, fixedpoint.Down) - // collect trades for fee // fee calculation is used to reduce the order quantity // because when 1.0 BTC buy order is filled without FEE token, then we will actually get 1.0 * (1 - feeRate) BTC // if we don't reduce the sell quantity, than we might fail to place the sell order - fee, feeCurrency := s.aggregateOrderFee(o) + orderExecutedQuoteAmount, fee, feeCurrency := s.aggregateOrderQuoteAmountAndFee(o) s.logger.Infof("GRID ORDER #%d %s FEE: %s %s", o.OrderID, o.Side, fee.String(), feeCurrency) diff --git a/pkg/strategy/grid2/strategy_test.go b/pkg/strategy/grid2/strategy_test.go index a96a9ab617..9f592959f5 100644 --- a/pkg/strategy/grid2/strategy_test.go +++ b/pkg/strategy/grid2/strategy_test.go @@ -651,7 +651,7 @@ func TestStrategy_calculateProfit(t *testing.T) { }) } -func TestStrategy_aggregateOrderBaseFee(t *testing.T) { +func TestStrategy_aggregateOrderQuoteAmountAndFee(t *testing.T) { s := newTestStrategy() mockCtrl := gomock.NewController(t) @@ -666,32 +666,34 @@ func TestStrategy_aggregateOrderBaseFee(t *testing.T) { OrderID: "3", }).Return([]types.Trade{ { - ID: 1, - OrderID: 3, - Exchange: "binance", - Price: number(20000.0), - Quantity: number(0.2), - Symbol: "BTCUSDT", - Side: types.SideTypeBuy, - IsBuyer: true, - FeeCurrency: "BTC", - Fee: number(0.2 * 0.01), + ID: 1, + OrderID: 3, + Exchange: "binance", + Price: number(20000.0), + Quantity: number(0.2), + QuoteQuantity: number(4000), + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + IsBuyer: true, + FeeCurrency: "BTC", + Fee: number(0.2 * 0.01), }, { - ID: 1, - OrderID: 3, - Exchange: "binance", - Price: number(20000.0), - Quantity: number(0.8), - Symbol: "BTCUSDT", - Side: types.SideTypeBuy, - IsBuyer: true, - FeeCurrency: "BTC", - Fee: number(0.8 * 0.01), + ID: 1, + OrderID: 3, + Exchange: "binance", + Price: number(20000.0), + Quantity: number(0.8), + QuoteQuantity: number(16000), + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + IsBuyer: true, + FeeCurrency: "BTC", + Fee: number(0.8 * 0.01), }, }, nil) - baseFee, _ := s.aggregateOrderFee(types.Order{ + quoteAmount, fee, _ := s.aggregateOrderQuoteAmountAndFee(types.Order{ SubmitOrder: types.SubmitOrder{ Symbol: "BTCUSDT", Side: types.SideTypeBuy, @@ -710,7 +712,8 @@ func TestStrategy_aggregateOrderBaseFee(t *testing.T) { ExecutedQuantity: number(1.0), IsWorking: false, }) - assert.Equal(t, "0.01", baseFee.String()) + assert.Equal(t, "0.01", fee.String()) + assert.Equal(t, "20000", quoteAmount.String()) } func TestStrategy_findDuplicatedPriceOpenOrders(t *testing.T) { @@ -1116,7 +1119,7 @@ func TestStrategy_handleOrderFilled(t *testing.T) { }) } -func TestStrategy_aggregateOrderBaseFeeRetry(t *testing.T) { +func TestStrategy_aggregateOrderQuoteAmountAndFeeRetry(t *testing.T) { s := newTestStrategy() mockCtrl := gomock.NewController(t) @@ -1161,7 +1164,7 @@ func TestStrategy_aggregateOrderBaseFeeRetry(t *testing.T) { }, }, nil) - baseFee, _ := s.aggregateOrderFee(types.Order{ + quoteAmount, fee, _ := s.aggregateOrderQuoteAmountAndFee(types.Order{ SubmitOrder: types.SubmitOrder{ Symbol: "BTCUSDT", Side: types.SideTypeBuy, @@ -1180,7 +1183,8 @@ func TestStrategy_aggregateOrderBaseFeeRetry(t *testing.T) { ExecutedQuantity: number(1.0), IsWorking: false, }) - assert.Equal(t, "0.01", baseFee.String()) + assert.Equal(t, "0.01", fee.String()) + assert.Equal(t, "20000", quoteAmount.String()) } func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { diff --git a/pkg/strategy/grid2/trade.go b/pkg/strategy/grid2/trade.go index c2a4d8eb67..dbb11bf341 100644 --- a/pkg/strategy/grid2/trade.go +++ b/pkg/strategy/grid2/trade.go @@ -29,3 +29,17 @@ func aggregateTradesQuantity(trades []types.Trade) fixedpoint.Value { } return tq } + +// aggregateTradesQuoteQuantity aggregates the quote quantity from the given trade slice +func aggregateTradesQuoteQuantity(trades []types.Trade) fixedpoint.Value { + quoteQuantity := fixedpoint.Zero + for _, t := range trades { + if t.QuoteQuantity.IsZero() { + quoteQuantity = quoteQuantity.Add(t.Price.Mul(t.Quantity)) + } else { + quoteQuantity = quoteQuantity.Add(t.QuoteQuantity) + } + } + + return quoteQuantity +} From faa259623dcf54d2577bcc7dc7aa80788e032fc8 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 13 Sep 2023 17:06:57 +0900 Subject: [PATCH 007/422] pkg/exchange: add auth callback to standard stream --- pkg/types/standardstream_callbacks.go | 12 ++++++++++++ pkg/types/stream.go | 3 +++ 2 files changed, 15 insertions(+) diff --git a/pkg/types/standardstream_callbacks.go b/pkg/types/standardstream_callbacks.go index e0aa0d0fc8..d5796a94af 100644 --- a/pkg/types/standardstream_callbacks.go +++ b/pkg/types/standardstream_callbacks.go @@ -34,6 +34,16 @@ func (s *StandardStream) EmitDisconnect() { } } +func (s *StandardStream) OnAuth(cb func()) { + s.authCallbacks = append(s.authCallbacks, cb) +} + +func (s *StandardStream) EmitAuth() { + for _, cb := range s.authCallbacks { + cb() + } +} + func (s *StandardStream) OnTradeUpdate(cb func(trade Trade)) { s.tradeUpdateCallbacks = append(s.tradeUpdateCallbacks, cb) } @@ -171,6 +181,8 @@ type StandardStreamEventHub interface { OnDisconnect(cb func()) + OnAuth(cb func()) + OnTradeUpdate(cb func(trade Trade)) OnOrderUpdate(cb func(order Order)) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 5b481d0d94..d6044aa268 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -98,6 +98,8 @@ type StandardStream struct { disconnectCallbacks []func() + authCallbacks []func() + // private trade update callbacks tradeUpdateCallbacks []func(trade Trade) @@ -138,6 +140,7 @@ type StandardStreamEmitter interface { EmitStart() EmitConnect() EmitDisconnect() + EmitAuth() EmitTradeUpdate(Trade) EmitOrderUpdate(Order) EmitBalanceSnapshot(BalanceMap) From e56d8d1607313b61208d111d9e418460de327c5f Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 14 Sep 2023 12:01:20 +0900 Subject: [PATCH 008/422] pkg/exchange: emit auth in each exchange --- pkg/exchange/binance/stream.go | 3 +++ pkg/exchange/bybit/stream.go | 3 +++ pkg/exchange/bybit/stream_test.go | 3 +++ pkg/exchange/bybit/types.go | 4 ++++ pkg/exchange/kucoin/stream.go | 3 +++ pkg/exchange/max/stream.go | 1 + pkg/exchange/okex/stream.go | 1 + 7 files changed, 18 insertions(+) diff --git a/pkg/exchange/binance/stream.go b/pkg/exchange/binance/stream.go index cc09d8eb7a..ce8de62d66 100644 --- a/pkg/exchange/binance/stream.go +++ b/pkg/exchange/binance/stream.go @@ -150,6 +150,9 @@ func (s *Stream) handleDisconnect() { func (s *Stream) handleConnect() { if !s.PublicOnly { + // Emit Auth before establishing the connection to prevent the caller from missing the Update data after + // creating the order. + s.EmitAuth() return } diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index db7f257709..ef756a7570 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -137,6 +137,9 @@ func (s *Stream) dispatchEvent(event interface{}) { if err := e.IsValid(); err != nil { log.Errorf("invalid event: %v", err) } + if e.IsAuthenticated() { + s.EmitAuth() + } case *BookEvent: s.EmitBookEvent(*e) diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index daf839872d..bcf3572a6f 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -126,6 +126,9 @@ func TestStream(t *testing.T) { }) t.Run("wallet test", func(t *testing.T) { + s.OnAuth(func() { + t.Log("authenticated") + }) err := s.Connect(context.Background()) assert.NoError(t, err) diff --git a/pkg/exchange/bybit/types.go b/pkg/exchange/bybit/types.go index f5caedf8dc..7ae55bd857 100644 --- a/pkg/exchange/bybit/types.go +++ b/pkg/exchange/bybit/types.go @@ -88,6 +88,10 @@ func (w *WebSocketOpEvent) IsValid() error { } } +func (w *WebSocketOpEvent) IsAuthenticated() bool { + return w.Op == WsOpTypeAuth && w.Success +} + type TopicType string const ( diff --git a/pkg/exchange/kucoin/stream.go b/pkg/exchange/kucoin/stream.go index 7b2bfe8b97..f7a938693e 100644 --- a/pkg/exchange/kucoin/stream.go +++ b/pkg/exchange/kucoin/stream.go @@ -178,6 +178,9 @@ func (s *Stream) handleConnect() { return } } else { + // Emit Auth before establishing the connection to prevent the caller from missing the Update data after + // creating the order. + s.EmitAuth() id := time.Now().UnixNano() / int64(time.Millisecond) cmds := []WebSocketCommand{ { diff --git a/pkg/exchange/max/stream.go b/pkg/exchange/max/stream.go index 27efbb51a9..bd02663b12 100644 --- a/pkg/exchange/max/stream.go +++ b/pkg/exchange/max/stream.go @@ -53,6 +53,7 @@ func NewStream(key, secret string) *Stream { stream.OnConnect(stream.handleConnect) stream.OnAuthEvent(func(e max.AuthEvent) { log.Infof("max websocket connection authenticated: %+v", e) + stream.EmitAuth() }) stream.OnKLineEvent(stream.handleKLineEvent) stream.OnOrderSnapshotEvent(stream.handleOrderSnapshotEvent) diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index 348f486540..7d7c7a77eb 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -118,6 +118,7 @@ func (s *Stream) handleEvent(event WebSocketEvent) { switch event.Event { case "login": if event.Code == "0" { + s.EmitAuth() var subs = []WebsocketSubscription{ {Channel: "account"}, {Channel: "orders", InstrumentType: string(okexapi.InstrumentTypeSpot)}, From a47c846fa54cf3b501efd66335346c98fd21d44c Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Fri, 8 Sep 2023 15:33:09 +0800 Subject: [PATCH 009/422] add QueryOrderTrades() for okex --- pkg/exchange/okex/convert.go | 79 +++-- pkg/exchange/okex/exchange.go | 53 ++- .../get_transaction_histories_request.go | 40 +++ ...ransaction_histories_request_requestgen.go | 305 ++++++++++++++++++ pkg/exchange/okex/okexapi/trade.go | 3 +- pkg/exchange/okex/query_order_trades_test.go | 40 +++ 6 files changed, 489 insertions(+), 31 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_transaction_histories_request.go create mode 100644 pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go create mode 100644 pkg/exchange/okex/query_order_trades_test.go diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 881f66ab9a..193ba233e1 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -128,36 +128,14 @@ func segmentOrderDetails(orderDetails []okexapi.OrderDetails) (trades, orders [] func toGlobalTrades(orderDetails []okexapi.OrderDetails) ([]types.Trade, error) { var trades []types.Trade + var err error for _, orderDetail := range orderDetails { - tradeID, err := strconv.ParseInt(orderDetail.LastTradeID, 10, 64) - if err != nil { - return trades, errors.Wrapf(err, "error parsing tradeId value: %s", orderDetail.LastTradeID) - } - - orderID, err := strconv.ParseInt(orderDetail.OrderID, 10, 64) - if err != nil { - return trades, errors.Wrapf(err, "error parsing ordId value: %s", orderDetail.OrderID) + trade, err2 := toGlobalTrade(&orderDetail) + if err2 != nil { + err = multierr.Append(err, err2) + continue } - - side := types.SideType(strings.ToUpper(string(orderDetail.Side))) - - trades = append(trades, types.Trade{ - ID: uint64(tradeID), - OrderID: uint64(orderID), - Exchange: types.ExchangeOKEx, - Price: orderDetail.LastFilledPrice, - Quantity: orderDetail.LastFilledQuantity, - QuoteQuantity: orderDetail.LastFilledPrice.Mul(orderDetail.LastFilledQuantity), - Symbol: toGlobalSymbol(orderDetail.InstrumentID), - Side: side, - IsBuyer: side == types.SideTypeBuy, - IsMaker: orderDetail.ExecutionType == "M", - Time: types.Time(orderDetail.LastFilledTime), - Fee: orderDetail.LastFilledFee, - FeeCurrency: orderDetail.LastFilledFeeCurrency, - IsMargin: false, - IsIsolated: false, - }) + trades = append(trades, *trade) } return trades, nil @@ -280,7 +258,7 @@ func toGlobalOrder(okexOrder *okexapi.OrderDetails) (*types.Order, error) { } isMargin := false - if okexOrder.InstrumentType == string(okexapi.InstrumentTypeMARGIN) { + if okexOrder.InstrumentType == okexapi.InstrumentTypeMARGIN { isMargin = true } @@ -306,3 +284,46 @@ func toGlobalOrder(okexOrder *okexapi.OrderDetails) (*types.Order, error) { IsIsolated: false, }, nil } + +func toGlobalTrade(orderDetail *okexapi.OrderDetails) (*types.Trade, error) { + tradeID, err := strconv.ParseInt(orderDetail.LastTradeID, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error parsing tradeId value: %s", orderDetail.LastTradeID) + } + + orderID, err := strconv.ParseInt(orderDetail.OrderID, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error parsing ordId value: %s", orderDetail.OrderID) + } + + side := toGlobalSide(orderDetail.Side) + + isMargin := false + if orderDetail.InstrumentType == okexapi.InstrumentTypeMARGIN { + isMargin = true + } + + isFuture := false + if orderDetail.InstrumentType == okexapi.InstrumentTypeFutures { + isFuture = true + } + + return &types.Trade{ + ID: uint64(tradeID), + OrderID: uint64(orderID), + Exchange: types.ExchangeOKEx, + Price: orderDetail.LastFilledPrice, + Quantity: orderDetail.LastFilledQuantity, + QuoteQuantity: orderDetail.LastFilledPrice.Mul(orderDetail.LastFilledQuantity), + Symbol: toGlobalSymbol(orderDetail.InstrumentID), + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: orderDetail.ExecutionType == "M", + Time: types.Time(orderDetail.LastFilledTime), + Fee: orderDetail.LastFilledFee, + FeeCurrency: orderDetail.LastFilledFeeCurrency, + IsMargin: isMargin, + IsFutures: isFuture, + IsIsolated: false, + }, nil +} diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index d09fd76bdc..b89d7740a6 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -2,12 +2,14 @@ package okex import ( "context" + "fmt" "math" "strconv" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.uber.org/multierr" "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/okex/okexapi" @@ -15,7 +17,15 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -var marketDataLimiter = rate.NewLimiter(rate.Every(time.Second/10), 1) +// Okex rate limit list in each api document +// The default order limiter apply 30 requests per second and a 5 initial bucket +// this includes QueryOrder, QueryOrderTrades, SubmitOrder, QueryOpenOrders, CancelOrders +// Market data limiter means public api, this includes QueryMarkets, QueryTicker, QueryTickers, QueryKLines +var ( + marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) + tradeRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) + orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) +) const ID = "okex" @@ -363,3 +373,44 @@ func (e *Exchange) QueryOrder(ctx context.Context, q types.OrderQuery) (*types.O return toGlobalOrder(order) } + +// Query order trades can query trades in last 3 months. +func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([]types.Trade, error) { + if len(q.ClientOrderID) != 0 { + log.Warn("!!!OKEX EXCHANGE API NOTICE!!! Okex does not support searching for trades using OrderClientId.") + } + + req := e.client.NewGetTransactionHistoriesRequest() + if len(q.Symbol) != 0 { + req.InstrumentID(q.Symbol) + } + + if len(q.OrderID) != 0 { + req.OrderID(q.OrderID) + } + + if err := orderRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("order rate limiter wait error: %w", err) + } + response, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query order trades, err: %w", err) + } + + var trades []types.Trade + var errs error + for _, trade := range response { + res, err := toGlobalTrade(&trade) + if err != nil { + errs = multierr.Append(errs, err) + continue + } + trades = append(trades, *res) + } + + if errs != nil { + return nil, errs + } + + return trades, nil +} diff --git a/pkg/exchange/okex/okexapi/get_transaction_histories_request.go b/pkg/exchange/okex/okexapi/get_transaction_histories_request.go new file mode 100644 index 0000000000..7e16d3b547 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_transaction_histories_request.go @@ -0,0 +1,40 @@ +package okexapi + +import ( + "time" + + "github.com/c9s/requestgen" +) + +//go:generate GetRequest -url "/api/v5/trade/fills-history" -type GetTransactionHistoriesRequest -responseDataType .APIResponse +type GetTransactionHistoriesRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentType InstrumentType `param:"instType,query"` + instrumentID *string `param:"instId,query"` + orderType *OrderType `param:"ordType,query"` + orderID string `param:"ordId,query"` + // Underlying and InstrumentFamily Applicable to FUTURES/SWAP/OPTION + underlying *string `param:"uly,query"` + instrumentFamily *string `param:"instFamily,query"` + + after *string `param:"after,query"` + before *string `param:"before,query"` + startTime *time.Time `param:"begin,query,milliseconds"` + + // endTime for each request, startTime and endTime can be any interval, but should be in last 3 months + endTime *time.Time `param:"end,query,milliseconds"` + + // limit for data size per page. Default: 100 + limit *uint64 `param:"limit,query"` +} + +type OrderList []OrderDetails + +// NewGetOrderHistoriesRequest is descending order by createdTime +func (c *RestClient) NewGetTransactionHistoriesRequest() *GetTransactionHistoriesRequest { + return &GetTransactionHistoriesRequest{ + client: c, + instrumentType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go b/pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go new file mode 100644 index 0000000000..f88a68513a --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go @@ -0,0 +1,305 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills-history -type GetTransactionHistoriesRequest -responseDataType .OrderList"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetTransactionHistoriesRequest) InstrumentType(instrumentType InstrumentType) *GetTransactionHistoriesRequest { + g.instrumentType = instrumentType + return g +} + +func (g *GetTransactionHistoriesRequest) InstrumentID(instrumentID string) *GetTransactionHistoriesRequest { + g.instrumentID = &instrumentID + return g +} + +func (g *GetTransactionHistoriesRequest) OrderType(orderType OrderType) *GetTransactionHistoriesRequest { + g.orderType = &orderType + return g +} + +func (g *GetTransactionHistoriesRequest) OrderID(orderID string) *GetTransactionHistoriesRequest { + g.orderID = orderID + return g +} + +func (g *GetTransactionHistoriesRequest) Underlying(underlying string) *GetTransactionHistoriesRequest { + g.underlying = &underlying + return g +} + +func (g *GetTransactionHistoriesRequest) InstrumentFamily(instrumentFamily string) *GetTransactionHistoriesRequest { + g.instrumentFamily = &instrumentFamily + return g +} + +func (g *GetTransactionHistoriesRequest) After(after string) *GetTransactionHistoriesRequest { + g.after = &after + return g +} + +func (g *GetTransactionHistoriesRequest) Before(before string) *GetTransactionHistoriesRequest { + g.before = &before + return g +} + +func (g *GetTransactionHistoriesRequest) StartTime(startTime time.Time) *GetTransactionHistoriesRequest { + g.startTime = &startTime + return g +} + +func (g *GetTransactionHistoriesRequest) EndTime(endTime time.Time) *GetTransactionHistoriesRequest { + g.endTime = &endTime + return g +} + +func (g *GetTransactionHistoriesRequest) Limit(limit uint64) *GetTransactionHistoriesRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTransactionHistoriesRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instrumentType field -> json key instType + instrumentType := g.instrumentType + + // TEMPLATE check-valid-values + switch instrumentType { + case InstrumentTypeSpot, InstrumentTypeSwap, InstrumentTypeFutures, InstrumentTypeOption, InstrumentTypeMARGIN: + params["instType"] = instrumentType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instrumentType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instrumentType + params["instType"] = instrumentType + // check instrumentID field -> json key instId + if g.instrumentID != nil { + instrumentID := *g.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + } else { + } + // check orderType field -> json key ordType + if g.orderType != nil { + orderType := *g.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeMarket, OrderTypeLimit, OrderTypePostOnly, OrderTypeFOK, OrderTypeIOC: + params["ordType"] = orderType + + default: + return nil, fmt.Errorf("ordType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["ordType"] = orderType + } else { + } + // check orderID field -> json key ordId + orderID := g.orderID + + // assign parameter of orderID + params["ordId"] = orderID + // check underlying field -> json key uly + if g.underlying != nil { + underlying := *g.underlying + + // assign parameter of underlying + params["uly"] = underlying + } else { + } + // check instrumentFamily field -> json key instFamily + if g.instrumentFamily != nil { + instrumentFamily := *g.instrumentFamily + + // assign parameter of instrumentFamily + params["instFamily"] = instrumentFamily + } else { + } + // check after field -> json key after + if g.after != nil { + after := *g.after + + // assign parameter of after + params["after"] = after + } else { + } + // check before field -> json key before + if g.before != nil { + before := *g.before + + // assign parameter of before + params["before"] = before + } else { + } + // check startTime field -> json key begin + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["begin"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key end + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["end"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTransactionHistoriesRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTransactionHistoriesRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTransactionHistoriesRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTransactionHistoriesRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTransactionHistoriesRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTransactionHistoriesRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTransactionHistoriesRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTransactionHistoriesRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetTransactionHistoriesRequest) Do(ctx context.Context) (OrderList, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v5/trade/fills-history" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data OrderList + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index 4c0ecf26ab..164ae46dae 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -251,7 +251,7 @@ func (r *BatchPlaceOrderRequest) Do(ctx context.Context) ([]OrderResponse, error } type OrderDetails struct { - InstrumentType string `json:"instType"` + InstrumentType InstrumentType `json:"instType"` InstrumentID string `json:"instId"` Tag string `json:"tag"` Price fixedpoint.Value `json:"px"` @@ -275,6 +275,7 @@ type OrderDetails struct { LastFilledTime types.MillisecondTimestamp `json:"fillTime"` LastFilledFee fixedpoint.Value `json:"fillFee"` LastFilledFeeCurrency string `json:"fillFeeCcy"` + LastFilledPnl fixedpoint.Value `json:"fillPnl"` // ExecutionType = liquidity (M = maker or T = taker) ExecutionType string `json:"execType"` diff --git a/pkg/exchange/okex/query_order_trades_test.go b/pkg/exchange/okex/query_order_trades_test.go new file mode 100644 index 0000000000..15aad9643e --- /dev/null +++ b/pkg/exchange/okex/query_order_trades_test.go @@ -0,0 +1,40 @@ +package okex + +import ( + "context" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/testutil" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func Test_QueryOrderTrades(t *testing.T) { + + key, secret, passphrase, ok := testutil.IntegrationTestWithPassphraseConfigured(t, "OKEX") + if !ok { + t.Skip("Please configure all credentials about OKEX") + } + + e := New(key, secret, passphrase) + + queryOrder := types.OrderQuery{ + OrderID: "609869603774656544", + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + transactionDetail, err := e.QueryOrderTrades(ctx, queryOrder) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) + queryOrder = types.OrderQuery{ + Symbol: "BTC-USDT", + } + transactionDetail, err = e.QueryOrderTrades(ctx, queryOrder) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) +} From 5f8a5e47d51309000027b86088d255627882d342 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 17:42:17 +0800 Subject: [PATCH 010/422] activeorderbook: add pending order logs --- pkg/bbgo/activeorderbook.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index ba0c0a6d9f..d66ff6af73 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -59,7 +59,9 @@ func (b *ActiveOrderBook) BindStream(stream types.Stream) { stream.OnOrderUpdate(b.orderUpdateHandler) } -func (b *ActiveOrderBook) waitClear(ctx context.Context, order types.Order, waitTime, timeout time.Duration) (bool, error) { +func (b *ActiveOrderBook) waitClear( + ctx context.Context, order types.Order, waitTime, timeout time.Duration, +) (bool, error) { if !b.orders.Exists(order.OrderID) { return true, nil } @@ -266,6 +268,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { b.mu.Lock() if !b.orders.Exists(order.OrderID) { + log.Infof("[ActiveOrderBook] order #%d does not exist, adding it to pending order update", order.OrderID) b.pendingOrderUpdates.Add(order) b.mu.Unlock() return @@ -275,6 +278,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { if previousOrder, ok := b.orders.Get(order.OrderID); ok { previousUpdateTime := previousOrder.UpdateTime.Time() if !previousUpdateTime.IsZero() && order.UpdateTime.Before(previousUpdateTime) { + log.Infof("[ActiveOrderBook] order #%d updateTime is out of date, skip it", order.OrderID) b.mu.Unlock() return } From 8314a7e7502d7436d9ee257f9d4b52b28965bc20 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 18:03:23 +0800 Subject: [PATCH 011/422] types: improve order string format --- pkg/types/order.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/types/order.go b/pkg/types/order.go index 5e29fdd36a..a24a82a528 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -320,9 +320,8 @@ func (o Order) String() string { orderID = strconv.FormatUint(o.OrderID, 10) } - desc := fmt.Sprintf("ORDER %s | %s | %s | %s | %s %-4s | %s/%s @ %s", + desc := fmt.Sprintf("ORDER %s | %s | %s | %s %-4s | %s/%s @ %s", o.Exchange.String(), - o.CreationTime.Time().Local().Format(time.StampMilli), orderID, o.Symbol, o.Type, @@ -335,7 +334,15 @@ func (o Order) String() string { desc += " Stop @ " + o.StopPrice.String() } - return desc + " | " + string(o.Status) + desc += " | " + string(o.Status) + " | " + + if time.Time(o.UpdateTime).IsZero() { + desc += "0/" + time.Time(o.CreationTime).UTC().Format(time.StampMilli) + } else { + desc += time.Time(o.UpdateTime).UTC().Format(time.StampMilli) + "/" + time.Time(o.CreationTime).UTC().Format(time.StampMilli) + } + + return desc } // PlainText is used for telegram-styled messages From 4b78bfcdfa3ca79ced617322a692d9934a9945ca Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 18:08:57 +0800 Subject: [PATCH 012/422] types: improve order string format --- pkg/types/order.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/types/order.go b/pkg/types/order.go index a24a82a528..ae4dc8425e 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -336,10 +336,12 @@ func (o Order) String() string { desc += " | " + string(o.Status) + " | " + desc += time.Time(o.CreationTime).UTC().Format(time.StampMilli) + if time.Time(o.UpdateTime).IsZero() { - desc += "0/" + time.Time(o.CreationTime).UTC().Format(time.StampMilli) + desc += " -> 0" } else { - desc += time.Time(o.UpdateTime).UTC().Format(time.StampMilli) + "/" + time.Time(o.CreationTime).UTC().Format(time.StampMilli) + desc += " -> " + time.Time(o.UpdateTime).UTC().Format(time.StampMilli) } return desc From 797ee4402c9a66a70ced6b0966bf8ee2eb362f07 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 18:20:29 +0800 Subject: [PATCH 013/422] types: fix pending order update comparison --- pkg/bbgo/activeorderbook.go | 39 +++++++++++++++++++++++++++++++- pkg/bbgo/activeorderbook_test.go | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index d66ff6af73..9c6cca1f4f 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -334,13 +334,50 @@ func (b *ActiveOrderBook) Add(orders ...types.Order) { } } +func isNewerUpdate(a, b types.Order) bool { + // compare state first + switch a.Status { + + case types.OrderStatusCanceled, types.OrderStatusRejected: // canceled is a final state + switch b.Status { + case types.OrderStatusNew, types.OrderStatusPartiallyFilled: + return true + } + + case types.OrderStatusPartiallyFilled: + switch b.Status { + case types.OrderStatusNew: + return true + } + + case types.OrderStatusFilled: + switch b.Status { + case types.OrderStatusFilled, types.OrderStatusPartiallyFilled, types.OrderStatusNew: + return true + } + } + + au := time.Time(a.UpdateTime) + bu := time.Time(b.UpdateTime) + + if !au.IsZero() && !bu.IsZero() && au.After(bu) { + return true + } + + if !au.IsZero() && bu.IsZero() { + return true + } + + return false +} + // add the order to the active order book and check the pending order func (b *ActiveOrderBook) add(order types.Order) { if pendingOrder, ok := b.pendingOrderUpdates.Get(order.OrderID); ok { // if the pending order update time is newer than the adding order // we should use the pending order rather than the adding order. // if pending order is older, than we should add the new one, and drop the pending order - if pendingOrder.UpdateTime.Time().After(order.UpdateTime.Time()) { + if isNewerUpdate(pendingOrder, order) { order = pendingOrder } diff --git a/pkg/bbgo/activeorderbook_test.go b/pkg/bbgo/activeorderbook_test.go index 848e07bc9f..65cd6aa57a 100644 --- a/pkg/bbgo/activeorderbook_test.go +++ b/pkg/bbgo/activeorderbook_test.go @@ -59,7 +59,7 @@ func TestActiveOrderBook_pendingOrders(t *testing.T) { }, Status: types.OrderStatusNew, CreationTime: types.Time(now), - UpdateTime: types.Time(now.Add(-time.Second)), + UpdateTime: types.Time(now), }) assert.True(t, filled, "filled event should be fired") From 89c88c48a3b9334c0bd04943aaa0acdd4f737e09 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 18:25:21 +0800 Subject: [PATCH 014/422] bbgo: log filled order --- pkg/bbgo/activeorderbook.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 9c6cca1f4f..86b90e64fa 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -291,6 +291,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { b.mu.Unlock() if removed { + log.Infof("[ActiveOrderBook] order #%d is filled: %s", order.OrderID, order.String()) b.EmitFilled(order) } b.C.Emit() From 542944b4cca4a07a63c91202ca94b561918537a9 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 17 Sep 2023 18:29:14 +0800 Subject: [PATCH 015/422] max: use websocket update time (TU) field --- pkg/bbgo/activeorderbook.go | 4 ++-- pkg/exchange/max/convert.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 86b90e64fa..994b7329d7 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -268,7 +268,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { b.mu.Lock() if !b.orders.Exists(order.OrderID) { - log.Infof("[ActiveOrderBook] order #%d does not exist, adding it to pending order update", order.OrderID) + log.Infof("[ActiveOrderBook] order #%d %s does not exist, adding it to pending order update", order.OrderID, order.Status) b.pendingOrderUpdates.Add(order) b.mu.Unlock() return @@ -278,7 +278,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { if previousOrder, ok := b.orders.Get(order.OrderID); ok { previousUpdateTime := previousOrder.UpdateTime.Time() if !previousUpdateTime.IsZero() && order.UpdateTime.Before(previousUpdateTime) { - log.Infof("[ActiveOrderBook] order #%d updateTime is out of date, skip it", order.OrderID) + log.Infof("[ActiveOrderBook] order #%d updateTime %s is out of date, skip it", order.OrderID, order.UpdateTime) b.mu.Unlock() return } diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index 0fb6c7462f..54550ce7e8 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -67,7 +67,9 @@ func toGlobalRewards(maxRewards []max.Reward) ([]types.Reward, error) { return rewards, nil } -func toGlobalOrderStatus(orderState max.OrderState, executedVolume, remainingVolume fixedpoint.Value) types.OrderStatus { +func toGlobalOrderStatus( + orderState max.OrderState, executedVolume, remainingVolume fixedpoint.Value, +) types.OrderStatus { switch orderState { case max.OrderStateCancel: @@ -328,6 +330,6 @@ func convertWebSocketOrderUpdate(u max.OrderUpdate) (*types.Order, error) { Status: toGlobalOrderStatus(u.State, u.ExecutedVolume, u.RemainingVolume), ExecutedQuantity: u.ExecutedVolume, CreationTime: types.Time(time.Unix(0, u.CreatedAtMs*int64(time.Millisecond))), - UpdateTime: types.Time(time.Unix(0, u.CreatedAtMs*int64(time.Millisecond))), + UpdateTime: types.Time(time.Unix(0, u.UpdateTime*int64(time.Millisecond))), }, nil } From 42ee9618b593f8188afc0a051d9ebb4efb04441f Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 18 Sep 2023 13:11:22 +0800 Subject: [PATCH 016/422] pkg/exchange: emit regardless of whether there is an error or not. --- pkg/types/standardstream_callbacks.go | 12 ++++++++++++ pkg/types/stream.go | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/pkg/types/standardstream_callbacks.go b/pkg/types/standardstream_callbacks.go index d5796a94af..55a3e7c6b6 100644 --- a/pkg/types/standardstream_callbacks.go +++ b/pkg/types/standardstream_callbacks.go @@ -44,6 +44,16 @@ func (s *StandardStream) EmitAuth() { } } +func (s *StandardStream) OnRawMessage(cb func(raw []byte)) { + s.rawMessageCallbacks = append(s.rawMessageCallbacks, cb) +} + +func (s *StandardStream) EmitRawMessage(raw []byte) { + for _, cb := range s.rawMessageCallbacks { + cb(raw) + } +} + func (s *StandardStream) OnTradeUpdate(cb func(trade Trade)) { s.tradeUpdateCallbacks = append(s.tradeUpdateCallbacks, cb) } @@ -183,6 +193,8 @@ type StandardStreamEventHub interface { OnAuth(cb func()) + OnRawMessage(cb func(raw []byte)) + OnTradeUpdate(cb func(trade Trade)) OnOrderUpdate(cb func(order Order)) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index d6044aa268..730188ef7e 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -100,6 +100,8 @@ type StandardStream struct { authCallbacks []func() + rawMessageCallbacks []func(raw []byte) + // private trade update callbacks tradeUpdateCallbacks []func(trade Trade) @@ -260,6 +262,8 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel continue } + s.EmitRawMessage(message) + if debugRawMessage { log.Info(string(message)) } From fdfa3639ffe2ee6a6b6cf4e15371bde332fdc706 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 19 Sep 2023 11:12:14 +0800 Subject: [PATCH 017/422] FEATURE: use retry query order until successful --- pkg/exchange/retry/order.go | 10 ++++++++++ pkg/strategy/grid2/recover.go | 6 ++++-- pkg/strategy/grid2/strategy.go | 11 +++-------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index 8c50de6803..15b59c6afa 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -57,6 +57,16 @@ func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symb return openOrders, err } +func QueryOrderUntilSuccessful(ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery) (order *types.Order, err error) { + var op = func() (err2 error) { + order, err2 = query.QueryOrder(ctx, opts) + return err2 + } + + err = GeneralBackoff(ctx, op) + return order, err +} + func CancelAllOrdersUntilSuccessful(ctx context.Context, service advancedOrderCancelService) error { var op = func() (err2 error) { _, err2 = service.CancelAllOrders(ctx) diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index d48eab1bba..945f2c0380 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -290,12 +291,13 @@ func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTr // already queries, skip continue } - order, err := queryOrderService.QueryOrder(ctx, types.OrderQuery{ + order, err := retry.QueryOrderUntilSuccessful(ctx, queryOrderService, types.OrderQuery{ + Symbol: trade.Symbol, OrderID: strconv.FormatUint(trade.OrderID, 10), }) if err != nil { - return errors.Wrapf(err, "failed to query order by trade") + return errors.Wrapf(err, "failed to query order by trade (trade id: %d, order id: %d)", trade.ID, trade.OrderID) } s.debugLog(order.String()) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 82751f4b4d..09e3895a2d 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -2159,14 +2159,9 @@ func (s *Strategy) recoverActiveOrders(ctx context.Context, session *bbgo.Exchan for _, o := range activeOrders { s.logger.Infof("updating %d order...", o.OrderID) - var updatedOrder *types.Order - err := retry.GeneralBackoff(ctx, func() error { - var err error - updatedOrder, err = s.orderQueryService.QueryOrder(ctx, types.OrderQuery{ - Symbol: o.Symbol, - OrderID: strconv.FormatUint(o.OrderID, 10), - }) - return err + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ + Symbol: o.Symbol, + OrderID: strconv.FormatUint(o.OrderID, 10), }) if err != nil { From c8316a36a0bdbdd7e4d6ccffb2e8bd7247338b18 Mon Sep 17 00:00:00 2001 From: narumi Date: Tue, 19 Sep 2023 14:51:04 +0800 Subject: [PATCH 018/422] use common strategy in fixedmaker --- pkg/strategy/fixedmaker/strategy.go | 56 +++++++++-------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 0910e536d6..a54d055bb0 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -9,7 +9,8 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/indicator" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -23,6 +24,8 @@ func init() { // Fixed spread market making strategy type Strategy struct { + *common.Strategy + Environment *bbgo.Environment StandardIndicatorSet *bbgo.StandardIndicatorSet Market types.Market @@ -42,22 +45,18 @@ type Strategy struct { ATRMultiplier fixedpoint.Value `json:"atrMultiplier"` ATRWindow int `json:"atrWindow"` - // persistence fields - Position *types.Position `json:"position,omitempty" persistence:"position"` - ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` - - session *bbgo.ExchangeSession - orderExecutor *bbgo.GeneralOrderExecutor activeOrderBook *bbgo.ActiveOrderBook - atr *indicator.ATR + atr *indicatorv2.ATRStream } func (s *Strategy) Defaults() error { if s.OrderType == "" { + log.Infof("order type is not set, using limit maker order type") s.OrderType = types.OrderTypeLimitMaker } if s.ATRWindow == 0 { + log.Infof("atr window is not set, using default value 14") s.ATRWindow = 14 } return nil @@ -102,36 +101,13 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.session = session + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.activeOrderBook.BindStream(session.UserDataStream) - instanceID := s.InstanceID() - - if s.Position == nil { - s.Position = types.NewPositionFromMarket(s.Market) - } - - // Always update the position fields - s.Position.Strategy = ID - s.Position.StrategyInstanceID = instanceID - - if s.ProfitStats == nil { - s.ProfitStats = types.NewProfitStats(s.Market) - } - - s.orderExecutor = bbgo.NewGeneralOrderExecutor(session, s.Symbol, ID, instanceID, s.Position) - s.orderExecutor.BindEnvironment(s.Environment) - - s.orderExecutor.BindProfitStats(s.ProfitStats) - - s.orderExecutor.Bind() - s.orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - bbgo.Sync(ctx, s) - }) - - s.atr = s.StandardIndicatorSet.ATR(types.IntervalWindow{Interval: s.Interval, Window: s.ATRWindow}) + s.atr = session.Indicators(s.Symbol).ATR(s.Interval, s.ATRWindow) session.UserDataStream.OnStart(func() { // you can place orders here when bbgo is started, this will be called only once. @@ -155,14 +131,14 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. // the shutdown handler, you can cancel all orders bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() - _ = s.orderExecutor.GracefulCancel(ctx) + _ = s.OrderExecutor.GracefulCancel(ctx) }) return nil } func (s *Strategy) cancelOrders(ctx context.Context) { - if err := s.session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { log.WithError(err).Errorf("failed to cancel orders") } } @@ -180,7 +156,7 @@ func (s *Strategy) replenish(ctx context.Context) { return } - createdOrders, err := s.orderExecutor.SubmitOrders(ctx, submitOrders...) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, submitOrders...) if err != nil { log.WithError(err).Error("failed to submit orders") return @@ -193,19 +169,19 @@ func (s *Strategy) replenish(ctx context.Context) { func (s *Strategy) generateSubmitOrders(ctx context.Context) ([]types.SubmitOrder, error) { orders := []types.SubmitOrder{} - baseBalance, ok := s.session.GetAccount().Balance(s.Market.BaseCurrency) + baseBalance, ok := s.Session.GetAccount().Balance(s.Market.BaseCurrency) if !ok { return nil, fmt.Errorf("base currency %s balance not found", s.Market.BaseCurrency) } log.Infof("base balance: %+v", baseBalance) - quoteBalance, ok := s.session.GetAccount().Balance(s.Market.QuoteCurrency) + quoteBalance, ok := s.Session.GetAccount().Balance(s.Market.QuoteCurrency) if !ok { return nil, fmt.Errorf("quote currency %s balance not found", s.Market.QuoteCurrency) } log.Infof("quote balance: %+v", quoteBalance) - ticker, err := s.session.Exchange.QueryTicker(ctx, s.Symbol) + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) if err != nil { return nil, err } From 294e5111dcc83a2b434732a67db589b6bb19b0c4 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 19 Sep 2023 14:52:07 +0800 Subject: [PATCH 019/422] pkg/types: ensure all routines are done --- pkg/types/stream.go | 20 +++++++++++--- pkg/types/syncgroup.go | 53 +++++++++++++++++++++++++++++++++++++ pkg/types/syncgroup_test.go | 29 ++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 pkg/types/syncgroup.go create mode 100644 pkg/types/syncgroup_test.go diff --git a/pkg/types/stream.go b/pkg/types/stream.go index d6044aa268..a2996c7416 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -80,6 +80,11 @@ type StandardStream struct { PublicOnly bool + // sg is used to wait until the previous routines are closed. + // only handle routines used internally, avoid including external callback func to prevent issues if they have + // bugs and cannot terminate. e.q. heartBeat + sg SyncGroup + // ReconnectC is a signal channel for reconnecting ReconnectC chan struct{} @@ -160,6 +165,7 @@ func NewStandardStream() StandardStream { return StandardStream{ ReconnectC: make(chan struct{}, 1), CloseC: make(chan struct{}), + sg: NewSyncGroup(), } } @@ -188,9 +194,10 @@ func (s *StandardStream) SetConn(ctx context.Context, conn *websocket.Conn) (con connCtx, connCancel := context.WithCancel(ctx) s.ConnLock.Lock() - // ensure the previous context is cancelled + // ensure the previous context is cancelled and all routines are closed. if s.ConnCancel != nil { s.ConnCancel() + s.sg.WaitAndClear() } // create a new context for this connection @@ -405,9 +412,16 @@ func (s *StandardStream) DialAndConnect(ctx context.Context) error { connCtx, connCancel := s.SetConn(ctx, conn) s.EmitConnect() - go s.Read(connCtx, conn, connCancel) - go s.ping(connCtx, conn, connCancel, pingInterval) + s.sg.Add(func() { + s.Read(connCtx, conn, connCancel) + }) + s.sg.Add(func() { + s.ping(connCtx, conn, connCancel, pingInterval) + }) + s.sg.Run() + if s.heartBeat != nil { + // not included in wg, as it is an external callback func. go s.heartBeat(connCtx, conn, connCancel) } return nil diff --git a/pkg/types/syncgroup.go b/pkg/types/syncgroup.go new file mode 100644 index 0000000000..f63829ee0c --- /dev/null +++ b/pkg/types/syncgroup.go @@ -0,0 +1,53 @@ +package types + +import ( + "sync" +) + +type syncGroupFunc func() + +// SyncGroup is essentially a wrapper around sync.WaitGroup, designed for ease of use. You only need to use Add() to +// add routines and Run() to execute them. When it's time to close or reset, you just need to call WaitAndClear(), +// which takes care of waiting for all the routines to complete before clearing routine. +// +// It eliminates the need for manual management of sync.WaitGroup. Specifically, it highlights that SyncGroup takes +// care of sync.WaitGroup.Add() and sync.WaitGroup.Done() automatically, reducing the chances of missing these crucial calls. +type SyncGroup struct { + wg sync.WaitGroup + + sgFuncsMu sync.Mutex + sgFuncs []syncGroupFunc +} + +func NewSyncGroup() SyncGroup { + return SyncGroup{} +} + +func (w *SyncGroup) WaitAndClear() { + w.wg.Wait() + + w.sgFuncsMu.Lock() + w.sgFuncs = []syncGroupFunc{} + w.sgFuncsMu.Unlock() +} + +func (w *SyncGroup) Add(fn syncGroupFunc) { + w.wg.Add(1) + + w.sgFuncsMu.Lock() + w.sgFuncs = append(w.sgFuncs, fn) + w.sgFuncsMu.Unlock() +} + +func (w *SyncGroup) Run() { + w.sgFuncsMu.Lock() + fns := w.sgFuncs + w.sgFuncsMu.Unlock() + + for _, fn := range fns { + go func(doFunc syncGroupFunc) { + defer w.wg.Done() + doFunc() + }(fn) + } +} diff --git a/pkg/types/syncgroup_test.go b/pkg/types/syncgroup_test.go new file mode 100644 index 0000000000..a2d69bf9af --- /dev/null +++ b/pkg/types/syncgroup_test.go @@ -0,0 +1,29 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_waitGroup_Run(t *testing.T) { + closeCh1 := make(chan struct{}) + closeCh2 := make(chan struct{}) + + wg := NewSyncGroup() + wg.Add(func() { + <-closeCh1 + }) + + wg.Add(func() { + <-closeCh2 + }) + + wg.Run() + + close(closeCh1) + close(closeCh2) + wg.WaitAndClear() + + assert.Len(t, wg.sgFuncs, 0) +} From 4a231b10c6105a2e650907759e5ba390a7f5d907 Mon Sep 17 00:00:00 2001 From: narumi Date: Thu, 21 Sep 2023 14:57:55 +0800 Subject: [PATCH 020/422] pull out ishalted method --- pkg/risk/riskcontrol/circuit_break.go | 7 +++---- pkg/strategy/common/strategy.go | 4 ++++ pkg/strategy/fixedmaker/strategy.go | 13 +++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/risk/riskcontrol/circuit_break.go b/pkg/risk/riskcontrol/circuit_break.go index 921696e21f..e8e8ad3b09 100644 --- a/pkg/risk/riskcontrol/circuit_break.go +++ b/pkg/risk/riskcontrol/circuit_break.go @@ -19,7 +19,6 @@ type CircuitBreakRiskControl struct { lossThreshold fixedpoint.Value haltedDuration time.Duration - isHalted bool haltedAt time.Time } @@ -59,10 +58,10 @@ func (c *CircuitBreakRiskControl) IsHalted(t time.Time) bool { c.profitStats.TodayPnL.Float64(), unrealized.Float64()) - c.isHalted = unrealized.Add(c.profitStats.TodayPnL).Compare(c.lossThreshold) <= 0 - if c.isHalted { + isHalted := unrealized.Add(c.profitStats.TodayPnL).Compare(c.lossThreshold) <= 0 + if isHalted { c.haltedAt = t } - return c.isHalted + return isHalted } diff --git a/pkg/strategy/common/strategy.go b/pkg/strategy/common/strategy.go index 4f159c7f62..4ad94e532a 100644 --- a/pkg/strategy/common/strategy.go +++ b/pkg/strategy/common/strategy.go @@ -88,3 +88,7 @@ func (s *Strategy) Initialize(ctx context.Context, environ *bbgo.Environment, se 24*time.Hour) } } + +func (s *Strategy) IsHalted(t time.Time) bool { + return s.circuitBreakRiskControl.IsHalted(t) +} diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index a54d055bb0..fb4a7f2262 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/sirupsen/logrus" @@ -111,13 +112,12 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. session.UserDataStream.OnStart(func() { // you can place orders here when bbgo is started, this will be called only once. - s.replenish(ctx) }) s.activeOrderBook.OnFilled(func(order types.Order) { if s.activeOrderBook.NumOfOrders() == 0 { log.Infof("no active orders, replenish") - s.replenish(ctx) + s.replenish(ctx, order.UpdateTime.Time()) } }) @@ -125,7 +125,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. log.Infof("%+v", kline) s.cancelOrders(ctx) - s.replenish(ctx) + s.replenish(ctx, kline.EndTime.Time()) }) // the shutdown handler, you can cancel all orders @@ -143,7 +143,12 @@ func (s *Strategy) cancelOrders(ctx context.Context) { } } -func (s *Strategy) replenish(ctx context.Context) { +func (s *Strategy) replenish(ctx context.Context, t time.Time) { + if s.IsHalted(t) { + log.Infof("circuit break halted, not replenishing") + return + } + submitOrders, err := s.generateSubmitOrders(ctx) if err != nil { log.WithError(err).Error("failed to generate submit orders") From db7a0df2548f42fb66b0103f0aec4600cebc8069 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 25 Sep 2023 13:55:59 +0800 Subject: [PATCH 021/422] types: change websocket error to warnf --- pkg/types/stream.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 1b91c84264..73bd448567 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -236,12 +236,12 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel mt, message, err := conn.ReadMessage() if err != nil { // if it's a network timeout error, we should re-connect - switch err := err.(type) { + switch err2 := err.(type) { // if it's a websocket related error case *websocket.CloseError: - if err.Code != websocket.CloseNormalClosure { - log.WithError(err).Errorf("websocket error abnormal close: %+v", err) + if err2.Code != websocket.CloseNormalClosure { + log.WithError(err2).Warnf("websocket error abnormal close: %+v", err2) } _ = conn.Close() @@ -251,13 +251,13 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel return case net.Error: - log.WithError(err).Warn("websocket read network error") + log.WithError(err2).Warn("websocket read network error") _ = conn.Close() s.Reconnect() return default: - log.WithError(err).Warn("unexpected websocket error") + log.WithError(err2).Warn("unexpected websocket error") _ = conn.Close() s.Reconnect() return @@ -291,7 +291,9 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel } } -func (s *StandardStream) ping(ctx context.Context, conn *websocket.Conn, cancel context.CancelFunc, interval time.Duration) { +func (s *StandardStream) ping( + ctx context.Context, conn *websocket.Conn, cancel context.CancelFunc, interval time.Duration, +) { defer func() { cancel() log.Debug("[websocket] ping worker stopped") From 550b0104991356aa1ad0dda6af5660234af690fa Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 25 Sep 2023 17:16:27 +0800 Subject: [PATCH 022/422] bbgo: add log fields support to the core --- pkg/bbgo/config.go | 5 +++-- pkg/bbgo/environment.go | 12 +++++++++++- pkg/bbgo/session.go | 33 ++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index c8d6e3120b..d5314283a6 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -92,8 +92,9 @@ type NotificationConfig struct { } type LoggingConfig struct { - Trade bool `json:"trade,omitempty"` - Order bool `json:"order,omitempty"` + Trade bool `json:"trade,omitempty"` + Order bool `json:"order,omitempty"` + Fields map[string]interface{} `json:"fields,omitempty"` } type Session struct { diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index b4e1ac7cda..71ef121eb4 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -126,6 +126,14 @@ func NewEnvironment() *Environment { } } +func (environ *Environment) Logger() log.FieldLogger { + if environ.loggingConfig != nil && len(environ.loggingConfig.Fields) > 0 { + return log.WithFields(environ.loggingConfig.Fields) + } + + return log.StandardLogger() +} + func (environ *Environment) Session(name string) (*ExchangeSession, bool) { s, ok := environ.sessions[name] return s, ok @@ -857,7 +865,9 @@ func (environ *Environment) setupSlack(userConfig *Config, slackToken string, pe interact.AddMessenger(messenger) } -func (environ *Environment) setupTelegram(userConfig *Config, telegramBotToken string, persistence service.PersistenceService) error { +func (environ *Environment) setupTelegram( + userConfig *Config, telegramBotToken string, persistence service.PersistenceService, +) error { tt := strings.Split(telegramBotToken, ":") telegramID := tt[0] diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 6a361a9d51..0465346661 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -116,7 +116,7 @@ type ExchangeSession struct { usedSymbols map[string]struct{} initializedSymbols map[string]struct{} - logger *log.Entry + logger log.FieldLogger } func NewExchangeSession(name string, exchange types.Exchange) *ExchangeSession { @@ -182,10 +182,14 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) return ErrSessionAlreadyInitialized } - var log = log.WithField("session", session.Name) + var logger = environ.Logger() + logger = logger.WithField("session", session.Name) + + // override the default logger + session.logger = logger // load markets first - log.Infof("querying market info from %s...", session.Name) + logger.Infof("querying market info from %s...", session.Name) var disableMarketsCache = false var markets types.MarketMap @@ -233,7 +237,7 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) // query and initialize the balances if !session.PublicOnly { - log.Infof("querying account balances...") + logger.Infof("querying account balances...") account, err := session.Exchange.QueryAccount(ctx) if err != nil { @@ -244,8 +248,7 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) session.Account = account session.accountMutex.Unlock() - log.Infof("account %s balances:", session.Name) - account.Balances().Print() + logger.Infof("account %s balances:\n%s", session.Name, account.Balances().String()) // forward trade updates and order updates to the order executor session.UserDataStream.OnTradeUpdate(session.OrderExecutor.EmitTradeUpdate) @@ -275,29 +278,29 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) if environ.loggingConfig != nil { if environ.loggingConfig.Trade { session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { - log.Info(trade.String()) + logger.Info(trade.String()) }) } if environ.loggingConfig.Order { session.UserDataStream.OnOrderUpdate(func(order types.Order) { - log.Info(order.String()) + logger.Info(order.String()) }) } } else { // if logging config is nil, then apply default logging setup // add trade logger session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { - log.Info(trade.String()) + logger.Info(trade.String()) }) } if viper.GetBool("debug-kline") { session.MarketDataStream.OnKLine(func(kline types.KLine) { - log.WithField("marketData", "kline").Infof("kline: %+v", kline) + logger.WithField("marketData", "kline").Infof("kline: %+v", kline) }) session.MarketDataStream.OnKLineClosed(func(kline types.KLine) { - log.WithField("marketData", "kline").Infof("kline closed: %+v", kline) + logger.WithField("marketData", "kline").Infof("kline closed: %+v", kline) }) } @@ -558,7 +561,9 @@ func (session *ExchangeSession) MarketDataStore(symbol string) (s *MarketDataSto } // KLine updates will be received in the order listend in intervals array -func (session *ExchangeSession) SerialMarketDataStore(ctx context.Context, symbol string, intervals []types.Interval, useAggTrade ...bool) (store *SerialMarketDataStore, ok bool) { +func (session *ExchangeSession) SerialMarketDataStore( + ctx context.Context, symbol string, intervals []types.Interval, useAggTrade ...bool, +) (store *SerialMarketDataStore, ok bool) { st, ok := session.MarketDataStore(symbol) if !ok { return nil, false @@ -628,7 +633,9 @@ func (session *ExchangeSession) OrderStores() map[string]*core.OrderStore { } // Subscribe save the subscription info, later it will be assigned to the stream -func (session *ExchangeSession) Subscribe(channel types.Channel, symbol string, options types.SubscribeOptions) *ExchangeSession { +func (session *ExchangeSession) Subscribe( + channel types.Channel, symbol string, options types.SubscribeOptions, +) *ExchangeSession { if channel == types.KLineChannel && len(options.Interval) == 0 { panic("subscription interval for kline can not be empty") } From b6d0e3ef273e3caa318fd3a924ee582f83f8f02c Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Sep 2023 17:57:12 +0800 Subject: [PATCH 023/422] grid2: only do active order update when grid is recovered --- pkg/strategy/grid2/strategy.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 09e3895a2d..d7e4210e5d 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -205,6 +206,8 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc + recovered int32 + // this ensures that bbgo.Sync to lock the object sync.Mutex } @@ -1982,11 +1985,16 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) } - session.UserDataStream.OnConnect(func() { + session.UserDataStream.OnAuth(func() { if !bbgo.IsBackTesting { // callback may block the stream execution, so we spawn the recover function to the background // add (5 seconds + random <10 seconds jitter) delay go time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { + recovered := atomic.LoadInt32(&s.recovered) + if recovered == 0 { + return + } + s.recoverActiveOrders(ctx, session) }) } @@ -2016,6 +2024,10 @@ func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSessi } func (s *Strategy) recoverGrid(ctx context.Context, session *bbgo.ExchangeSession) error { + defer func() { + atomic.AddInt32(&s.recovered, 1) + }() + if s.RecoverGridByScanningTrades { s.debugLog("recovering grid by scanning trades") return s.recoverByScanningTrades(ctx, session) From 94f6cefd709fdfc9f459deb157e5cc78b7c89efa Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 25 Sep 2023 17:43:00 +0800 Subject: [PATCH 024/422] grid2: improve active order recover logs --- pkg/strategy/grid2/strategy.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index d7e4210e5d..90af91cba0 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -2168,8 +2168,8 @@ func (s *Strategy) recoverActiveOrders(ctx context.Context, session *bbgo.Exchan } s.logger.Infof("found %d active orders to update...", len(activeOrders)) - for _, o := range activeOrders { - s.logger.Infof("updating %d order...", o.OrderID) + for i, o := range activeOrders { + s.logger.Infof("updating %d/%d order #%d...", i+1, len(activeOrders), o.OrderID) updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ Symbol: o.Symbol, @@ -2181,6 +2181,7 @@ func (s *Strategy) recoverActiveOrders(ctx context.Context, session *bbgo.Exchan return } + s.logger.Infof("triggering updated order #%d: %s", o.OrderID, o.String()) activeOrderBook.Update(*updatedOrder) } } From 99a69f4f2f5b0f7f092644d8e271db40e3f86be3 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Mon, 18 Sep 2023 15:55:58 +0800 Subject: [PATCH 025/422] add QueryClosedOrders() and QueryTrades() for okex, also fix conflict for QueryOrderTrades() and update typo error in QueryOrderTrades() --- pkg/exchange/okex/exchange.go | 138 +++++++- .../okex/okexapi/get_order_history_request.go | 40 +++ .../get_order_history_request_requestgen.go | 319 ++++++++++++++++++ ....go => get_transaction_history_request.go} | 10 +- ...transaction_history_request_requestgen.go} | 44 +-- pkg/exchange/okex/query_closed_orders_test.go | 62 ++++ pkg/exchange/okex/query_order_test.go | 2 +- pkg/exchange/okex/query_trades_test.go | 57 ++++ pkg/testutil/auth.go | 2 + 9 files changed, 641 insertions(+), 33 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_order_history_request.go create mode 100644 pkg/exchange/okex/okexapi/get_order_history_request_requestgen.go rename pkg/exchange/okex/okexapi/{get_transaction_histories_request.go => get_transaction_history_request.go} (79%) rename pkg/exchange/okex/okexapi/{get_transaction_histories_request_requestgen.go => get_transaction_history_request_requestgen.go} (73%) create mode 100644 pkg/exchange/okex/query_closed_orders_test.go create mode 100644 pkg/exchange/okex/query_trades_test.go diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index b89d7740a6..d578554fe0 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -23,7 +23,7 @@ import ( // Market data limiter means public api, this includes QueryMarkets, QueryTicker, QueryTickers, QueryKLines var ( marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) - tradeRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) + tradeRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) ) @@ -32,6 +32,11 @@ const ID = "okex" // PlatformToken is the platform currency of OKEx, pre-allocate static string here const PlatformToken = "OKB" +const ( + // Constant For query limit + defaultQueryLimit = 100 +) + var log = logrus.WithFields(logrus.Fields{ "exchange": ID, }) @@ -360,7 +365,7 @@ func (e *Exchange) QueryOrder(ctx context.Context, q types.OrderQuery) (*types.O return nil, errors.New("okex.QueryOrder: OrderId or ClientOrderId is required parameter") } req := e.client.NewGetOrderDetailsRequest() - req.InstrumentID(q.Symbol). + req.InstrumentID(toLocalSymbol(q.Symbol)). OrderID(q.OrderID). ClientOrderID(q.ClientOrderID) @@ -380,9 +385,9 @@ func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([] log.Warn("!!!OKEX EXCHANGE API NOTICE!!! Okex does not support searching for trades using OrderClientId.") } - req := e.client.NewGetTransactionHistoriesRequest() + req := e.client.NewGetTransactionHistoryRequest() if len(q.Symbol) != 0 { - req.InstrumentID(q.Symbol) + req.InstrumentID(toLocalSymbol(q.Symbol)) } if len(q.OrderID) != 0 { @@ -411,6 +416,131 @@ func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([] if errs != nil { return nil, errs } + return trades, nil +} + +/* +QueryClosedOrders can query closed orders in last 3 months, there are no time interval limitations, as long as until >= since. +Please Use lastOrderID as cursor, only return orders later than that order, that order is not included. +If you want to query orders by time range, please just pass since and until. +If you want to query by cursor, please pass lastOrderID. +Because it gets the correct response even when you pass all parameters with the right time interval and invalid lastOrderID, like 0. +Time interval boundary unit is second. +since is inclusive, ex. order created in 1694155903, get response if query since 1694155903, get empty if query since 1694155904 +until is not inclusive, ex. order created in 1694155903, get response if query until 1694155904, get empty if query until 1694155903 +*/ +func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) { + if symbol == "" { + return nil, ErrSymbolRequired + } + + if err := tradeRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) + } + + var lastOrder string + if lastOrderID <= 0 { + lastOrder = "" + } else { + lastOrder = strconv.FormatUint(lastOrderID, 10) + } + + res, err := e.client.NewGetOrderHistoryRequest(). + InstrumentID(toLocalSymbol(symbol)). + StartTime(since). + EndTime(until). + Limit(defaultQueryLimit). + Before(lastOrder). + Do(ctx) + + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + + var orders []types.Order + + for _, order := range res { + o, err2 := toGlobalOrder(&order) + if err2 != nil { + err = multierr.Append(err, err2) + continue + } + + orders = append(orders, *o) + } + if err != nil { + return nil, err + } + return types.SortOrdersAscending(orders), nil +} + +/* +QueryTrades can query trades in last 3 months, there are no time interval limitations, as long as end_time >= start_time. +OKEX do not provide api to query by tradeID, So use /api/v5/trade/orders-history-archive as its official site do. +Please Use LastTradeID as cursor, only return trades later than that trade, that trade is not included. +If you want to query trades by time range, please just pass start_time and end_time. +If you want to query by cursor, please pass LastTradeID. +Because it gets the correct response even when you pass all parameters with the right time interval and invalid LastTradeID, like 0. +*/ +func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { + if symbol == "" { + return nil, ErrSymbolRequired + } + + req := e.client.NewGetOrderHistoryRequest().InstrumentID(toLocalSymbol(symbol)) + + limit := uint64(options.Limit) + if limit > defaultQueryLimit || limit <= 0 { + log.Debugf("limit is exceeded default limit %d or zero, got: %d, Do not pass limit", defaultQueryLimit, options.Limit) + } else { + req.Limit(limit) + } + + if err := tradeRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query trades rate limiter wait error: %w", err) + } + + var err error + var response []okexapi.OrderDetails + // query by time interval + if options.StartTime != nil || options.EndTime != nil { + if options.StartTime != nil { + req.StartTime(*options.StartTime) + } + if options.EndTime != nil { + req.EndTime(*options.EndTime) + } + + response, err = req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + } else if options.StartTime == nil && options.EndTime == nil && options.LastTradeID == 0 { // query by no any parameters + response, err = req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + } else { // query by trade id + lastTradeID := strconv.FormatUint(options.LastTradeID, 10) + res, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + for _, trade := range res { + if trade.LastTradeID == lastTradeID { + response, err = req.Before(trade.OrderID).Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + break + } + } + } + + trades, err := toGlobalTrades(response) + if err != nil { + return nil, fmt.Errorf("failed to trans order detail to trades error: %w", err) + } return trades, nil } diff --git a/pkg/exchange/okex/okexapi/get_order_history_request.go b/pkg/exchange/okex/okexapi/get_order_history_request.go new file mode 100644 index 0000000000..00b4eca7a2 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_order_history_request.go @@ -0,0 +1,40 @@ +package okexapi + +import ( + "time" + + "github.com/c9s/requestgen" +) + +//go:generate GetRequest -url "/api/v5/trade/orders-history-archive" -type GetOrderHistoryRequest -responseDataType .APIResponse +type GetOrderHistoryRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentType InstrumentType `param:"instType,query"` + instrumentID *string `param:"instId,query"` + orderType *OrderType `param:"ordType,query"` + // underlying and instrumentFamil Applicable to FUTURES/SWAP/OPTION + underlying *string `param:"uly,query"` + instrumentFamily *string `param:"instFamily,query"` + + state *OrderState `param:"state,query"` + after *string `param:"after,query"` + before *string `param:"before,query"` + startTime *time.Time `param:"begin,query,milliseconds"` + + // endTime for each request, startTime and endTime can be any interval, but should be in last 3 months + endTime *time.Time `param:"end,query,milliseconds"` + + // limit for data size per page. Default: 100 + limit *uint64 `param:"limit,query"` +} + +type OrderList []OrderDetails + +// NewGetOrderHistoriesRequest is descending order by createdTime +func (c *RestClient) NewGetOrderHistoryRequest() *GetOrderHistoryRequest { + return &GetOrderHistoryRequest{ + client: c, + instrumentType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_order_history_request_requestgen.go b/pkg/exchange/okex/okexapi/get_order_history_request_requestgen.go new file mode 100644 index 0000000000..b9bc43596d --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_order_history_request_requestgen.go @@ -0,0 +1,319 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/orders-history-archive -type GetOrderHistoryRequest -responseDataType .OrderList"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetOrderHistoryRequest) InstrumentType(instrumentType InstrumentType) *GetOrderHistoryRequest { + g.instrumentType = instrumentType + return g +} + +func (g *GetOrderHistoryRequest) InstrumentID(instrumentID string) *GetOrderHistoryRequest { + g.instrumentID = &instrumentID + return g +} + +func (g *GetOrderHistoryRequest) OrderType(orderType OrderType) *GetOrderHistoryRequest { + g.orderType = &orderType + return g +} + +func (g *GetOrderHistoryRequest) Underlying(underlying string) *GetOrderHistoryRequest { + g.underlying = &underlying + return g +} + +func (g *GetOrderHistoryRequest) InstrumentFamily(instrumentFamily string) *GetOrderHistoryRequest { + g.instrumentFamily = &instrumentFamily + return g +} + +func (g *GetOrderHistoryRequest) State(state OrderState) *GetOrderHistoryRequest { + g.state = &state + return g +} + +func (g *GetOrderHistoryRequest) After(after string) *GetOrderHistoryRequest { + g.after = &after + return g +} + +func (g *GetOrderHistoryRequest) Before(before string) *GetOrderHistoryRequest { + g.before = &before + return g +} + +func (g *GetOrderHistoryRequest) StartTime(startTime time.Time) *GetOrderHistoryRequest { + g.startTime = &startTime + return g +} + +func (g *GetOrderHistoryRequest) EndTime(endTime time.Time) *GetOrderHistoryRequest { + g.endTime = &endTime + return g +} + +func (g *GetOrderHistoryRequest) Limit(limit uint64) *GetOrderHistoryRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetOrderHistoryRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instrumentType field -> json key instType + instrumentType := g.instrumentType + + // TEMPLATE check-valid-values + switch instrumentType { + case InstrumentTypeSpot, InstrumentTypeSwap, InstrumentTypeFutures, InstrumentTypeOption, InstrumentTypeMARGIN: + params["instType"] = instrumentType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instrumentType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instrumentType + params["instType"] = instrumentType + // check instrumentID field -> json key instId + if g.instrumentID != nil { + instrumentID := *g.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + } else { + } + // check orderType field -> json key ordType + if g.orderType != nil { + orderType := *g.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeMarket, OrderTypeLimit, OrderTypePostOnly, OrderTypeFOK, OrderTypeIOC: + params["ordType"] = orderType + + default: + return nil, fmt.Errorf("ordType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["ordType"] = orderType + } else { + } + // check underlying field -> json key uly + if g.underlying != nil { + underlying := *g.underlying + + // assign parameter of underlying + params["uly"] = underlying + } else { + } + // check instrumentFamily field -> json key instFamily + if g.instrumentFamily != nil { + instrumentFamily := *g.instrumentFamily + + // assign parameter of instrumentFamily + params["instFamily"] = instrumentFamily + } else { + } + // check state field -> json key state + if g.state != nil { + state := *g.state + + // TEMPLATE check-valid-values + switch state { + case OrderStateCanceled, OrderStateLive, OrderStatePartiallyFilled, OrderStateFilled: + params["state"] = state + + default: + return nil, fmt.Errorf("state value %v is invalid", state) + + } + // END TEMPLATE check-valid-values + + // assign parameter of state + params["state"] = state + } else { + } + // check after field -> json key after + if g.after != nil { + after := *g.after + + // assign parameter of after + params["after"] = after + } else { + } + // check before field -> json key before + if g.before != nil { + before := *g.before + + // assign parameter of before + params["before"] = before + } else { + } + // check startTime field -> json key begin + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["begin"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key end + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["end"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetOrderHistoryRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetOrderHistoryRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetOrderHistoryRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetOrderHistoryRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetOrderHistoryRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetOrderHistoryRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetOrderHistoryRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetOrderHistoryRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetOrderHistoryRequest) Do(ctx context.Context) (OrderList, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v5/trade/orders-history-archive" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data OrderList + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/get_transaction_histories_request.go b/pkg/exchange/okex/okexapi/get_transaction_history_request.go similarity index 79% rename from pkg/exchange/okex/okexapi/get_transaction_histories_request.go rename to pkg/exchange/okex/okexapi/get_transaction_history_request.go index 7e16d3b547..23e02fd4aa 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_histories_request.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request.go @@ -6,8 +6,8 @@ import ( "github.com/c9s/requestgen" ) -//go:generate GetRequest -url "/api/v5/trade/fills-history" -type GetTransactionHistoriesRequest -responseDataType .APIResponse -type GetTransactionHistoriesRequest struct { +//go:generate GetRequest -url "/api/v5/trade/fills-history" -type GetTransactionHistoryRequest -responseDataType .APIResponse +type GetTransactionHistoryRequest struct { client requestgen.AuthenticatedAPIClient instrumentType InstrumentType `param:"instType,query"` @@ -29,11 +29,9 @@ type GetTransactionHistoriesRequest struct { limit *uint64 `param:"limit,query"` } -type OrderList []OrderDetails - // NewGetOrderHistoriesRequest is descending order by createdTime -func (c *RestClient) NewGetTransactionHistoriesRequest() *GetTransactionHistoriesRequest { - return &GetTransactionHistoriesRequest{ +func (c *RestClient) NewGetTransactionHistoryRequest() *GetTransactionHistoryRequest { + return &GetTransactionHistoryRequest{ client: c, instrumentType: InstrumentTypeSpot, } diff --git a/pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go similarity index 73% rename from pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go rename to pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go index f88a68513a..00c3d71da5 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_histories_request_requestgen.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go @@ -1,4 +1,4 @@ -// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills-history -type GetTransactionHistoriesRequest -responseDataType .OrderList"; DO NOT EDIT. +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills-history -type GetTransactionHistoryRequest -responseDataType .OrderList"; DO NOT EDIT. package okexapi @@ -13,63 +13,63 @@ import ( "time" ) -func (g *GetTransactionHistoriesRequest) InstrumentType(instrumentType InstrumentType) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) InstrumentType(instrumentType InstrumentType) *GetTransactionHistoryRequest { g.instrumentType = instrumentType return g } -func (g *GetTransactionHistoriesRequest) InstrumentID(instrumentID string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) InstrumentID(instrumentID string) *GetTransactionHistoryRequest { g.instrumentID = &instrumentID return g } -func (g *GetTransactionHistoriesRequest) OrderType(orderType OrderType) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) OrderType(orderType OrderType) *GetTransactionHistoryRequest { g.orderType = &orderType return g } -func (g *GetTransactionHistoriesRequest) OrderID(orderID string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) OrderID(orderID string) *GetTransactionHistoryRequest { g.orderID = orderID return g } -func (g *GetTransactionHistoriesRequest) Underlying(underlying string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) Underlying(underlying string) *GetTransactionHistoryRequest { g.underlying = &underlying return g } -func (g *GetTransactionHistoriesRequest) InstrumentFamily(instrumentFamily string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) InstrumentFamily(instrumentFamily string) *GetTransactionHistoryRequest { g.instrumentFamily = &instrumentFamily return g } -func (g *GetTransactionHistoriesRequest) After(after string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) After(after string) *GetTransactionHistoryRequest { g.after = &after return g } -func (g *GetTransactionHistoriesRequest) Before(before string) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) Before(before string) *GetTransactionHistoryRequest { g.before = &before return g } -func (g *GetTransactionHistoriesRequest) StartTime(startTime time.Time) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) StartTime(startTime time.Time) *GetTransactionHistoryRequest { g.startTime = &startTime return g } -func (g *GetTransactionHistoriesRequest) EndTime(endTime time.Time) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) EndTime(endTime time.Time) *GetTransactionHistoryRequest { g.endTime = &endTime return g } -func (g *GetTransactionHistoriesRequest) Limit(limit uint64) *GetTransactionHistoriesRequest { +func (g *GetTransactionHistoryRequest) Limit(limit uint64) *GetTransactionHistoryRequest { g.limit = &limit return g } // GetQueryParameters builds and checks the query parameters and returns url.Values -func (g *GetTransactionHistoriesRequest) GetQueryParameters() (url.Values, error) { +func (g *GetTransactionHistoryRequest) GetQueryParameters() (url.Values, error) { var params = map[string]interface{}{} // check instrumentType field -> json key instType instrumentType := g.instrumentType @@ -187,14 +187,14 @@ func (g *GetTransactionHistoriesRequest) GetQueryParameters() (url.Values, error } // GetParameters builds and checks the parameters and return the result in a map object -func (g *GetTransactionHistoriesRequest) GetParameters() (map[string]interface{}, error) { +func (g *GetTransactionHistoryRequest) GetParameters() (map[string]interface{}, error) { var params = map[string]interface{}{} return params, nil } // GetParametersQuery converts the parameters from GetParameters into the url.Values format -func (g *GetTransactionHistoriesRequest) GetParametersQuery() (url.Values, error) { +func (g *GetTransactionHistoryRequest) GetParametersQuery() (url.Values, error) { query := url.Values{} params, err := g.GetParameters() @@ -216,7 +216,7 @@ func (g *GetTransactionHistoriesRequest) GetParametersQuery() (url.Values, error } // GetParametersJSON converts the parameters from GetParameters into the JSON format -func (g *GetTransactionHistoriesRequest) GetParametersJSON() ([]byte, error) { +func (g *GetTransactionHistoryRequest) GetParametersJSON() ([]byte, error) { params, err := g.GetParameters() if err != nil { return nil, err @@ -226,13 +226,13 @@ func (g *GetTransactionHistoriesRequest) GetParametersJSON() ([]byte, error) { } // GetSlugParameters builds and checks the slug parameters and return the result in a map object -func (g *GetTransactionHistoriesRequest) GetSlugParameters() (map[string]interface{}, error) { +func (g *GetTransactionHistoryRequest) GetSlugParameters() (map[string]interface{}, error) { var params = map[string]interface{}{} return params, nil } -func (g *GetTransactionHistoriesRequest) applySlugsToUrl(url string, slugs map[string]string) string { +func (g *GetTransactionHistoryRequest) applySlugsToUrl(url string, slugs map[string]string) string { for _k, _v := range slugs { needleRE := regexp.MustCompile(":" + _k + "\\b") url = needleRE.ReplaceAllString(url, _v) @@ -241,7 +241,7 @@ func (g *GetTransactionHistoriesRequest) applySlugsToUrl(url string, slugs map[s return url } -func (g *GetTransactionHistoriesRequest) iterateSlice(slice interface{}, _f func(it interface{})) { +func (g *GetTransactionHistoryRequest) iterateSlice(slice interface{}, _f func(it interface{})) { sliceValue := reflect.ValueOf(slice) for _i := 0; _i < sliceValue.Len(); _i++ { it := sliceValue.Index(_i).Interface() @@ -249,7 +249,7 @@ func (g *GetTransactionHistoriesRequest) iterateSlice(slice interface{}, _f func } } -func (g *GetTransactionHistoriesRequest) isVarSlice(_v interface{}) bool { +func (g *GetTransactionHistoryRequest) isVarSlice(_v interface{}) bool { rt := reflect.TypeOf(_v) switch rt.Kind() { case reflect.Slice: @@ -258,7 +258,7 @@ func (g *GetTransactionHistoriesRequest) isVarSlice(_v interface{}) bool { return false } -func (g *GetTransactionHistoriesRequest) GetSlugsMap() (map[string]string, error) { +func (g *GetTransactionHistoryRequest) GetSlugsMap() (map[string]string, error) { slugs := map[string]string{} params, err := g.GetSlugParameters() if err != nil { @@ -272,7 +272,7 @@ func (g *GetTransactionHistoriesRequest) GetSlugsMap() (map[string]string, error return slugs, nil } -func (g *GetTransactionHistoriesRequest) Do(ctx context.Context) (OrderList, error) { +func (g *GetTransactionHistoryRequest) Do(ctx context.Context) (OrderList, error) { // no body params var params interface{} diff --git a/pkg/exchange/okex/query_closed_orders_test.go b/pkg/exchange/okex/query_closed_orders_test.go new file mode 100644 index 0000000000..9b77e76658 --- /dev/null +++ b/pkg/exchange/okex/query_closed_orders_test.go @@ -0,0 +1,62 @@ +package okex + +import ( + "context" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/testutil" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func Test_QueryClosedOrders(t *testing.T) { + + key, secret, passphrase, ok := testutil.IntegrationTestWithPassphraseConfigured(t, types.ExchangeOKEx.String()) + if !ok { + t.Skip("Please configure all credentials about OKEX") + } + + e := New(key, secret, passphrase) + + queryOrder := types.OrderQuery{ + Symbol: "BTCUSDT", + } + + // test by order id as a cursor + closedOrder, err := e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Time{}, time.Time{}, 609869603774656544) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + // test by time interval + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Now().Add(-90*24*time.Hour), time.Now(), 0) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + // test by no parameter + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Time{}, time.Time{}, 0) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + // test by time interval (boundary test) + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Unix(1694155903, 999), time.Now(), 0) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + // test by time interval (boundary test) + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Unix(1694154903, 999), time.Unix(1694155904, 0), 0) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + // test by time interval and order id together + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Unix(1694154903, 999), time.Now(), 609869603774656544) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) +} diff --git a/pkg/exchange/okex/query_order_test.go b/pkg/exchange/okex/query_order_test.go index 3c32da40cf..03c3af1a44 100644 --- a/pkg/exchange/okex/query_order_test.go +++ b/pkg/exchange/okex/query_order_test.go @@ -25,7 +25,7 @@ func Test_QueryOrder(t *testing.T) { e := New(key, secret, passphrase) queryOrder := types.OrderQuery{ - Symbol: "BTC-USDT", + Symbol: "BTCUSDT", OrderID: "609869603774656544", } orderDetail, err := e.QueryOrder(context.Background(), queryOrder) diff --git a/pkg/exchange/okex/query_trades_test.go b/pkg/exchange/okex/query_trades_test.go new file mode 100644 index 0000000000..0058b6bcc6 --- /dev/null +++ b/pkg/exchange/okex/query_trades_test.go @@ -0,0 +1,57 @@ +package okex + +import ( + "context" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/testutil" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func Test_QueryTrades(t *testing.T) { + key, secret, passphrase, ok := testutil.IntegrationTestWithPassphraseConfigured(t, "OKEX") + if !ok { + t.Skip("Please configure all credentials about OKEX") + } + + e := New(key, secret, passphrase) + + queryOrder := types.OrderQuery{ + Symbol: "BTCUSDT", + } + + since := time.Now().AddDate(0, -3, 0) + until := time.Now() + + queryOption := types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + Limit: 100, + } + // query by time interval + transactionDetail, err := e.QueryTrades(context.Background(), queryOrder.Symbol, &queryOption) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) + // query by trade id + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{LastTradeID: 432044402}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) + // query by no time interval and no trade id + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) + // query by limit exceed default value + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{Limit: 150}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + t.Logf("transaction detail: %+v", transactionDetail) +} diff --git a/pkg/testutil/auth.go b/pkg/testutil/auth.go index 8e5bd43c7c..a4fae74b03 100644 --- a/pkg/testutil/auth.go +++ b/pkg/testutil/auth.go @@ -3,6 +3,7 @@ package testutil import ( "os" "regexp" + "strings" "testing" ) @@ -26,6 +27,7 @@ func IntegrationTestConfigured(t *testing.T, prefix string) (key, secret string, func IntegrationTestWithPassphraseConfigured(t *testing.T, prefix string) (key, secret, passphrase string, ok bool) { var hasKey, hasSecret, hasPassphrase bool + prefix = strings.ToUpper(prefix) key, hasKey = os.LookupEnv(prefix + "_API_KEY") secret, hasSecret = os.LookupEnv(prefix + "_API_SECRET") passphrase, hasPassphrase = os.LookupEnv(prefix + "_API_PASSPHRASE") From ad7206271f65218dd2b92de7a0e9e62613c53cc9 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Fri, 22 Sep 2023 13:42:42 +0800 Subject: [PATCH 026/422] QueryTrades only allow query by time interval, required --- pkg/exchange/okex/exchange.go | 39 ++++++++------------------ pkg/exchange/okex/query_trades_test.go | 12 ++++---- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index d578554fe0..337fff97d6 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -23,7 +23,6 @@ import ( // Market data limiter means public api, this includes QueryMarkets, QueryTicker, QueryTickers, QueryKLines var ( marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) - tradeRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) ) @@ -434,7 +433,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, return nil, ErrSymbolRequired } - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := orderRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) } @@ -478,9 +477,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, /* QueryTrades can query trades in last 3 months, there are no time interval limitations, as long as end_time >= start_time. OKEX do not provide api to query by tradeID, So use /api/v5/trade/orders-history-archive as its official site do. -Please Use LastTradeID as cursor, only return trades later than that trade, that trade is not included. If you want to query trades by time range, please just pass start_time and end_time. -If you want to query by cursor, please pass LastTradeID. Because it gets the correct response even when you pass all parameters with the right time interval and invalid LastTradeID, like 0. */ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { @@ -488,6 +485,10 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type return nil, ErrSymbolRequired } + if options.LastTradeID > 0 { + log.Warn("!!!OKEX EXCHANGE API NOTICE!!! Okex does not support searching for trades using TradeId.") + } + req := e.client.NewGetOrderHistoryRequest().InstrumentID(toLocalSymbol(symbol)) limit := uint64(options.Limit) @@ -497,14 +498,15 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req.Limit(limit) } - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := orderRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("query trades rate limiter wait error: %w", err) } var err error var response []okexapi.OrderDetails - // query by time interval - if options.StartTime != nil || options.EndTime != nil { + if options.StartTime == nil && options.EndTime == nil { + return nil, fmt.Errorf("StartTime and EndTime are require parameter!") + } else { // query by time interval if options.StartTime != nil { req.StartTime(*options.StartTime) } @@ -512,30 +514,11 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req.EndTime(*options.EndTime) } - response, err = req.Do(ctx) - if err != nil { - return nil, fmt.Errorf("failed to call get order histories error: %w", err) - } - } else if options.StartTime == nil && options.EndTime == nil && options.LastTradeID == 0 { // query by no any parameters - response, err = req.Do(ctx) - if err != nil { - return nil, fmt.Errorf("failed to call get order histories error: %w", err) - } - } else { // query by trade id - lastTradeID := strconv.FormatUint(options.LastTradeID, 10) - res, err := req.Do(ctx) + response, err = req. + Do(ctx) if err != nil { return nil, fmt.Errorf("failed to call get order histories error: %w", err) } - for _, trade := range res { - if trade.LastTradeID == lastTradeID { - response, err = req.Before(trade.OrderID).Do(ctx) - if err != nil { - return nil, fmt.Errorf("failed to call get order histories error: %w", err) - } - break - } - } } trades, err := toGlobalTrades(response) diff --git a/pkg/exchange/okex/query_trades_test.go b/pkg/exchange/okex/query_trades_test.go index 0058b6bcc6..5188be8d5b 100644 --- a/pkg/exchange/okex/query_trades_test.go +++ b/pkg/exchange/okex/query_trades_test.go @@ -38,20 +38,20 @@ func Test_QueryTrades(t *testing.T) { t.Logf("transaction detail: %+v", transactionDetail) // query by trade id transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{LastTradeID: 432044402}) - if assert.NoError(t, err) { - assert.NotEmpty(t, transactionDetail) + if assert.Error(t, err) { + assert.Empty(t, transactionDetail) } t.Logf("transaction detail: %+v", transactionDetail) // query by no time interval and no trade id transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{}) - if assert.NoError(t, err) { - assert.NotEmpty(t, transactionDetail) + if assert.Error(t, err) { + assert.Empty(t, transactionDetail) } t.Logf("transaction detail: %+v", transactionDetail) // query by limit exceed default value transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{Limit: 150}) - if assert.NoError(t, err) { - assert.NotEmpty(t, transactionDetail) + if assert.Error(t, err) { + assert.Empty(t, transactionDetail) } t.Logf("transaction detail: %+v", transactionDetail) } From 7ae56a83da186c0a3aca72ea7dfa52c56ea7f05c Mon Sep 17 00:00:00 2001 From: zenix Date: Fri, 14 Jul 2023 15:20:44 +0900 Subject: [PATCH 027/422] feature: add forceOrder api for binance to show liquid info --- pkg/exchange/binance/convert.go | 2 + pkg/exchange/binance/parse.go | 62 +++++++++++++++++++++++- pkg/exchange/binance/stream.go | 6 +++ pkg/exchange/binance/stream_callbacks.go | 12 +++++ pkg/types/channel.go | 1 + pkg/types/liquidation_info.go | 15 ++++++ pkg/types/standardstream_callbacks.go | 12 +++++ pkg/types/stream.go | 3 ++ 8 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 pkg/types/liquidation_info.go diff --git a/pkg/exchange/binance/convert.go b/pkg/exchange/binance/convert.go index 7586770d48..378c647e08 100644 --- a/pkg/exchange/binance/convert.go +++ b/pkg/exchange/binance/convert.go @@ -340,6 +340,8 @@ func convertSubscription(s types.Subscription) string { return fmt.Sprintf("%s@trade", strings.ToLower(s.Symbol)) case types.AggTradeChannel: return fmt.Sprintf("%s@aggTrade", strings.ToLower(s.Symbol)) + case types.ForceOrderChannel: + return fmt.Sprintf("%s@forceOrder", strings.ToLower(s.Symbol)) } return fmt.Sprintf("%s@%s", strings.ToLower(s.Symbol), s.Channel) diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index a1350faf4b..0d2f2f431e 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -362,7 +362,10 @@ func parseWebSocketEvent(message []byte) (interface{}, error) { var event AggTradeEvent err = json.Unmarshal([]byte(message), &event) return &event, err - + case "forceOrder": + var event ForceOrderEvent + err = json.Unmarshal([]byte(message), &event) + return &event, err } // futures stream @@ -530,6 +533,63 @@ func parseDepthEvent(val *fastjson.Value) (*DepthEvent, error) { return depth, err } +type ForceOrderEventInner struct { + Symbol string `json:"s"` + TradeTime int64 `json:"T"` + Side string `json:"S"` + OrderType string `json:"o"` + TimeInForce string `json:"f"` + Quantity fixedpoint.Value `json:"q"` + Price fixedpoint.Value `json:"p"` + AveragePrice fixedpoint.Value `json:"ap"` + OrderStatus string `json:"X"` + LastFilledQuantity fixedpoint.Value `json:"l"` + LastFilledAccQuantity fixedpoint.Value `json:"z"` +} + +type ForceOrderEvent struct { + EventBase + Order ForceOrderEventInner `json:"o"` +} + +func (e *ForceOrderEvent) LiquidationInfo() types.LiquidationInfo { + o := e.Order + tt := time.Unix(0, o.TradeTime*int64(time.Millisecond)) + return types.LiquidationInfo{ + Symbol: o.Symbol, + Side: types.SideType(o.Side), + OrderType: types.OrderType(o.OrderType), + TimeInForce: types.TimeInForce(o.TimeInForce), + Quantity: o.Quantity, + Price: o.Price, + AveragePrice: o.AveragePrice, + OrderStatus: types.OrderStatus(o.OrderStatus), + TradeTime: types.Time(tt), + } +} + +/* +ForceOrderEvent + +{ + "E" : 1689303434028, + "e" : "forceOrder", + "o" : { + "S" : "BUY", // Side + "T" : 1689303434025, // Order Trade Time + "X" : "FILLED", // Order Status + "ap" : "2011.09", // Average Price + "f" : "IOC", // TimeInForce + "l" : "0.003", // Last filled Quantity + "o" : "LIMIT", // Order Type + "p" : "2021.37", // Price + "q" : "0.003", // Original Quantity + "s" : "ETHUSDT", // Symbol + "z" : "0.003" // Order Filed Accumulated Quantity + } +} +*/ + type MarketTradeEvent struct { EventBase Symbol string `json:"s"` diff --git a/pkg/exchange/binance/stream.go b/pkg/exchange/binance/stream.go index ce8de62d66..d409659e76 100644 --- a/pkg/exchange/binance/stream.go +++ b/pkg/exchange/binance/stream.go @@ -48,6 +48,7 @@ type Stream struct { marketTradeEventCallbacks []func(e *MarketTradeEvent) aggTradeEventCallbacks []func(e *AggTradeEvent) + forceOrderEventCallbacks []func(e *ForceOrderEvent) balanceUpdateEventCallbacks []func(event *BalanceUpdateEvent) outboundAccountInfoEventCallbacks []func(event *OutboundAccountInfoEvent) @@ -126,6 +127,7 @@ func NewStream(ex *Exchange, client *binance.Client, futuresClient *futures.Clie stream.OnContinuousKLineEvent(stream.handleContinuousKLineEvent) stream.OnMarketTradeEvent(stream.handleMarketTradeEvent) stream.OnAggTradeEvent(stream.handleAggTradeEvent) + stream.OnForceOrderEvent(stream.handleForceOrderEvent) // Futures User Data Stream // =================================== @@ -233,6 +235,10 @@ func (s *Stream) handleAggTradeEvent(e *AggTradeEvent) { s.EmitAggTrade(e.Trade()) } +func (s *Stream) handleForceOrderEvent(e *ForceOrderEvent) { + s.EmitForceOrder(e.LiquidationInfo()) +} + func (s *Stream) handleKLineEvent(e *KLineEvent) { kline := e.KLine.KLine() if e.KLine.Closed { diff --git a/pkg/exchange/binance/stream_callbacks.go b/pkg/exchange/binance/stream_callbacks.go index a90b4f668c..ecf80718bb 100644 --- a/pkg/exchange/binance/stream_callbacks.go +++ b/pkg/exchange/binance/stream_callbacks.go @@ -54,6 +54,16 @@ func (s *Stream) EmitAggTradeEvent(e *AggTradeEvent) { } } +func (s *Stream) OnForceOrderEvent(cb func(e *ForceOrderEvent)) { + s.forceOrderEventCallbacks = append(s.forceOrderEventCallbacks, cb) +} + +func (s *Stream) EmitForceOrderEvent(e *ForceOrderEvent) { + for _, cb := range s.forceOrderEventCallbacks { + cb(e) + } +} + func (s *Stream) OnBalanceUpdateEvent(cb func(event *BalanceUpdateEvent)) { s.balanceUpdateEventCallbacks = append(s.balanceUpdateEventCallbacks, cb) } @@ -195,6 +205,8 @@ type StreamEventHub interface { OnAggTradeEvent(cb func(e *AggTradeEvent)) + OnForceOrderEvent(cb func(e *ForceOrderEvent)) + OnBalanceUpdateEvent(cb func(event *BalanceUpdateEvent)) OnOutboundAccountInfoEvent(cb func(event *OutboundAccountInfoEvent)) diff --git a/pkg/types/channel.go b/pkg/types/channel.go index 2e85b92364..f9b8a78fa6 100644 --- a/pkg/types/channel.go +++ b/pkg/types/channel.go @@ -8,6 +8,7 @@ const ( BookTickerChannel = Channel("bookTicker") MarketTradeChannel = Channel("trade") AggTradeChannel = Channel("aggTrade") + ForceOrderChannel = Channel("forceOrder") // channels for futures MarkPriceChannel = Channel("markPrice") diff --git a/pkg/types/liquidation_info.go b/pkg/types/liquidation_info.go new file mode 100644 index 0000000000..9c9477c1c5 --- /dev/null +++ b/pkg/types/liquidation_info.go @@ -0,0 +1,15 @@ +package types + +import "github.com/c9s/bbgo/pkg/fixedpoint" + +type LiquidationInfo struct { + Symbol string + Side SideType + OrderType OrderType + TimeInForce TimeInForce + Quantity fixedpoint.Value + Price fixedpoint.Value + AveragePrice fixedpoint.Value + OrderStatus OrderStatus + TradeTime Time +} diff --git a/pkg/types/standardstream_callbacks.go b/pkg/types/standardstream_callbacks.go index 55a3e7c6b6..c75e752f10 100644 --- a/pkg/types/standardstream_callbacks.go +++ b/pkg/types/standardstream_callbacks.go @@ -164,6 +164,16 @@ func (s *StandardStream) EmitAggTrade(trade Trade) { } } +func (s *StandardStream) OnForceOrder(cb func(info LiquidationInfo)) { + s.forceOrderCallbacks = append(s.forceOrderCallbacks, cb) +} + +func (s *StandardStream) EmitForceOrder(info LiquidationInfo) { + for _, cb := range s.forceOrderCallbacks { + cb(info) + } +} + func (s *StandardStream) OnFuturesPositionUpdate(cb func(futuresPositions FuturesPositionMap)) { s.FuturesPositionUpdateCallbacks = append(s.FuturesPositionUpdateCallbacks, cb) } @@ -217,6 +227,8 @@ type StandardStreamEventHub interface { OnAggTrade(cb func(trade Trade)) + OnForceOrder(cb func(info LiquidationInfo)) + OnFuturesPositionUpdate(cb func(futuresPositions FuturesPositionMap)) OnFuturesPositionSnapshot(cb func(futuresPositions FuturesPositionMap)) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 73bd448567..50cbf59a03 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -132,6 +132,8 @@ type StandardStream struct { aggTradeCallbacks []func(trade Trade) + forceOrderCallbacks []func(info LiquidationInfo) + // Futures FuturesPositionUpdateCallbacks []func(futuresPositions FuturesPositionMap) @@ -159,6 +161,7 @@ type StandardStreamEmitter interface { EmitBookSnapshot(SliceOrderBook) EmitMarketTrade(Trade) EmitAggTrade(Trade) + EmitForceOrder(LiquidationInfo) EmitFuturesPositionUpdate(FuturesPositionMap) EmitFuturesPositionSnapshot(FuturesPositionMap) } From 9fffa4a47fdf87a51d4f44ddc2abb10f48dec516 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 15:32:55 +0800 Subject: [PATCH 028/422] add atrpin strategy --- pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/atrpin/strategy.go | 161 ++++++++++++++++++++++++++++++++ pkg/types/position.go | 2 + 3 files changed, 164 insertions(+) create mode 100644 pkg/strategy/atrpin/strategy.go diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 45ac2d7343..05b652a25c 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -2,6 +2,7 @@ package strategy // import built-in strategies import ( + _ "github.com/c9s/bbgo/pkg/strategy/atrpin" _ "github.com/c9s/bbgo/pkg/strategy/audacitymaker" _ "github.com/c9s/bbgo/pkg/strategy/autoborrow" _ "github.com/c9s/bbgo/pkg/strategy/bollgrid" diff --git a/pkg/strategy/atrpin/strategy.go b/pkg/strategy/atrpin/strategy.go new file mode 100644 index 0000000000..a5a462bf41 --- /dev/null +++ b/pkg/strategy/atrpin/strategy.go @@ -0,0 +1,161 @@ +package atrpin + +import ( + "context" + "fmt" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "atrpin" + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + + Interval types.Interval `json:"interval"` + Window int `json:"slowWindow"` + Multiplier float64 `json:"multiplier"` + + bbgo.QuantityOrAmount + // bbgo.OpenPositionOptions +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s:%s:%d", ID, s.Symbol, s.Interval, s.Window) +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) +} + +func (s *Strategy) Defaults() error { + if s.Multiplier == 0.0 { + s.Multiplier = 10.0 + } + + if s.Interval == "" { + s.Interval = types.Interval5m + } + + return nil +} + +func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + + atr := session.Indicators(s.Symbol).ATR(s.Interval, s.Window) + session.UserDataStream.OnKLine(types.KLineWith(s.Symbol, s.Interval, func(k types.KLine) { + if err := s.Strategy.OrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Error("unable to cancel open orders...") + } + + lastAtr := atr.Last(0) + + // protection + if lastAtr <= k.High.Sub(k.Low).Float64() { + lastAtr = k.High.Sub(k.Low).Float64() + } + + priceRange := fixedpoint.NewFromFloat(lastAtr * s.Multiplier) + + ticker, err := session.Exchange.QueryTicker(ctx, s.Symbol) + if err != nil { + log.WithError(err).Error("unable to query ticker") + return + } + + bidPrice := ticker.Buy.Sub(priceRange) + askPrice := ticker.Sell.Add(priceRange) + + bidQuantity := s.QuantityOrAmount.CalculateQuantity(bidPrice) + askQuantity := s.QuantityOrAmount.CalculateQuantity(askPrice) + + var orderForms []types.SubmitOrder + + position := s.Strategy.OrderExecutor.Position() + if !position.IsDust() { + side := types.SideTypeSell + takerPrice := fixedpoint.Zero + + if position.IsShort() { + side = types.SideTypeBuy + takerPrice = askPrice + } else if position.IsLong() { + side = types.SideTypeSell + takerPrice = bidPrice + } + + orderForms = append(orderForms, types.SubmitOrder{ + Symbol: s.Symbol, + Side: side, + Price: takerPrice, + Quantity: position.GetQuantity(), + Market: s.Market, + }) + } + + orderForms = append(orderForms, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeSell, + Price: askPrice, + Quantity: askQuantity, + Market: s.Market, + }) + + orderForms = append(orderForms, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Price: bidPrice, + Quantity: bidQuantity, + Market: s.Market, + }) + + if _, err := s.Strategy.OrderExecutor.SubmitOrders(ctx, orderForms...); err != nil { + log.WithError(err).Error("unable to submit orders") + } + })) + + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + }) + + return nil +} + +func logErr(err error, msgAndArgs ...interface{}) bool { + if err == nil { + return false + } + + if len(msgAndArgs) == 0 { + log.WithError(err).Error(err.Error()) + } else if len(msgAndArgs) == 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Error(msg) + } else if len(msgAndArgs) > 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Errorf(msg, msgAndArgs[1:]...) + } + + return true +} diff --git a/pkg/types/position.go b/pkg/types/position.go index 5f2bf58f79..983c1c5511 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -192,6 +192,8 @@ func (p *Position) GetBase() (base fixedpoint.Value) { return base } +// GetQuantity calls GetBase() and then convert the number into a positive number +// that could be treated as a quantity. func (p *Position) GetQuantity() fixedpoint.Value { base := p.GetBase() return base.Abs() From 7a5a027a629bd532cf4a71a7d7405d813d81961c Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 16:45:00 +0800 Subject: [PATCH 029/422] bbgo: add logging filledOrder option --- pkg/bbgo/config.go | 7 ++++--- pkg/bbgo/session.go | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index d5314283a6..6abcff37be 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -92,9 +92,10 @@ type NotificationConfig struct { } type LoggingConfig struct { - Trade bool `json:"trade,omitempty"` - Order bool `json:"order,omitempty"` - Fields map[string]interface{} `json:"fields,omitempty"` + Trade bool `json:"trade,omitempty"` + Order bool `json:"order,omitempty"` + FilledOrderOnly bool `json:"filledOrder,omitempty"` + Fields map[string]interface{} `json:"fields,omitempty"` } type Session struct { diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 0465346661..dc3bd54fd9 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -282,7 +282,13 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) }) } - if environ.loggingConfig.Order { + if environ.loggingConfig.FilledOrderOnly { + session.UserDataStream.OnOrderUpdate(func(order types.Order) { + if order.Status == types.OrderStatusFilled { + logger.Info(order.String()) + } + }) + } else if environ.loggingConfig.Order { session.UserDataStream.OnOrderUpdate(func(order types.Order) { logger.Info(order.String()) }) From 70884538bc7dc09899d37c2284fc881d21886037 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 26 Sep 2023 16:05:52 +0800 Subject: [PATCH 030/422] pkg/exchange: emit balance snapshot --- pkg/exchange/bybit/mocks/stream.go | 51 +++++++++++++++++++----------- pkg/exchange/bybit/stream.go | 51 ++++++++++++++++++++++++------ pkg/exchange/bybit/stream_test.go | 15 +++++---- 3 files changed, 84 insertions(+), 33 deletions(-) diff --git a/pkg/exchange/bybit/mocks/stream.go b/pkg/exchange/bybit/mocks/stream.go index 9d20878e85..6a3c9d8766 100644 --- a/pkg/exchange/bybit/mocks/stream.go +++ b/pkg/exchange/bybit/mocks/stream.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/c9s/bbgo/pkg/exchange/bybit (interfaces: MarketInfoProvider) +// Source: github.com/c9s/bbgo/pkg/exchange/bybit (interfaces: StreamDataProvider) // Package mocks is a generated GoMock package. package mocks @@ -13,31 +13,31 @@ import ( gomock "github.com/golang/mock/gomock" ) -// MockMarketInfoProvider is a mock of MarketInfoProvider interface. -type MockMarketInfoProvider struct { +// MockStreamDataProvider is a mock of StreamDataProvider interface. +type MockStreamDataProvider struct { ctrl *gomock.Controller - recorder *MockMarketInfoProviderMockRecorder + recorder *MockStreamDataProviderMockRecorder } -// MockMarketInfoProviderMockRecorder is the mock recorder for MockMarketInfoProvider. -type MockMarketInfoProviderMockRecorder struct { - mock *MockMarketInfoProvider +// MockStreamDataProviderMockRecorder is the mock recorder for MockStreamDataProvider. +type MockStreamDataProviderMockRecorder struct { + mock *MockStreamDataProvider } -// NewMockMarketInfoProvider creates a new mock instance. -func NewMockMarketInfoProvider(ctrl *gomock.Controller) *MockMarketInfoProvider { - mock := &MockMarketInfoProvider{ctrl: ctrl} - mock.recorder = &MockMarketInfoProviderMockRecorder{mock} +// NewMockStreamDataProvider creates a new mock instance. +func NewMockStreamDataProvider(ctrl *gomock.Controller) *MockStreamDataProvider { + mock := &MockStreamDataProvider{ctrl: ctrl} + mock.recorder = &MockStreamDataProviderMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMarketInfoProvider) EXPECT() *MockMarketInfoProviderMockRecorder { +func (m *MockStreamDataProvider) EXPECT() *MockStreamDataProviderMockRecorder { return m.recorder } // GetAllFeeRates mocks base method. -func (m *MockMarketInfoProvider) GetAllFeeRates(arg0 context.Context) (bybitapi.FeeRates, error) { +func (m *MockStreamDataProvider) GetAllFeeRates(arg0 context.Context) (bybitapi.FeeRates, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllFeeRates", arg0) ret0, _ := ret[0].(bybitapi.FeeRates) @@ -46,13 +46,28 @@ func (m *MockMarketInfoProvider) GetAllFeeRates(arg0 context.Context) (bybitapi. } // GetAllFeeRates indicates an expected call of GetAllFeeRates. -func (mr *MockMarketInfoProviderMockRecorder) GetAllFeeRates(arg0 interface{}) *gomock.Call { +func (mr *MockStreamDataProviderMockRecorder) GetAllFeeRates(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllFeeRates", reflect.TypeOf((*MockMarketInfoProvider)(nil).GetAllFeeRates), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllFeeRates", reflect.TypeOf((*MockStreamDataProvider)(nil).GetAllFeeRates), arg0) +} + +// QueryAccountBalances mocks base method. +func (m *MockStreamDataProvider) QueryAccountBalances(arg0 context.Context) (types.BalanceMap, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryAccountBalances", arg0) + ret0, _ := ret[0].(types.BalanceMap) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryAccountBalances indicates an expected call of QueryAccountBalances. +func (mr *MockStreamDataProviderMockRecorder) QueryAccountBalances(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryAccountBalances", reflect.TypeOf((*MockStreamDataProvider)(nil).QueryAccountBalances), arg0) } // QueryMarkets mocks base method. -func (m *MockMarketInfoProvider) QueryMarkets(arg0 context.Context) (types.MarketMap, error) { +func (m *MockStreamDataProvider) QueryMarkets(arg0 context.Context) (types.MarketMap, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "QueryMarkets", arg0) ret0, _ := ret[0].(types.MarketMap) @@ -61,7 +76,7 @@ func (m *MockMarketInfoProvider) QueryMarkets(arg0 context.Context) (types.Marke } // QueryMarkets indicates an expected call of QueryMarkets. -func (mr *MockMarketInfoProviderMockRecorder) QueryMarkets(arg0 interface{}) *gomock.Call { +func (mr *MockStreamDataProviderMockRecorder) QueryMarkets(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkets", reflect.TypeOf((*MockMarketInfoProvider)(nil).QueryMarkets), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryMarkets", reflect.TypeOf((*MockStreamDataProvider)(nil).QueryMarkets), arg0) } diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index ef756a7570..eab0c83e74 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -11,6 +11,7 @@ import ( "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" ) const ( @@ -27,18 +28,29 @@ var ( wsAuthRequest = 10 * time.Second ) -//go:generate mockgen -destination=mocks/stream.go -package=mocks . MarketInfoProvider +// MarketInfoProvider calculates trade fees since trading fees are not supported by streaming. type MarketInfoProvider interface { GetAllFeeRates(ctx context.Context) (bybitapi.FeeRates, error) QueryMarkets(ctx context.Context) (types.MarketMap, error) } +// AccountBalanceProvider provides a function to query all balances at streaming connected and emit balance snapshot. +type AccountBalanceProvider interface { + QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) +} + +//go:generate mockgen -destination=mocks/stream.go -package=mocks . StreamDataProvider +type StreamDataProvider interface { + MarketInfoProvider + AccountBalanceProvider +} + //go:generate callbackgen -type Stream type Stream struct { types.StandardStream - key, secret string - marketProvider MarketInfoProvider + key, secret string + streamDataProvider StreamDataProvider // TODO: update the fee rate at 7:00 am UTC; rotation required. symbolFeeDetails map[string]*symbolFeeDetail @@ -50,13 +62,13 @@ type Stream struct { tradeEventCallbacks []func(e []TradeEvent) } -func NewStream(key, secret string, marketProvider MarketInfoProvider) *Stream { +func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream { stream := &Stream{ StandardStream: types.NewStandardStream(), // pragma: allowlist nextline secret - key: key, - secret: secret, - marketProvider: marketProvider, + key: key, + secret: secret, + streamDataProvider: userDataProvider, } stream.SetEndpointCreator(stream.createEndpoint) @@ -65,6 +77,7 @@ func NewStream(key, secret string, marketProvider MarketInfoProvider) *Stream { stream.SetHeartBeat(stream.ping) stream.SetBeforeConnect(stream.getAllFeeRates) stream.OnConnect(stream.handlerConnect) + stream.OnAuth(stream.handleAuthEvent) stream.OnBookEvent(stream.handleBookEvent) stream.OnMarketTradeEvent(stream.handleMarketTradeEvent) @@ -326,6 +339,26 @@ func (s *Stream) convertSubscription(sub types.Subscription) (string, error) { return "", fmt.Errorf("unsupported stream channel: %s", sub.Channel) } +func (s *Stream) handleAuthEvent() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + var balnacesMap types.BalanceMap + var err error + err = util.Retry(ctx, 10, 300*time.Millisecond, func() error { + balnacesMap, err = s.streamDataProvider.QueryAccountBalances(ctx) + return err + }, func(err error) { + log.WithError(err).Error("failed to call query account balances") + }) + if err != nil { + log.WithError(err).Error("no more attempts to retrieve balances") + return + } + + s.EmitBalanceSnapshot(balnacesMap) +} + func (s *Stream) handleBookEvent(e BookEvent) { orderBook := e.OrderBook() switch { @@ -417,7 +450,7 @@ type symbolFeeDetail struct { // getAllFeeRates retrieves all fee rates from the Bybit API and then fetches markets to ensure the base coin and quote coin // are correct. func (e *Stream) getAllFeeRates(ctx context.Context) error { - feeRates, err := e.marketProvider.GetAllFeeRates(ctx) + feeRates, err := e.streamDataProvider.GetAllFeeRates(ctx) if err != nil { return fmt.Errorf("failed to call get fee rates: %w", err) } @@ -429,7 +462,7 @@ func (e *Stream) getAllFeeRates(ctx context.Context) error { } } - mkts, err := e.marketProvider.QueryMarkets(ctx) + mkts, err := e.streamDataProvider.QueryMarkets(ctx) if err != nil { return fmt.Errorf("failed to get markets: %w", err) } diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index bcf3572a6f..8e322a4391 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -54,6 +54,9 @@ func TestStream(t *testing.T) { } t.Run("Auth test", func(t *testing.T) { + s.OnBalanceSnapshot(func(balances types.BalanceMap) { + t.Log("got balance snapshot", balances) + }) s.Connect(context.Background()) c := make(chan struct{}) <-c @@ -450,9 +453,9 @@ func TestStream_getFeeRate(t *testing.T) { unknownErr := errors.New("unknown err") t.Run("succeeds", func(t *testing.T) { - mockMarketProvider := mocks.NewMockMarketInfoProvider(mockCtrl) + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) s := &Stream{ - marketProvider: mockMarketProvider, + streamDataProvider: mockMarketProvider, } ctx := context.Background() @@ -510,9 +513,9 @@ func TestStream_getFeeRate(t *testing.T) { }) t.Run("failed to query markets", func(t *testing.T) { - mockMarketProvider := mocks.NewMockMarketInfoProvider(mockCtrl) + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) s := &Stream{ - marketProvider: mockMarketProvider, + streamDataProvider: mockMarketProvider, } ctx := context.Background() @@ -545,9 +548,9 @@ func TestStream_getFeeRate(t *testing.T) { }) t.Run("failed to get fee rates", func(t *testing.T) { - mockMarketProvider := mocks.NewMockMarketInfoProvider(mockCtrl) + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) s := &Stream{ - marketProvider: mockMarketProvider, + streamDataProvider: mockMarketProvider, } ctx := context.Background() From 9f83165032ae515f91643bc9c8c2a6817126720f Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 26 Sep 2023 16:07:54 +0800 Subject: [PATCH 031/422] pkg/exchange: use balance update instead of snapshot event --- pkg/exchange/bybit/stream.go | 2 +- pkg/exchange/bybit/stream_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index eab0c83e74..145591cac0 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -385,7 +385,7 @@ func (s *Stream) handleMarketTradeEvent(events []MarketTradeEvent) { } func (s *Stream) handleWalletEvent(events []bybitapi.WalletBalances) { - s.StandardStream.EmitBalanceSnapshot(toGlobalBalanceMap(events)) + s.StandardStream.EmitBalanceUpdate(toGlobalBalanceMap(events)) } func (s *Stream) handleOrderEvent(events []OrderEvent) { diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index 8e322a4391..7cfc4d99b6 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -135,8 +135,8 @@ func TestStream(t *testing.T) { err := s.Connect(context.Background()) assert.NoError(t, err) - s.OnBalanceSnapshot(func(balances types.BalanceMap) { - t.Log("got snapshot", balances) + s.OnBalanceUpdate(func(balances types.BalanceMap) { + t.Log("got update", balances) }) c := make(chan struct{}) <-c From 13b9fc425243c7e3ad7dc56769b3cc0480064488 Mon Sep 17 00:00:00 2001 From: zenix Date: Tue, 26 Sep 2023 18:36:46 +0900 Subject: [PATCH 032/422] add forgotten emit --- pkg/exchange/binance/stream.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/exchange/binance/stream.go b/pkg/exchange/binance/stream.go index d409659e76..a13bc175e6 100644 --- a/pkg/exchange/binance/stream.go +++ b/pkg/exchange/binance/stream.go @@ -384,6 +384,9 @@ func (s *Stream) dispatchEvent(e interface{}) { case *ListenKeyExpired: s.EmitListenKeyExpired(e) + case *ForceOrderEvent: + s.EmitForceOrderEvent(e) + case *MarginCallEvent: } From 2e4336a6043a7ff93d5bb80e89eb518d83f967ab Mon Sep 17 00:00:00 2001 From: zenix Date: Tue, 26 Sep 2023 18:41:15 +0900 Subject: [PATCH 033/422] fix: listenKeyExpired event sends string timestamp --- pkg/exchange/binance/parse.go | 13 +++++++++---- pkg/exchange/binance/stream.go | 8 ++++++-- pkg/strategy/xfunding/strategy.go | 10 +++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index a1350faf4b..03af3c3472 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "time" "github.com/adshao/go-binance/v2/futures" @@ -17,8 +18,8 @@ import ( ) type EventBase struct { - Event string `json:"e"` // event name - Time int64 `json:"E"` // event time + Event string `json:"e"` // event name + Time json.Number `json:"E"` // event time } /* @@ -461,7 +462,11 @@ func (e *DepthEvent) String() (o string) { func (e *DepthEvent) OrderBook() (book types.SliceOrderBook, err error) { book.Symbol = e.Symbol - book.Time = types.NewMillisecondTimestampFromInt(e.EventBase.Time).Time() + t, err := e.EventBase.Time.Int64() + if err != nil { + return book, err + } + book.Time = types.NewMillisecondTimestampFromInt(t).Time() // already in descending order book.Bids = e.Bids @@ -500,7 +505,7 @@ func parseDepthEvent(val *fastjson.Value) (*DepthEvent, error) { var depth = &DepthEvent{ EventBase: EventBase{ Event: string(val.GetStringBytes("e")), - Time: val.GetInt64("E"), + Time: json.Number(strconv.FormatInt(val.GetInt64("E"), 10)), }, Symbol: string(val.GetStringBytes("s")), FirstUpdateID: val.GetInt64("U"), diff --git a/pkg/exchange/binance/stream.go b/pkg/exchange/binance/stream.go index ce8de62d66..e2314ad0f2 100644 --- a/pkg/exchange/binance/stream.go +++ b/pkg/exchange/binance/stream.go @@ -86,9 +86,13 @@ func NewStream(ex *Exchange, client *binance.Client, futuresClient *futures.Clie stream.OnDepthEvent(func(e *DepthEvent) { f, ok := stream.depthBuffers[e.Symbol] if ok { - err := f.AddUpdate(types.SliceOrderBook{ + t, err := e.EventBase.Time.Int64() + if err != nil { + log.WithError(err).Errorf("Time parsing failed: %v", e.EventBase.Time) + } + err = f.AddUpdate(types.SliceOrderBook{ Symbol: e.Symbol, - Time: types.NewMillisecondTimestampFromInt(e.EventBase.Time).Time(), + Time: types.NewMillisecondTimestampFromInt(t).Time(), Bids: e.Bids, Asks: e.Asks, }, e.FirstUpdateID, e.FinalUpdateID) diff --git a/pkg/strategy/xfunding/strategy.go b/pkg/strategy/xfunding/strategy.go index a5e75c082a..02a68a2428 100644 --- a/pkg/strategy/xfunding/strategy.go +++ b/pkg/strategy/xfunding/strategy.go @@ -527,15 +527,19 @@ func (s *Strategy) handleAccountUpdate(ctx context.Context, e *binance.AccountUp if b.Asset != s.ProfitStats.FundingFeeCurrency { continue } - - txnTime := time.UnixMilli(e.Time) + t, err := e.EventBase.Time.Int64() + if err != nil { + log.WithError(err).Error("unable to parse event timestamp") + continue + } + txnTime := time.UnixMilli(t) fee := FundingFee{ Asset: b.Asset, Amount: b.BalanceChange, Txn: e.Transaction, Time: txnTime, } - err := s.ProfitStats.AddFundingFee(fee) + err = s.ProfitStats.AddFundingFee(fee) if err != nil { log.WithError(err).Error("unable to add funding fee to profitStats") continue From 117d7f008f9ed5a0a8c232660bbb2f641c42a48c Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:41:23 +0800 Subject: [PATCH 034/422] types: add stringer on type ticker --- pkg/types/ticker.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/types/ticker.go b/pkg/types/ticker.go index a1649f4194..d985fbc352 100644 --- a/pkg/types/ticker.go +++ b/pkg/types/ticker.go @@ -1,8 +1,10 @@ package types import ( - "github.com/c9s/bbgo/pkg/fixedpoint" + "fmt" "time" + + "github.com/c9s/bbgo/pkg/fixedpoint" ) type Ticker struct { @@ -15,3 +17,7 @@ type Ticker struct { Buy fixedpoint.Value // `buy` from Max, `bidPrice` from binance Sell fixedpoint.Value // `sell` from Max, `askPrice` from binance } + +func (t *Ticker) String() string { + return fmt.Sprintf("O:%s H:%s L:%s LAST:%s BID/ASK:%s/%s TIME:%s", t.Open, t.High, t.Low, t.Last, t.Buy, t.Sell, t.Time.String()) +} From 716fea885f322a2d10e5da282fd0cbb28ffd65c6 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:41:37 +0800 Subject: [PATCH 035/422] backtest: add more order checking --- pkg/backtest/exchange.go | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index b358973ee1..851c4df14e 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -50,6 +50,7 @@ var log = logrus.WithField("cmd", "backtest") var ErrUnimplemented = errors.New("unimplemented method") var ErrNegativeQuantity = errors.New("order quantity can not be negative") var ErrZeroQuantity = errors.New("order quantity can not be zero") +var ErrEmptyOrderType = errors.New("order type can not be empty string") type Exchange struct { sourceName types.ExchangeName @@ -76,7 +77,9 @@ type Exchange struct { Src *ExchangeDataSource } -func NewExchange(sourceName types.ExchangeName, sourceExchange types.Exchange, srv *service.BacktestService, config *bbgo.Backtest) (*Exchange, error) { +func NewExchange( + sourceName types.ExchangeName, sourceExchange types.Exchange, srv *service.BacktestService, config *bbgo.Backtest, +) (*Exchange, error) { ex := sourceExchange markets, err := cache.LoadExchangeMarketsWithCache(context.Background(), ex) @@ -178,6 +181,10 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr return nil, fmt.Errorf("matching engine is not initialized for symbol %s", symbol) } + if order.Price.Sign() < 0 { + return nil, fmt.Errorf("order price can not be negative, %s given", order.Price.String()) + } + if order.Quantity.Sign() < 0 { return nil, ErrNegativeQuantity } @@ -186,6 +193,10 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr return nil, ErrZeroQuantity } + if order.Type == "" { + return nil, ErrEmptyOrderType + } + createdOrder, _, err = matching.PlaceOrder(order) if createdOrder != nil { // market order can be closed immediately. @@ -207,7 +218,9 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return append(matching.bidOrders, matching.askOrders...), nil } -func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { +func (e *Exchange) QueryClosedOrders( + ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, +) (orders []types.Order, err error) { orders, ok := e.closedOrders[symbol] if !ok { return orders, fmt.Errorf("matching engine is not initialized for symbol %s", symbol) @@ -239,7 +252,9 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, return e.account.Balances(), nil } -func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { +func (e *Exchange) QueryKLines( + ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions, +) ([]types.KLine, error) { if options.EndTime != nil { return e.srv.QueryKLinesBackward(e.sourceName, symbol, interval, *options.EndTime, 1000) } @@ -251,7 +266,9 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type return nil, errors.New("endTime or startTime can not be nil") } -func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { +func (e *Exchange) QueryTrades( + ctx context.Context, symbol string, options *types.TradeQueryOptions, +) ([]types.Trade, error) { // we don't need query trades for backtest return nil, nil } @@ -292,11 +309,15 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { return e.markets, nil } -func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []types.Deposit, err error) { +func (e *Exchange) QueryDepositHistory( + ctx context.Context, asset string, since, until time.Time, +) (allDeposits []types.Deposit, err error) { return nil, nil } -func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []types.Withdraw, err error) { +func (e *Exchange) QueryWithdrawHistory( + ctx context.Context, asset string, since, until time.Time, +) (allWithdraws []types.Withdraw, err error) { return nil, nil } @@ -321,7 +342,9 @@ func (e *Exchange) BindUserData(userDataStream types.StandardStreamEmitter) { e.matchingBooksMutex.Unlock() } -func (e *Exchange) SubscribeMarketData(startTime, endTime time.Time, requiredInterval types.Interval, extraIntervals ...types.Interval) (chan types.KLine, error) { +func (e *Exchange) SubscribeMarketData( + startTime, endTime time.Time, requiredInterval types.Interval, extraIntervals ...types.Interval, +) (chan types.KLine, error) { log.Infof("collecting backtest configurations...") loadedSymbols := map[string]struct{}{} From 2d578db12f12a9e5e3084417e4c2b052c5679fa3 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:42:00 +0800 Subject: [PATCH 036/422] bbgo: simplify marketDataStore accessor --- pkg/bbgo/session.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index dc3bd54fd9..1343eb5d85 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -556,14 +556,14 @@ func (session *ExchangeSession) Positions() map[string]*types.Position { // MarketDataStore returns the market data store of a symbol func (session *ExchangeSession) MarketDataStore(symbol string) (s *MarketDataStore, ok bool) { s, ok = session.marketDataStores[symbol] - // FIXME: the returned MarketDataStore when !ok will be empty - if !ok { - s = NewMarketDataStore(symbol) - s.BindStream(session.MarketDataStream) - session.marketDataStores[symbol] = s + if ok { return s, true } - return s, ok + + s = NewMarketDataStore(symbol) + s.BindStream(session.MarketDataStream) + session.marketDataStores[symbol] = s + return s, true } // KLine updates will be received in the order listend in intervals array From bc7f2687f886955c614c378dcdbcd8daa164b888 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:42:18 +0800 Subject: [PATCH 037/422] indicator: check valid window value for RMA --- pkg/indicator/v2/rma.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go index 247b469f9f..1aa08ccdc0 100644 --- a/pkg/indicator/v2/rma.go +++ b/pkg/indicator/v2/rma.go @@ -20,6 +20,8 @@ type RMAStream struct { } func RMA2(source types.Float64Source, window int, adjust bool) *RMAStream { + checkWindow(window) + s := &RMAStream{ Float64Series: types.NewFloat64Series(), window: window, @@ -53,7 +55,6 @@ func (s *RMAStream) Calculate(x float64) float64 { s.Slice.Push(tmp) s.previous = tmp - return tmp } @@ -62,3 +63,9 @@ func (s *RMAStream) Truncate() { s.Slice = s.Slice[MaxNumOfRMATruncateSize-1:] } } + +func checkWindow(window int) { + if window == 0 { + panic("window can not be zero") + } +} From 9a7b70d3678565022685763719f1da71214d75ff Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:42:38 +0800 Subject: [PATCH 038/422] bbgo: reformat order executor --- pkg/bbgo/order_executor_general.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/bbgo/order_executor_general.go b/pkg/bbgo/order_executor_general.go index 7b04a17c5e..c836a29102 100644 --- a/pkg/bbgo/order_executor_general.go +++ b/pkg/bbgo/order_executor_general.go @@ -68,7 +68,9 @@ type GeneralOrderExecutor struct { disableNotify bool } -func NewGeneralOrderExecutor(session *ExchangeSession, symbol, strategy, strategyInstanceID string, position *types.Position) *GeneralOrderExecutor { +func NewGeneralOrderExecutor( + session *ExchangeSession, symbol, strategy, strategyInstanceID string, position *types.Position, +) *GeneralOrderExecutor { // Always update the position fields position.Strategy = strategy position.StrategyInstanceID = strategyInstanceID @@ -114,7 +116,9 @@ func (e *GeneralOrderExecutor) startMarginAssetUpdater(ctx context.Context) { go e.marginAssetMaxBorrowableUpdater(ctx, 30*time.Minute, marginService, e.position.Market) } -func (e *GeneralOrderExecutor) updateMarginAssetMaxBorrowable(ctx context.Context, marginService types.MarginBorrowRepayService, market types.Market) { +func (e *GeneralOrderExecutor) updateMarginAssetMaxBorrowable( + ctx context.Context, marginService types.MarginBorrowRepayService, market types.Market, +) { maxBorrowable, err := marginService.QueryMarginAssetMaxBorrowable(ctx, market.BaseCurrency) if err != nil { log.WithError(err).Errorf("can not query margin base asset %s max borrowable", market.BaseCurrency) @@ -132,7 +136,9 @@ func (e *GeneralOrderExecutor) updateMarginAssetMaxBorrowable(ctx context.Contex } } -func (e *GeneralOrderExecutor) marginAssetMaxBorrowableUpdater(ctx context.Context, interval time.Duration, marginService types.MarginBorrowRepayService, market types.Market) { +func (e *GeneralOrderExecutor) marginAssetMaxBorrowableUpdater( + ctx context.Context, interval time.Duration, marginService types.MarginBorrowRepayService, market types.Market, +) { t := time.NewTicker(util.MillisecondsJitter(interval, 500)) defer t.Stop() @@ -212,7 +218,9 @@ func (e *GeneralOrderExecutor) SetLogger(logger log.FieldLogger) { e.logger = logger } -func (e *GeneralOrderExecutor) SubmitOrders(ctx context.Context, submitOrders ...types.SubmitOrder) (types.OrderSlice, error) { +func (e *GeneralOrderExecutor) SubmitOrders( + ctx context.Context, submitOrders ...types.SubmitOrder, +) (types.OrderSlice, error) { formattedOrders, err := e.session.FormatOrders(submitOrders) if err != nil { return nil, err @@ -268,7 +276,9 @@ type OpenPositionOptions struct { Tags []string `json:"-" yaml:"-"` } -func (e *GeneralOrderExecutor) reduceQuantityAndSubmitOrder(ctx context.Context, price fixedpoint.Value, submitOrder types.SubmitOrder) (types.OrderSlice, error) { +func (e *GeneralOrderExecutor) reduceQuantityAndSubmitOrder( + ctx context.Context, price fixedpoint.Value, submitOrder types.SubmitOrder, +) (types.OrderSlice, error) { var err error for i := 0; i < submitOrderRetryLimit; i++ { q := submitOrder.Quantity.Mul(fixedpoint.One.Sub(quantityReduceDelta)) @@ -309,7 +319,9 @@ func (e *GeneralOrderExecutor) reduceQuantityAndSubmitOrder(ctx context.Context, // @param options: OpenPositionOptions to control the generated SubmitOrder in a higher level way. Notice that the Price in options will be updated as the submitOrder price. // @return *types.SubmitOrder: SubmitOrder with calculated quantity and price. // @return error: Error message. -func (e *GeneralOrderExecutor) NewOrderFromOpenPosition(ctx context.Context, options *OpenPositionOptions) (*types.SubmitOrder, error) { +func (e *GeneralOrderExecutor) NewOrderFromOpenPosition( + ctx context.Context, options *OpenPositionOptions, +) (*types.SubmitOrder, error) { price := options.Price submitOrder := types.SubmitOrder{ Symbol: e.position.Symbol, @@ -408,7 +420,9 @@ func (e *GeneralOrderExecutor) NewOrderFromOpenPosition(ctx context.Context, opt // @param options: OpenPositionOptions to control the generated SubmitOrder in a higher level way. Notice that the Price in options will be updated as the submitOrder price. // @return types.OrderSlice: Created orders with information from exchange. // @return error: Error message. -func (e *GeneralOrderExecutor) OpenPosition(ctx context.Context, options OpenPositionOptions) (types.OrderSlice, error) { +func (e *GeneralOrderExecutor) OpenPosition( + ctx context.Context, options OpenPositionOptions, +) (types.OrderSlice, error) { if e.position.IsClosing() { return nil, errors.Wrap(ErrPositionAlreadyClosing, "unable to open position") } From 3b6e1e32a4c6b1f90d6cb0ba3556ffb71d9ddf47 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:42:54 +0800 Subject: [PATCH 039/422] indicator/v2/tr: use PushAndEmit instead of just EmitUpdate --- pkg/indicator/v2/tr.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/indicator/v2/tr.go b/pkg/indicator/v2/tr.go index e33044cf28..66d7cec5df 100644 --- a/pkg/indicator/v2/tr.go +++ b/pkg/indicator/v2/tr.go @@ -38,10 +38,12 @@ func (s *TRStream) calculateAndPush(high, low, cls float64) { if trueRange < hc { trueRange = hc } + if trueRange < lc { trueRange = lc } s.previousClose = cls - s.EmitUpdate(trueRange) + + s.PushAndEmit(trueRange) } From e52e53aa42af1cd975e099f4ccd3036e8d899dd6 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Sep 2023 20:43:14 +0800 Subject: [PATCH 040/422] refine atrpin strategy --- config/atrpin.yaml | 39 +++++++++++++ pkg/strategy/atrpin/strategy.go | 100 ++++++++++++++++++++++++-------- 2 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 config/atrpin.yaml diff --git a/config/atrpin.yaml b/config/atrpin.yaml new file mode 100644 index 0000000000..759ee60a6f --- /dev/null +++ b/config/atrpin.yaml @@ -0,0 +1,39 @@ +sessions: + max: + exchange: max + envVarPrefix: max + +persistence: + json: + directory: var/data + redis: + host: 127.0.0.1 + port: 6379 + db: 0 + +exchangeStrategies: +- on: max + atrpin: + symbol: BTCUSDT + interval: 5m + window: 14 + multiplier: 100.0 + amount: 1000 + +backtest: + startTime: "2018-10-01" + endTime: "2018-11-01" + symbols: + - BTCUSDT + sessions: + - max + # syncSecKLines: true + accounts: + max: + makerFeeRate: 0.0% + takerFeeRate: 0.075% + balances: + BTC: 1.0 + USDT: 10_000.0 + + diff --git a/pkg/strategy/atrpin/strategy.go b/pkg/strategy/atrpin/strategy.go index a5a462bf41..dbc5261ce9 100644 --- a/pkg/strategy/atrpin/strategy.go +++ b/pkg/strategy/atrpin/strategy.go @@ -28,7 +28,7 @@ type Strategy struct { Symbol string `json:"symbol"` Interval types.Interval `json:"interval"` - Window int `json:"slowWindow"` + Window int `json:"window"` Multiplier float64 `json:"multiplier"` bbgo.QuantityOrAmount @@ -45,6 +45,7 @@ func (s *Strategy) InstanceID() string { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) } func (s *Strategy) Defaults() error { @@ -64,12 +65,23 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) atr := session.Indicators(s.Symbol).ATR(s.Interval, s.Window) - session.UserDataStream.OnKLine(types.KLineWith(s.Symbol, s.Interval, func(k types.KLine) { + + session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, s.Interval, func(k types.KLine) { if err := s.Strategy.OrderExecutor.GracefulCancel(ctx); err != nil { log.WithError(err).Error("unable to cancel open orders...") } + account, err := session.UpdateAccount(ctx) + if err != nil { + log.WithError(err).Error("unable to update account") + return + } + + baseBalance, _ := account.Balance(s.Market.BaseCurrency) + quoteBalance, _ := account.Balance(s.Market.QuoteCurrency) + lastAtr := atr.Last(0) + log.Infof("atr: %f", lastAtr) // protection if lastAtr <= k.High.Sub(k.Low).Float64() { @@ -78,13 +90,20 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se priceRange := fixedpoint.NewFromFloat(lastAtr * s.Multiplier) + // if the atr is too small, apply the price range protection with 10% + // priceRange protection 10% + priceRange = fixedpoint.Max(priceRange, k.Close.Mul(fixedpoint.NewFromFloat(0.1))) + log.Infof("priceRange: %f", priceRange.Float64()) + ticker, err := session.Exchange.QueryTicker(ctx, s.Symbol) if err != nil { log.WithError(err).Error("unable to query ticker") return } - bidPrice := ticker.Buy.Sub(priceRange) + log.Info(ticker.String()) + + bidPrice := fixedpoint.Max(ticker.Buy.Sub(priceRange), s.Market.TickSize) askPrice := ticker.Sell.Add(priceRange) bidQuantity := s.QuantityOrAmount.CalculateQuantity(bidPrice) @@ -94,41 +113,72 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se position := s.Strategy.OrderExecutor.Position() if !position.IsDust() { + log.Infof("position: %+v", position) + side := types.SideTypeSell takerPrice := fixedpoint.Zero if position.IsShort() { side = types.SideTypeBuy - takerPrice = askPrice + takerPrice = ticker.Sell } else if position.IsLong() { side = types.SideTypeSell - takerPrice = bidPrice + takerPrice = ticker.Buy } + orderForms = append(orderForms, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimit, + Side: side, + Price: takerPrice, + Quantity: position.GetQuantity(), + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + Tag: "takeProfit", + }) + + log.Infof("SUBMIT TAKER ORDER: %+v", orderForms) + + if _, err := s.Strategy.OrderExecutor.SubmitOrders(ctx, orderForms...); err != nil { + log.WithError(err).Error("unable to submit orders") + } + + return + } + + askQuantity = s.Market.AdjustQuantityByMinNotional(askQuantity, askPrice) + if !s.Market.IsDustQuantity(askQuantity, askPrice) && askQuantity.Compare(baseBalance.Available) < 0 { + orderForms = append(orderForms, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeLimitMaker, + Quantity: askQuantity, + Price: askPrice, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + Tag: "pinOrder", + }) + } + + bidQuantity = s.Market.AdjustQuantityByMinNotional(bidQuantity, bidPrice) + if !s.Market.IsDustQuantity(bidQuantity, bidPrice) && bidQuantity.Mul(bidPrice).Compare(quoteBalance.Available) < 0 { orderForms = append(orderForms, types.SubmitOrder{ Symbol: s.Symbol, - Side: side, - Price: takerPrice, - Quantity: position.GetQuantity(), + Side: types.SideTypeBuy, + Type: types.OrderTypeLimitMaker, + Price: bidPrice, + Quantity: bidQuantity, Market: s.Market, + Tag: "pinOrder", }) } - orderForms = append(orderForms, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeSell, - Price: askPrice, - Quantity: askQuantity, - Market: s.Market, - }) - - orderForms = append(orderForms, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeBuy, - Price: bidPrice, - Quantity: bidQuantity, - Market: s.Market, - }) + if len(orderForms) == 0 { + log.Infof("no order to place") + return + } + + log.Infof("bid/ask: %f/%f", bidPrice.Float64(), askPrice.Float64()) if _, err := s.Strategy.OrderExecutor.SubmitOrders(ctx, orderForms...); err != nil { log.WithError(err).Error("unable to submit orders") @@ -137,6 +187,10 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() + + if err := s.Strategy.OrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Error("unable to cancel open orders...") + } }) return nil From 3b63858d23a9b5ac766c0d70d663905f1a8eed08 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Wed, 27 Sep 2023 11:06:41 +0800 Subject: [PATCH 041/422] handle pagenation for QueryTrade --- pkg/exchange/okex/exchange.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 337fff97d6..e7454fe768 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -514,10 +514,21 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req.EndTime(*options.EndTime) } - response, err = req. - Do(ctx) - if err != nil { - return nil, fmt.Errorf("failed to call get order histories error: %w", err) + var res []okexapi.OrderDetails + var lastOrderID = "0" + for { // pagenation should use "after" (earlier than) + res, err = req. + After(lastOrderID). + Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + response = append(response, res...) + if len(res) == defaultQueryLimit { + lastOrderID = res[defaultQueryLimit-1].OrderID + } else { + break + } } } From d4330a7a3260776d7f4b48ab289d38e88597d0b8 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 27 Sep 2023 14:25:49 +0800 Subject: [PATCH 042/422] atrpin: add minPriceRange config --- config/atrpin.yaml | 9 +++++---- pkg/strategy/atrpin/strategy.go | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/config/atrpin.yaml b/config/atrpin.yaml index 759ee60a6f..18b6c67b12 100644 --- a/config/atrpin.yaml +++ b/config/atrpin.yaml @@ -1,6 +1,6 @@ sessions: max: - exchange: max + exchange: &exchange max envVarPrefix: max persistence: @@ -12,13 +12,14 @@ persistence: db: 0 exchangeStrategies: -- on: max +- on: *exchange atrpin: symbol: BTCUSDT interval: 5m window: 14 multiplier: 100.0 - amount: 1000 + minPriceRange: 20% + amount: 100 backtest: startTime: "2018-10-01" @@ -26,7 +27,7 @@ backtest: symbols: - BTCUSDT sessions: - - max + - *exchange # syncSecKLines: true accounts: max: diff --git a/pkg/strategy/atrpin/strategy.go b/pkg/strategy/atrpin/strategy.go index dbc5261ce9..cd817fa693 100644 --- a/pkg/strategy/atrpin/strategy.go +++ b/pkg/strategy/atrpin/strategy.go @@ -27,9 +27,10 @@ type Strategy struct { Symbol string `json:"symbol"` - Interval types.Interval `json:"interval"` - Window int `json:"window"` - Multiplier float64 `json:"multiplier"` + Interval types.Interval `json:"interval"` + Window int `json:"window"` + Multiplier float64 `json:"multiplier"` + MinPriceRange fixedpoint.Value `json:"minPriceRange"` bbgo.QuantityOrAmount // bbgo.OpenPositionOptions @@ -92,7 +93,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se // if the atr is too small, apply the price range protection with 10% // priceRange protection 10% - priceRange = fixedpoint.Max(priceRange, k.Close.Mul(fixedpoint.NewFromFloat(0.1))) + priceRange = fixedpoint.Max(priceRange, k.Close.Mul(s.MinPriceRange)) log.Infof("priceRange: %f", priceRange.Float64()) ticker, err := session.Exchange.QueryTicker(ctx, s.Symbol) From 9a0535735037205764582ad896d6e266f5a5426f Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 27 Sep 2023 14:44:11 +0800 Subject: [PATCH 043/422] pkg/exchange: remove the limitation of query range due to bybit support the query --- pkg/exchange/bybit/exchange.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index e087abf118..c72523f7c0 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -437,13 +437,6 @@ ticketId in ascend. Otherwise, the result is sorted by ticketId in descend. ** StartTime and EndTime cannot exceed 180 days. ** */ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { - if options.StartTime != nil && options.EndTime != nil && options.EndTime.Sub(*options.StartTime) > halfYearDuration { - return nil, fmt.Errorf("StartTime and EndTime cannot exceed 180 days, startTime: %v, endTime: %v, diff: %v", - options.StartTime.String(), - options.EndTime.String(), - options.EndTime.Sub(*options.StartTime)/24) - } - // using v3 client, since the v5 API does not support feeCurrency. req := e.v3client.NewGetTradesRequest() req.Symbol(symbol) From 08dad1c4979c4cd8c91a155ade4782fd325de355 Mon Sep 17 00:00:00 2001 From: zenix Date: Wed, 27 Sep 2023 15:52:02 +0900 Subject: [PATCH 044/422] fix: replace json.Number with MillisecondTimestamp in types --- pkg/exchange/binance/parse.go | 13 ++++--------- pkg/exchange/binance/parse_test.go | 2 +- pkg/exchange/binance/stream.go | 8 ++------ pkg/strategy/xfunding/strategy.go | 9 ++------- 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index 03af3c3472..0685168da7 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "time" "github.com/adshao/go-binance/v2/futures" @@ -18,8 +17,8 @@ import ( ) type EventBase struct { - Event string `json:"e"` // event name - Time json.Number `json:"E"` // event time + Event string `json:"e"` // event name + Time types.MillisecondTimestamp `json:"E"` // event time } /* @@ -462,11 +461,7 @@ func (e *DepthEvent) String() (o string) { func (e *DepthEvent) OrderBook() (book types.SliceOrderBook, err error) { book.Symbol = e.Symbol - t, err := e.EventBase.Time.Int64() - if err != nil { - return book, err - } - book.Time = types.NewMillisecondTimestampFromInt(t).Time() + book.Time = e.EventBase.Time.Time() // already in descending order book.Bids = e.Bids @@ -505,7 +500,7 @@ func parseDepthEvent(val *fastjson.Value) (*DepthEvent, error) { var depth = &DepthEvent{ EventBase: EventBase{ Event: string(val.GetStringBytes("e")), - Time: json.Number(strconv.FormatInt(val.GetInt64("E"), 10)), + Time: types.NewMillisecondTimestampFromInt(val.GetInt64("E")), }, Symbol: string(val.GetStringBytes("s")), FirstUpdateID: val.GetInt64("U"), diff --git a/pkg/exchange/binance/parse_test.go b/pkg/exchange/binance/parse_test.go index 92f7228cf6..5e80326ff7 100644 --- a/pkg/exchange/binance/parse_test.go +++ b/pkg/exchange/binance/parse_test.go @@ -397,7 +397,7 @@ func TestParseOrderFuturesUpdate(t *testing.T) { assert.Equal(t, "SELL", orderTradeEvent.OrderTrade.Side) assert.Equal(t, "x-NSUYEBKMe60cf610-f5c7-49a4-9c1", orderTradeEvent.OrderTrade.ClientOrderID) assert.Equal(t, "MARKET", orderTradeEvent.OrderTrade.OrderType) - assert.Equal(t, int64(1639933384763), orderTradeEvent.Time) + assert.Equal(t, types.NewMillisecondTimestampFromInt(1639933384763), orderTradeEvent.Time) assert.Equal(t, types.MillisecondTimestamp(time.UnixMilli(1639933384755)), orderTradeEvent.OrderTrade.OrderTradeTime) assert.Equal(t, fixedpoint.MustNewFromString("0.001"), orderTradeEvent.OrderTrade.OriginalQuantity) assert.Equal(t, fixedpoint.MustNewFromString("0.001"), orderTradeEvent.OrderTrade.OrderLastFilledQuantity) diff --git a/pkg/exchange/binance/stream.go b/pkg/exchange/binance/stream.go index e2314ad0f2..ed80f09799 100644 --- a/pkg/exchange/binance/stream.go +++ b/pkg/exchange/binance/stream.go @@ -86,13 +86,9 @@ func NewStream(ex *Exchange, client *binance.Client, futuresClient *futures.Clie stream.OnDepthEvent(func(e *DepthEvent) { f, ok := stream.depthBuffers[e.Symbol] if ok { - t, err := e.EventBase.Time.Int64() - if err != nil { - log.WithError(err).Errorf("Time parsing failed: %v", e.EventBase.Time) - } - err = f.AddUpdate(types.SliceOrderBook{ + err := f.AddUpdate(types.SliceOrderBook{ Symbol: e.Symbol, - Time: types.NewMillisecondTimestampFromInt(t).Time(), + Time: e.EventBase.Time.Time(), Bids: e.Bids, Asks: e.Asks, }, e.FirstUpdateID, e.FinalUpdateID) diff --git a/pkg/strategy/xfunding/strategy.go b/pkg/strategy/xfunding/strategy.go index 02a68a2428..11f8be1e3f 100644 --- a/pkg/strategy/xfunding/strategy.go +++ b/pkg/strategy/xfunding/strategy.go @@ -527,19 +527,14 @@ func (s *Strategy) handleAccountUpdate(ctx context.Context, e *binance.AccountUp if b.Asset != s.ProfitStats.FundingFeeCurrency { continue } - t, err := e.EventBase.Time.Int64() - if err != nil { - log.WithError(err).Error("unable to parse event timestamp") - continue - } - txnTime := time.UnixMilli(t) + txnTime := e.EventBase.Time.Time() fee := FundingFee{ Asset: b.Asset, Amount: b.BalanceChange, Txn: e.Transaction, Time: txnTime, } - err = s.ProfitStats.AddFundingFee(fee) + err := s.ProfitStats.AddFundingFee(fee) if err != nil { log.WithError(err).Error("unable to add funding fee to profitStats") continue From add1c73656bf98994aea14142f4b6018b823306b Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 27 Sep 2023 14:44:25 +0800 Subject: [PATCH 045/422] pkg/exchange: support pagination --- pkg/exchange/bybit/exchange.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index c72523f7c0..e1c3545c59 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -429,27 +429,31 @@ If options.StartTime is not specified, you can only query for records in the las If you want to query for records older than 7 days, options.StartTime is required. It supports to query records up to 180 days. -If the orderId is null, fromTradeId is passed, and toTradeId is null, then the result is sorted by -ticketId in ascend. Otherwise, the result is sorted by ticketId in descend. - ** Here includes MakerRebate. If needed, let's discuss how to modify it to return in trade. ** ** StartTime and EndTime are inclusive. ** ** StartTime and EndTime cannot exceed 180 days. ** +** StartTime, EndTime, FromTradeId can be used together. ** +** If the `FromTradeId` is passed, and `ToTradeId` is null, then the result is sorted by tradeId in `ascend`. +Otherwise, the result is sorted by tradeId in `descend`. ** */ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { // using v3 client, since the v5 API does not support feeCurrency. req := e.v3client.NewGetTradesRequest() req.Symbol(symbol) - if options.StartTime != nil || options.EndTime != nil { - if options.StartTime != nil { - req.StartTime(options.StartTime.UTC()) - } - if options.EndTime != nil { - req.EndTime(options.EndTime.UTC()) - } - } else { - req.FromTradeId(strconv.FormatUint(options.LastTradeID, 10)) + // If `lastTradeId` is given and greater than 0, the query will use it as a condition and the retrieved result will be + // in `ascending` order. We can use `lastTradeId` to retrieve all the data. So we hack it to '1' if `lastTradeID` is '0'. + // If 0 is given, it will not be used as a condition and the result will be in `descending` order. The FromTradeId + // option cannot be used to retrieve more data. + req.FromTradeId(strconv.FormatUint(options.LastTradeID, 10)) + if options.LastTradeID == 0 { + req.FromTradeId("1") + } + if options.StartTime != nil { + req.StartTime(options.StartTime.UTC()) + } + if options.EndTime != nil { + req.EndTime(options.EndTime.UTC()) } limit := uint64(options.Limit) From 4b9c933df154b1c781ae06f59ce0947586bcb17a Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 29 Sep 2023 01:06:58 +0800 Subject: [PATCH 046/422] remove skew --- pkg/strategy/fixedmaker/strategy.go | 95 ++++++----------------------- 1 file changed, 19 insertions(+), 76 deletions(-) diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index fb4a7f2262..af4ed77c6e 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -10,7 +10,6 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" - indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -27,27 +26,17 @@ func init() { type Strategy struct { *common.Strategy - Environment *bbgo.Environment - StandardIndicatorSet *bbgo.StandardIndicatorSet - Market types.Market + Environment *bbgo.Environment + Market types.Market - Interval types.Interval `json:"interval"` - Symbol string `json:"symbol"` - Quantity fixedpoint.Value `json:"quantity"` - HalfSpreadRatio fixedpoint.Value `json:"halfSpreadRatio"` - OrderType types.OrderType `json:"orderType"` - DryRun bool `json:"dryRun"` - - // SkewFactor is used to calculate the skew of bid/ask price - SkewFactor fixedpoint.Value `json:"skewFactor"` - TargetWeight fixedpoint.Value `json:"targetWeight"` - - // replace halfSpreadRatio by ATR - ATRMultiplier fixedpoint.Value `json:"atrMultiplier"` - ATRWindow int `json:"atrWindow"` + Symbol string `json:"symbol"` + Interval types.Interval `json:"interval"` + Quantity fixedpoint.Value `json:"quantity"` + HalfSpread fixedpoint.Value `json:"halfSpread"` + OrderType types.OrderType `json:"orderType"` + DryRun bool `json:"dryRun"` activeOrderBook *bbgo.ActiveOrderBook - atr *indicatorv2.ATRStream } func (s *Strategy) Defaults() error { @@ -55,11 +44,6 @@ func (s *Strategy) Defaults() error { log.Infof("order type is not set, using limit maker order type") s.OrderType = types.OrderTypeLimitMaker } - - if s.ATRWindow == 0 { - log.Infof("atr window is not set, using default value 14") - s.ATRWindow = 14 - } return nil } func (s *Strategy) Initialize() error { @@ -79,21 +63,9 @@ func (s *Strategy) Validate() error { return fmt.Errorf("quantity should be positive") } - if s.HalfSpreadRatio.Float64() <= 0 { + if s.HalfSpread.Float64() <= 0 { return fmt.Errorf("halfSpreadRatio should be positive") } - - if s.SkewFactor.Float64() < 0 { - return fmt.Errorf("skewFactor should be non-negative") - } - - if s.ATRMultiplier.Float64() < 0 { - return fmt.Errorf("atrMultiplier should be non-negative") - } - - if s.ATRWindow < 0 { - return fmt.Errorf("atrWindow should be non-negative") - } return nil } @@ -108,12 +80,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.activeOrderBook.BindStream(session.UserDataStream) - s.atr = session.Indicators(s.Symbol).ATR(s.Interval, s.ATRWindow) - - session.UserDataStream.OnStart(func() { - // you can place orders here when bbgo is started, this will be called only once. - }) - s.activeOrderBook.OnFilled(func(order types.Order) { if s.activeOrderBook.NumOfOrders() == 0 { log.Infof("no active orders, replenish") @@ -122,9 +88,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) session.MarketDataStream.OnKLineClosed(func(kline types.KLine) { - log.Infof("%+v", kline) - - s.cancelOrders(ctx) + log.Infof("%s", kline.String()) s.replenish(ctx, kline.EndTime.Time()) }) @@ -133,17 +97,14 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. defer wg.Done() _ = s.OrderExecutor.GracefulCancel(ctx) }) - return nil } -func (s *Strategy) cancelOrders(ctx context.Context) { +func (s *Strategy) replenish(ctx context.Context, t time.Time) { if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { log.WithError(err).Errorf("failed to cancel orders") } -} -func (s *Strategy) replenish(ctx context.Context, t time.Time) { if s.IsHalted(t) { log.Infof("circuit break halted, not replenishing") return @@ -193,39 +154,21 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) ([]types.SubmitOrde midPrice := ticker.Buy.Add(ticker.Sell).Div(fixedpoint.NewFromFloat(2.0)) log.Infof("mid price: %+v", midPrice) - if s.ATRMultiplier.Float64() > 0 { - atr := fixedpoint.NewFromFloat(s.atr.Last(0)) - log.Infof("atr: %s", atr.String()) - s.HalfSpreadRatio = s.ATRMultiplier.Mul(atr).Div(midPrice) - log.Infof("half spread ratio: %s", s.HalfSpreadRatio.String()) - } - - // calcualte skew by the difference between base weight and target weight - baseValue := baseBalance.Total().Mul(midPrice) - baseWeight := baseValue.Div(baseValue.Add(quoteBalance.Total())) - skew := s.SkewFactor.Mul(s.HalfSpreadRatio).Mul(baseWeight.Sub(s.TargetWeight)) - - // let the skew be in the range of [-r, r] - skew = skew.Clamp(s.HalfSpreadRatio.Neg(), s.HalfSpreadRatio) - // calculate bid and ask price - // bid price = mid price * (1 - r - skew)) - bidSpreadRatio := fixedpoint.Max(s.HalfSpreadRatio.Add(skew), fixedpoint.Zero) - bidPrice := midPrice.Mul(fixedpoint.One.Sub(bidSpreadRatio)) - log.Infof("bid price: %s", bidPrice.String()) - // ask price = mid price * (1 + r - skew)) - askSrasedRatio := fixedpoint.Max(s.HalfSpreadRatio.Sub(skew), fixedpoint.Zero) - askPrice := midPrice.Mul(fixedpoint.One.Add(askSrasedRatio)) - log.Infof("ask price: %s", askPrice.String()) + // ask price = mid price * (1 + r)) + // bid price = mid price * (1 - r)) + sellPrice := midPrice.Mul(fixedpoint.One.Add(s.HalfSpread)).Round(s.Market.PricePrecision, fixedpoint.Up) + buyPrice := midPrice.Mul(fixedpoint.One.Sub(s.HalfSpread)).Round(s.Market.PricePrecision, fixedpoint.Down) + log.Infof("sell price: %s, buy price: %s", sellPrice.String(), buyPrice.String()) // check balance and generate orders - amount := s.Quantity.Mul(bidPrice) + amount := s.Quantity.Mul(buyPrice) if quoteBalance.Available.Compare(amount) > 0 { orders = append(orders, types.SubmitOrder{ Symbol: s.Symbol, Side: types.SideTypeBuy, Type: s.OrderType, - Price: bidPrice, + Price: buyPrice, Quantity: s.Quantity, }) } else { @@ -237,7 +180,7 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) ([]types.SubmitOrde Symbol: s.Symbol, Side: types.SideTypeSell, Type: s.OrderType, - Price: askPrice, + Price: sellPrice, Quantity: s.Quantity, }) } else { From c5cd6bc95e764438ad226f90a43533a5378a292d Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 29 Sep 2023 01:12:53 +0800 Subject: [PATCH 047/422] fix common.Strategy.IsHalted --- config/fixedmaker.yaml | 19 ++++--------------- pkg/strategy/common/strategy.go | 3 +++ pkg/strategy/fixedmaker/strategy.go | 4 ++-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/config/fixedmaker.yaml b/config/fixedmaker.yaml index cdd474c026..98f2130297 100644 --- a/config/fixedmaker.yaml +++ b/config/fixedmaker.yaml @@ -2,20 +2,9 @@ exchangeStrategies: - on: max fixedmaker: - interval: 5m symbol: BTCUSDT - halfSpreadRatio: 0.05% + interval: 5m + halfSpread: 0.05% quantity: 0.005 - dryRun: false - - - on: max - rebalance: - interval: 1h - quoteCurrency: USDT - targetWeights: - BTC: 50% - USDT: 50% - threshold: 2% - maxAmount: 200 # max amount to buy or sell per order - orderType: LIMIT_MAKER # LIMIT, LIMIT_MAKER or MARKET - dryRun: false + orderType: LIMIT_MAKER + dryRun: true diff --git a/pkg/strategy/common/strategy.go b/pkg/strategy/common/strategy.go index 4ad94e532a..1ed590a84a 100644 --- a/pkg/strategy/common/strategy.go +++ b/pkg/strategy/common/strategy.go @@ -90,5 +90,8 @@ func (s *Strategy) Initialize(ctx context.Context, environ *bbgo.Environment, se } func (s *Strategy) IsHalted(t time.Time) bool { + if s.circuitBreakRiskControl == nil { + return false + } return s.circuitBreakRiskControl.IsHalted(t) } diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index af4ed77c6e..28de0a9179 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -155,8 +155,8 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) ([]types.SubmitOrde log.Infof("mid price: %+v", midPrice) // calculate bid and ask price - // ask price = mid price * (1 + r)) - // bid price = mid price * (1 - r)) + // sell price = mid price * (1 + r)) + // buy price = mid price * (1 - r)) sellPrice := midPrice.Mul(fixedpoint.One.Add(s.HalfSpread)).Round(s.Market.PricePrecision, fixedpoint.Up) buyPrice := midPrice.Mul(fixedpoint.One.Sub(s.HalfSpread)).Round(s.Market.PricePrecision, fixedpoint.Down) log.Infof("sell price: %s, buy price: %s", sellPrice.String(), buyPrice.String()) From 6fd86fefda5398a22deac42ab7e3cd04397dbc0a Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Mon, 2 Oct 2023 10:49:33 +0800 Subject: [PATCH 048/422] add unit test for QueryTrade() --- pkg/exchange/okex/exchange.go | 8 ++++---- pkg/exchange/okex/query_trades_test.go | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index e7454fe768..633e4b367d 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -493,6 +493,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type limit := uint64(options.Limit) if limit > defaultQueryLimit || limit <= 0 { + limit = defaultQueryLimit log.Debugf("limit is exceeded default limit %d or zero, got: %d, Do not pass limit", defaultQueryLimit, options.Limit) } else { req.Limit(limit) @@ -514,18 +515,17 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req.EndTime(*options.EndTime) } - var res []okexapi.OrderDetails var lastOrderID = "0" for { // pagenation should use "after" (earlier than) - res, err = req. + res, err := req. After(lastOrderID). Do(ctx) if err != nil { return nil, fmt.Errorf("failed to call get order histories error: %w", err) } response = append(response, res...) - if len(res) == defaultQueryLimit { - lastOrderID = res[defaultQueryLimit-1].OrderID + if len(res) == int(limit) { + lastOrderID = res[limit-1].OrderID } else { break } diff --git a/pkg/exchange/okex/query_trades_test.go b/pkg/exchange/okex/query_trades_test.go index 5188be8d5b..10afdecd1c 100644 --- a/pkg/exchange/okex/query_trades_test.go +++ b/pkg/exchange/okex/query_trades_test.go @@ -54,4 +54,11 @@ func Test_QueryTrades(t *testing.T) { assert.Empty(t, transactionDetail) } t.Logf("transaction detail: %+v", transactionDetail) + // pagenation test + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{EndTime: &until, Limit: 1}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + assert.Less(t, 1, len(transactionDetail)) + } + t.Logf("transaction detail: %+v", transactionDetail) } From a40d4a6b81c719b74d7a08601208061914f9cbb3 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 11:43:02 +0800 Subject: [PATCH 049/422] compile and update migration package --- pkg/migrations/mysql/migration_api_test.go | 4 ++-- pkg/migrations/sqlite3/migration_api_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/migrations/mysql/migration_api_test.go b/pkg/migrations/mysql/migration_api_test.go index 9864095ce0..2684076d34 100644 --- a/pkg/migrations/mysql/migration_api_test.go +++ b/pkg/migrations/mysql/migration_api_test.go @@ -14,7 +14,7 @@ func TestGetMigrationsMap(t *testing.T) { func TestMergeMigrationsMap(t *testing.T) { MergeMigrationsMap(map[int64]*rockhopper.Migration{ - 2: {}, - 3: {}, + 2: &rockhopper.Migration{}, + 3: &rockhopper.Migration{}, }) } diff --git a/pkg/migrations/sqlite3/migration_api_test.go b/pkg/migrations/sqlite3/migration_api_test.go index d7f77c875c..d1f4fe1ab0 100644 --- a/pkg/migrations/sqlite3/migration_api_test.go +++ b/pkg/migrations/sqlite3/migration_api_test.go @@ -14,7 +14,7 @@ func TestGetMigrationsMap(t *testing.T) { func TestMergeMigrationsMap(t *testing.T) { MergeMigrationsMap(map[int64]*rockhopper.Migration{ - 2: {}, - 3: {}, + 2: &rockhopper.Migration{}, + 3: &rockhopper.Migration{}, }) } From ca15ee6e0291f9b853044fca5e07a1af57b60718 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 11:43:10 +0800 Subject: [PATCH 050/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index 0eae9c63d4..cf60746a8b 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index e14e9b3acf..973650f24c 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index 069fd6e0f8..f8c477e0b0 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 2b32168a21..634db28630 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index da17527463..77010d6d88 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 18b29ca5bf..2d0fcdfdc3 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index b971fdb1a8..a59a6bb681 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 8aac740d99..10b5a5942a 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index a0a3033964..b18f69fc6d 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index 6323327349..b73d4bc4a2 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index 3b6f14a934..684ca6e99e 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index 8f9cc089c4..fca05d90be 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 01935cf345..6d44204538 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index 06b9c1ce25..fe99b1f648 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index c241d0442d..707effa64d 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index 068140b28e..1572edb082 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index d32acd4446..e01fe74072 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 0ac4b451dc..3df96f1312 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index 56b1822adb..bb4f4d4436 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index 7ef43eee9b..dd88e0a1a9 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index ba269a82c9..c7b28f2f39 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index bcaf0e392b..578af4b2e2 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 2b4a17c2bb..8a0c602929 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index e60a0edcf8..26650fc61f 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index f92e0941f7..bd3edcbea1 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index c231842d88..916e8d7dcb 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index 3b53e86382..cdc5373c65 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index d3254c8cf2..72df8d23cd 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 1c1de0ddf2..91e741f82f 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Aug-2023 +###### Auto generated by spf13/cobra on 2-Oct-2023 From 43fd404505530a78d10d4214c3ef7cfa2b9c69d7 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 11:43:10 +0800 Subject: [PATCH 051/422] bump version to v1.52.0 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index 3a4b041fdd..df7940b6e7 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.51.1-71d86aa4-dev" +const Version = "v1.52.0-2058ce80-dev" -const VersionGitRef = "71d86aa4" +const VersionGitRef = "2058ce80" diff --git a/pkg/version/version.go b/pkg/version/version.go index ab00e5ac1e..1be5d77039 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.51.1-71d86aa4" +const Version = "v1.52.0-2058ce80" -const VersionGitRef = "71d86aa4" +const VersionGitRef = "2058ce80" From 51ac5dd5d0bafa6fd19b1f336f1e5ceda06fd9f9 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 11:43:10 +0800 Subject: [PATCH 052/422] add v1.52.0 release note --- doc/release/v1.52.0.md | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 doc/release/v1.52.0.md diff --git a/doc/release/v1.52.0.md b/doc/release/v1.52.0.md new file mode 100644 index 0000000000..f4694024cf --- /dev/null +++ b/doc/release/v1.52.0.md @@ -0,0 +1,55 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.51.1...main) + + - [#1325](https://github.com/c9s/bbgo/pull/1325): FIX: listenKeyExpired event sends string timestamp + - [#1326](https://github.com/c9s/bbgo/pull/1326): FIX: [bybit] fix bybit query trades + - [#1323](https://github.com/c9s/bbgo/pull/1323): FEATURE: add atrpin strategy + - [#1324](https://github.com/c9s/bbgo/pull/1324): FEATURE: [bybit] emit balance snapshot + - [#1318](https://github.com/c9s/bbgo/pull/1318): CHORE: add IsHalted method to common.Strategy for CircuitBreakRiskControl + - [#1313](https://github.com/c9s/bbgo/pull/1313): FIX: [grid2] only do active order update when grid is recovered + - [#1320](https://github.com/c9s/bbgo/pull/1320): FEATURE: add log fields support to the core + - [#1319](https://github.com/c9s/bbgo/pull/1319): CHORE: change websocket error to warnf + - [#1317](https://github.com/c9s/bbgo/pull/1317): FIX: Wait for all routines to close while streaming is reconnecting + - [#1315](https://github.com/c9s/bbgo/pull/1315): REFACTOR: use common strategy in fixedmaker + - [#1311](https://github.com/c9s/bbgo/pull/1311): FEATURE: emit regardless of whether there is an error or not on subscription. + - [#1314](https://github.com/c9s/bbgo/pull/1314): FEATURE: use retry query order until successful + - [#1302](https://github.com/c9s/bbgo/pull/1302): FEATURE: use quote quantity if there is QuoteQuantity in trade + - [#1307](https://github.com/c9s/bbgo/pull/1307): FEATURE: add QueryOrderTrades() for okex + - [#1310](https://github.com/c9s/bbgo/pull/1310): FIX: fix pending order update comparison + - [#1309](https://github.com/c9s/bbgo/pull/1309): FEATURE: add auth event + - [#1308](https://github.com/c9s/bbgo/pull/1308): FEATURE: [bybit] support market trade + - [#1306](https://github.com/c9s/bbgo/pull/1306): IMPROVE: Update Binance futures account api to v2 + - [#1304](https://github.com/c9s/bbgo/pull/1304): FEATURE: [bybit] support unsubscribe + - [#1301](https://github.com/c9s/bbgo/pull/1301): FEATURE: add Reconnect and Resubscribe for stream + - [#1305](https://github.com/c9s/bbgo/pull/1305): FEATURE: refactor okex to future use + - [#1303](https://github.com/c9s/bbgo/pull/1303): FEATURE: set default 30d for closed order batch query + - [#1299](https://github.com/c9s/bbgo/pull/1299): CHORE: add time to SliceOrderBook + - [#1295](https://github.com/c9s/bbgo/pull/1295): FEATURE: round down executed amount to avoid insufficient balance + - [#1297](https://github.com/c9s/bbgo/pull/1297): FIX: reset profit stats when over given duration in circuit break risk control + - [#1298](https://github.com/c9s/bbgo/pull/1298): FIX: [grid2] fix active order recover, add start process delay + - [#1300](https://github.com/c9s/bbgo/pull/1300): FIX: fix okex bookticker bug + - [#1238](https://github.com/c9s/bbgo/pull/1238): TEST: add unit test for okex exchange + - [#1293](https://github.com/c9s/bbgo/pull/1293): FIX: [bybit] quantity in buy market order + - [#1290](https://github.com/c9s/bbgo/pull/1290): FEATURE: [grid2] update local active orders after re-connected + - [#1291](https://github.com/c9s/bbgo/pull/1291): FIX: [max] Fix QuerySpotAccount method + - [#1287](https://github.com/c9s/bbgo/pull/1287): FEATURE: [bybit] add kline backtest + - [#1289](https://github.com/c9s/bbgo/pull/1289): FIX: [bybit] fix misc + - [#1285](https://github.com/c9s/bbgo/pull/1285): FEATURE: [bybit] implement ExchangeOrderQueryService interface + - [#1288](https://github.com/c9s/bbgo/pull/1288): FIX: [deposit2transfer] call QuerySpotAccount for getting the spot balance + - [#1283](https://github.com/c9s/bbgo/pull/1283): IMPROVE: profitStatsTracker, Add a parameter for window to sum up trades + - [#1284](https://github.com/c9s/bbgo/pull/1284): FIX: [deposit2transfer] apply rate limiter on checkDeposits + - [#1282](https://github.com/c9s/bbgo/pull/1282): FEATURE: [bybit] add trade info event + - [#1277](https://github.com/c9s/bbgo/pull/1277): FEATURE: [bybit] add k line api + - [#1281](https://github.com/c9s/bbgo/pull/1281): FIX: [deposit2transfer] add lastAssetDepositTimes for immediate success deposits + - [#1279](https://github.com/c9s/bbgo/pull/1279): FEATURE: [bybit] support query account/balance api + - [#1278](https://github.com/c9s/bbgo/pull/1278): FEATURE: [max] update deposit states and add more fields to deposit + - [#1275](https://github.com/c9s/bbgo/pull/1275): FEATURE: [strategy] add deposit2transfer tool + - [#1276](https://github.com/c9s/bbgo/pull/1276): REFACTOR: add order event + - [#1274](https://github.com/c9s/bbgo/pull/1274): FEATURE: [bybit] add balance snapshot event + - [#1271](https://github.com/c9s/bbgo/pull/1271): REFACTOR: apply market.GreaterThanMinimalOrderQuantity on both convert and xalign + - [#1273](https://github.com/c9s/bbgo/pull/1273): FEATURE: [bybit] add auth func in WebSocket + - [#1268](https://github.com/c9s/bbgo/pull/1268): FEATURE: [bybit] implement order book streaming + - [#1270](https://github.com/c9s/bbgo/pull/1270): FEATURE: [strategy] Add convert strategy + - [#1269](https://github.com/c9s/bbgo/pull/1269): FIX: supertrend uses strconv instead of fmt + - [#1265](https://github.com/c9s/bbgo/pull/1265): FEATURE: [bybit] implement stream ping + - [#1267](https://github.com/c9s/bbgo/pull/1267): FEATURE: add custom heart beat func to StandardStream + - [#1266](https://github.com/c9s/bbgo/pull/1266): FIX: types: exit ping worker when error is happened From 648b82ead35381aa82686929c1f86c4fcf4d9dc9 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Mon, 2 Oct 2023 18:47:05 +0800 Subject: [PATCH 053/422] use NewGetTransactionHistoryRequest for QueryTrades and use billID for pagination --- pkg/exchange/okex/convert.go | 7 ++++--- pkg/exchange/okex/exchange.go | 10 +++++----- .../okex/okexapi/get_transaction_history_request.go | 2 ++ .../get_transaction_history_request_requestgen.go | 12 +++++++++++- pkg/exchange/okex/okexapi/trade.go | 1 + pkg/exchange/okex/query_trades_test.go | 5 ----- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 193ba233e1..a83ce014ac 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -286,9 +286,10 @@ func toGlobalOrder(okexOrder *okexapi.OrderDetails) (*types.Order, error) { } func toGlobalTrade(orderDetail *okexapi.OrderDetails) (*types.Trade, error) { - tradeID, err := strconv.ParseInt(orderDetail.LastTradeID, 10, 64) + // Should use tradeId, but okex use billId to perform pagination, so use billID as tradeID instead. + billID, err := strconv.ParseInt(orderDetail.BillID, 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error parsing tradeId value: %s", orderDetail.LastTradeID) + return nil, errors.Wrapf(err, "error parsing billId value: %s", orderDetail.BillID) } orderID, err := strconv.ParseInt(orderDetail.OrderID, 10, 64) @@ -309,7 +310,7 @@ func toGlobalTrade(orderDetail *okexapi.OrderDetails) (*types.Trade, error) { } return &types.Trade{ - ID: uint64(tradeID), + ID: uint64(billID), OrderID: uint64(orderID), Exchange: types.ExchangeOKEx, Price: orderDetail.LastFilledPrice, diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 633e4b367d..1e7773760d 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -489,7 +489,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type log.Warn("!!!OKEX EXCHANGE API NOTICE!!! Okex does not support searching for trades using TradeId.") } - req := e.client.NewGetOrderHistoryRequest().InstrumentID(toLocalSymbol(symbol)) + req := e.client.NewGetTransactionHistoryRequest().InstrumentID(toLocalSymbol(symbol)) limit := uint64(options.Limit) if limit > defaultQueryLimit || limit <= 0 { @@ -515,17 +515,17 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req.EndTime(*options.EndTime) } - var lastOrderID = "0" - for { // pagenation should use "after" (earlier than) + var billID = "" // billId should be emtpy, can't be 0 + for { // pagenation should use "after" (earlier than) res, err := req. - After(lastOrderID). + After(billID). Do(ctx) if err != nil { return nil, fmt.Errorf("failed to call get order histories error: %w", err) } response = append(response, res...) if len(res) == int(limit) { - lastOrderID = res[limit-1].OrderID + billID = res[limit-1].BillID } else { break } diff --git a/pkg/exchange/okex/okexapi/get_transaction_history_request.go b/pkg/exchange/okex/okexapi/get_transaction_history_request.go index 23e02fd4aa..2b7a0ddf77 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_history_request.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request.go @@ -14,6 +14,8 @@ type GetTransactionHistoryRequest struct { instrumentID *string `param:"instId,query"` orderType *OrderType `param:"ordType,query"` orderID string `param:"ordId,query"` + billID string `param:"billId"` + // Underlying and InstrumentFamily Applicable to FUTURES/SWAP/OPTION underlying *string `param:"uly,query"` instrumentFamily *string `param:"instFamily,query"` diff --git a/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go index 00c3d71da5..f2eaf336dc 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go @@ -68,6 +68,11 @@ func (g *GetTransactionHistoryRequest) Limit(limit uint64) *GetTransactionHistor return g } +func (g *GetTransactionHistoryRequest) BillID(billID string) *GetTransactionHistoryRequest { + g.billID = billID + return g +} + // GetQueryParameters builds and checks the query parameters and returns url.Values func (g *GetTransactionHistoryRequest) GetQueryParameters() (url.Values, error) { var params = map[string]interface{}{} @@ -189,6 +194,11 @@ func (g *GetTransactionHistoryRequest) GetQueryParameters() (url.Values, error) // GetParameters builds and checks the parameters and return the result in a map object func (g *GetTransactionHistoryRequest) GetParameters() (map[string]interface{}, error) { var params = map[string]interface{}{} + // check billID field -> json key billId + billID := g.billID + + // assign parameter of billID + params["billId"] = billID return params, nil } @@ -274,7 +284,7 @@ func (g *GetTransactionHistoryRequest) GetSlugsMap() (map[string]string, error) func (g *GetTransactionHistoryRequest) Do(ctx context.Context) (OrderList, error) { - // no body params + // empty params for GET operation var params interface{} query, err := g.GetQueryParameters() if err != nil { diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index 164ae46dae..8d673ee454 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -276,6 +276,7 @@ type OrderDetails struct { LastFilledFee fixedpoint.Value `json:"fillFee"` LastFilledFeeCurrency string `json:"fillFeeCcy"` LastFilledPnl fixedpoint.Value `json:"fillPnl"` + BillID string `json:"billId"` // ExecutionType = liquidity (M = maker or T = taker) ExecutionType string `json:"execType"` diff --git a/pkg/exchange/okex/query_trades_test.go b/pkg/exchange/okex/query_trades_test.go index 10afdecd1c..6f082e3638 100644 --- a/pkg/exchange/okex/query_trades_test.go +++ b/pkg/exchange/okex/query_trades_test.go @@ -35,30 +35,25 @@ func Test_QueryTrades(t *testing.T) { if assert.NoError(t, err) { assert.NotEmpty(t, transactionDetail) } - t.Logf("transaction detail: %+v", transactionDetail) // query by trade id transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{LastTradeID: 432044402}) if assert.Error(t, err) { assert.Empty(t, transactionDetail) } - t.Logf("transaction detail: %+v", transactionDetail) // query by no time interval and no trade id transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{}) if assert.Error(t, err) { assert.Empty(t, transactionDetail) } - t.Logf("transaction detail: %+v", transactionDetail) // query by limit exceed default value transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{Limit: 150}) if assert.Error(t, err) { assert.Empty(t, transactionDetail) } - t.Logf("transaction detail: %+v", transactionDetail) // pagenation test transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{EndTime: &until, Limit: 1}) if assert.NoError(t, err) { assert.NotEmpty(t, transactionDetail) assert.Less(t, 1, len(transactionDetail)) } - t.Logf("transaction detail: %+v", transactionDetail) } From cc55d67eebed0fdd2b10f8d053d1cfc1eb6c21be Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Tue, 3 Oct 2023 12:29:30 +0800 Subject: [PATCH 054/422] use default limit if not pass AND add more unit test --- pkg/exchange/okex/exchange.go | 10 +++++----- pkg/exchange/okex/query_trades_test.go | 25 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 1e7773760d..371809c8f1 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -494,7 +494,8 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type limit := uint64(options.Limit) if limit > defaultQueryLimit || limit <= 0 { limit = defaultQueryLimit - log.Debugf("limit is exceeded default limit %d or zero, got: %d, Do not pass limit", defaultQueryLimit, options.Limit) + req.Limit(defaultQueryLimit) + log.Debugf("limit is exceeded default limit %d or zero, got: %d, use default limit", defaultQueryLimit, options.Limit) } else { req.Limit(limit) } @@ -506,7 +507,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type var err error var response []okexapi.OrderDetails if options.StartTime == nil && options.EndTime == nil { - return nil, fmt.Errorf("StartTime and EndTime are require parameter!") + return nil, fmt.Errorf("StartTime and EndTime are required parameter!") } else { // query by time interval if options.StartTime != nil { req.StartTime(*options.StartTime) @@ -524,11 +525,10 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type return nil, fmt.Errorf("failed to call get order histories error: %w", err) } response = append(response, res...) - if len(res) == int(limit) { - billID = res[limit-1].BillID - } else { + if len(res) != int(limit) { break } + billID = res[limit-1].BillID } } diff --git a/pkg/exchange/okex/query_trades_test.go b/pkg/exchange/okex/query_trades_test.go index 6f082e3638..361074934a 100644 --- a/pkg/exchange/okex/query_trades_test.go +++ b/pkg/exchange/okex/query_trades_test.go @@ -50,10 +50,33 @@ func Test_QueryTrades(t *testing.T) { if assert.Error(t, err) { assert.Empty(t, transactionDetail) } - // pagenation test + // pagenation test and test time interval : only end time transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{EndTime: &until, Limit: 1}) if assert.NoError(t, err) { assert.NotEmpty(t, transactionDetail) assert.Less(t, 1, len(transactionDetail)) } + // query by time interval: only start time + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{StartTime: &since, Limit: 100}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + // query by combination: start time, end time and after + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{StartTime: &since, EndTime: &until, Limit: 1}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } + // query by time interval: 3 months earlier with start time and end time + since = time.Now().AddDate(0, -6, 0) + until = time.Now().AddDate(0, -3, 0) + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{StartTime: &since, EndTime: &until, Limit: 100}) + if assert.NoError(t, err) { + assert.Empty(t, transactionDetail) + } + // query by time interval: 3 months earlier with start time + since = time.Now().AddDate(0, -6, 0) + transactionDetail, err = e.QueryTrades(context.Background(), queryOrder.Symbol, &types.TradeQueryOptions{StartTime: &since, Limit: 100}) + if assert.NoError(t, err) { + assert.NotEmpty(t, transactionDetail) + } } From b1c6e01e45dd0259bbeb718b805a6d925540bac0 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Tue, 3 Oct 2023 15:14:49 +0800 Subject: [PATCH 055/422] use types.StrInt64 for billID and add more comment for QueryTrades() and comment out personal unit test --- pkg/exchange/okex/convert.go | 5 +--- pkg/exchange/okex/exchange.go | 5 +++- pkg/exchange/okex/okexapi/trade.go | 2 +- pkg/exchange/okex/query_closed_orders_test.go | 26 +++++++++++-------- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index a83ce014ac..bc05ce744a 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -287,10 +287,7 @@ func toGlobalOrder(okexOrder *okexapi.OrderDetails) (*types.Order, error) { func toGlobalTrade(orderDetail *okexapi.OrderDetails) (*types.Trade, error) { // Should use tradeId, but okex use billId to perform pagination, so use billID as tradeID instead. - billID, err := strconv.ParseInt(orderDetail.BillID, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "error parsing billId value: %s", orderDetail.BillID) - } + billID := orderDetail.BillID orderID, err := strconv.ParseInt(orderDetail.OrderID, 10, 64) if err != nil { diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 371809c8f1..6122545d53 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -479,6 +479,9 @@ QueryTrades can query trades in last 3 months, there are no time interval limita OKEX do not provide api to query by tradeID, So use /api/v5/trade/orders-history-archive as its official site do. If you want to query trades by time range, please just pass start_time and end_time. Because it gets the correct response even when you pass all parameters with the right time interval and invalid LastTradeID, like 0. +No matter how you pass parameter, QueryTrades return descending order. +If you query time period 3 months earlier with start time and end time, will return [] empty slice +But If you query time period 3 months earlier JUST with start time, will return like start with 3 months ago. */ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { if symbol == "" { @@ -528,7 +531,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type if len(res) != int(limit) { break } - billID = res[limit-1].BillID + billID = strconv.Itoa(int(res[limit-1].BillID)) } } diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index 8d673ee454..f42553c517 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -276,7 +276,7 @@ type OrderDetails struct { LastFilledFee fixedpoint.Value `json:"fillFee"` LastFilledFeeCurrency string `json:"fillFeeCcy"` LastFilledPnl fixedpoint.Value `json:"fillPnl"` - BillID string `json:"billId"` + BillID types.StrInt64 `json:"billId"` // ExecutionType = liquidity (M = maker or T = taker) ExecutionType string `json:"execType"` diff --git a/pkg/exchange/okex/query_closed_orders_test.go b/pkg/exchange/okex/query_closed_orders_test.go index 9b77e76658..67bdae2910 100644 --- a/pkg/exchange/okex/query_closed_orders_test.go +++ b/pkg/exchange/okex/query_closed_orders_test.go @@ -24,13 +24,15 @@ func Test_QueryClosedOrders(t *testing.T) { } // test by order id as a cursor - closedOrder, err := e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Time{}, time.Time{}, 609869603774656544) - if assert.NoError(t, err) { - assert.NotEmpty(t, closedOrder) - } - t.Logf("closed order detail: %+v", closedOrder) + /* + closedOrder, err := e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Time{}, time.Time{}, 609869603774656544) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + */ // test by time interval - closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Now().Add(-90*24*time.Hour), time.Now(), 0) + closedOrder, err := e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Now().Add(-90*24*time.Hour), time.Now(), 0) if assert.NoError(t, err) { assert.NotEmpty(t, closedOrder) } @@ -54,9 +56,11 @@ func Test_QueryClosedOrders(t *testing.T) { } t.Logf("closed order detail: %+v", closedOrder) // test by time interval and order id together - closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Unix(1694154903, 999), time.Now(), 609869603774656544) - if assert.NoError(t, err) { - assert.NotEmpty(t, closedOrder) - } - t.Logf("closed order detail: %+v", closedOrder) + /* + closedOrder, err = e.QueryClosedOrders(context.Background(), string(queryOrder.Symbol), time.Unix(1694154903, 999), time.Now(), 609869603774656544) + if assert.NoError(t, err) { + assert.NotEmpty(t, closedOrder) + } + t.Logf("closed order detail: %+v", closedOrder) + */ } From d200232c13c0efb9111c27e7161e5b90458da820 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Mon, 2 Oct 2023 12:55:30 +0800 Subject: [PATCH 056/422] add supported interval for okex --- pkg/exchange/okex/convert.go | 22 ++++++-- pkg/exchange/okex/exchange.go | 14 ++++- pkg/exchange/okex/query_kline_test.go | 78 +++++++++++++++++++++++++++ pkg/exchange/okex/types.go | 40 ++++++++++++++ 4 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 pkg/exchange/okex/query_kline_test.go create mode 100644 pkg/exchange/okex/types.go diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index bc05ce744a..afe8d545a9 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -206,11 +206,23 @@ func toGlobalOrderType(orderType okexapi.OrderType) (types.OrderType, error) { return "", fmt.Errorf("unknown or unsupported okex order type: %s", orderType) } -func toLocalInterval(src string) string { - var re = regexp.MustCompile(`\d+[hdw]`) - return re.ReplaceAllStringFunc(src, func(w string) string { - return strings.ToUpper(w) - }) +func toLocalInterval(interval types.Interval) (string, error) { + if _, ok := SupportedIntervals[interval]; !ok { + return "", fmt.Errorf("interval %s is not supported", interval) + } + + switch i := interval.String(); { + case strings.HasSuffix(i, "m"): + return i, nil + case strings.HasSuffix(i, "mo"): + return "1M", nil + default: + hdwRegex := regexp.MustCompile("\\d+[hdw]$") + if hdwRegex.Match([]byte(i)) { + return strings.ToUpper(i), nil + } + } + return "", fmt.Errorf("interval %s is not supported", interval) } func toGlobalSide(side okexapi.SideType) (s types.SideType) { diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 6122545d53..a631629080 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -316,7 +316,10 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type return nil, err } - intervalParam := toLocalInterval(interval.String()) + intervalParam, err := toLocalInterval(interval) + if err != nil { + return nil, fmt.Errorf("fail to get interval: %w", err) + } req := e.client.NewCandlesticksRequest(toLocalSymbol(symbol)) req.Bar(intervalParam) @@ -541,3 +544,12 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type } return trades, nil } + +func (e *Exchange) SupportedInterval() map[types.Interval]int { + return SupportedIntervals +} + +func (e *Exchange) IsSupportedInterval(interval types.Interval) bool { + _, ok := SupportedIntervals[interval] + return ok +} diff --git a/pkg/exchange/okex/query_kline_test.go b/pkg/exchange/okex/query_kline_test.go new file mode 100644 index 0000000000..0a06c07abf --- /dev/null +++ b/pkg/exchange/okex/query_kline_test.go @@ -0,0 +1,78 @@ +package okex + +import ( + "context" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/testutil" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func Test_QueryKlines(t *testing.T) { + key, secret, passphrase, ok := testutil.IntegrationTestWithPassphraseConfigured(t, "OKEX") + if !ok { + t.Skip("Please configure all credentials about OKEX") + } + + e := New(key, secret, passphrase) + + queryOrder := types.OrderQuery{ + Symbol: "BTC-USDT", + } + + now := time.Now() + // test supported interval - minute + klineDetail, err := e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1m, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + } + // test supported interval - hour + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1h, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + } + // test supported interval - day + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1d, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + assert.NotEmpty(t, klineDetail[0].Exchange) + assert.NotEmpty(t, klineDetail[0].Symbol) + assert.NotEmpty(t, klineDetail[0].StartTime) + assert.NotEmpty(t, klineDetail[0].EndTime) + assert.NotEmpty(t, klineDetail[0].Interval) + assert.NotEmpty(t, klineDetail[0].Open) + assert.NotEmpty(t, klineDetail[0].Close) + assert.NotEmpty(t, klineDetail[0].High) + assert.NotEmpty(t, klineDetail[0].Low) + assert.NotEmpty(t, klineDetail[0].Volume) + } + // test supported interval - week + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1w, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + } + // test supported interval - month + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1mo, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + } + // test not supported interval + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval("2m"), types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.Error(t, err) { + assert.Empty(t, klineDetail) + } +} diff --git a/pkg/exchange/okex/types.go b/pkg/exchange/okex/types.go new file mode 100644 index 0000000000..f09b78eb36 --- /dev/null +++ b/pkg/exchange/okex/types.go @@ -0,0 +1,40 @@ +package okex + +import "github.com/c9s/bbgo/pkg/types" + +var ( + // below are supported UTC timezone interval for okex + SupportedIntervals = map[types.Interval]int{ + types.Interval1m: 1 * 60, + types.Interval3m: 3 * 60, + types.Interval5m: 5 * 60, + types.Interval15m: 15 * 60, + types.Interval30m: 30 * 60, + types.Interval1h: 60 * 60, + types.Interval2h: 60 * 60 * 2, + types.Interval4h: 60 * 60 * 4, + types.Interval6h: 60 * 60 * 6, + types.Interval12h: 60 * 60 * 12, + types.Interval1d: 60 * 60 * 24, + types.Interval3d: 60 * 60 * 24 * 3, + types.Interval1w: 60 * 60 * 24 * 7, + types.Interval1mo: 60 * 60 * 24 * 30, + } + + ToGlobalInterval = map[string]types.Interval{ + "1m": types.Interval1m, + "3m": types.Interval3m, + "5m": types.Interval5m, + "15m": types.Interval15m, + "30m": types.Interval30m, + "1H": types.Interval1h, + "2H": types.Interval2h, + "4H": types.Interval4h, + "6H": types.Interval6h, + "12H": types.Interval12h, + "1D": types.Interval1d, + "3D": types.Interval3d, + "1W": types.Interval1w, + "1M": types.Interval1mo, + } +) From a83335817e43b718cd5414f4989a850dcb5de509 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Tue, 3 Oct 2023 16:59:46 +0800 Subject: [PATCH 057/422] use interval [1m/3m/5m/15m/30m/1H/2H/4H] and [/6Hutc/12Hutc/1Dutc/2Dutc/3Dutc/1Wutc/1Mutc] and add unit test --- pkg/exchange/okex/convert.go | 19 +++++++++++++++--- pkg/exchange/okex/query_kline_test.go | 9 ++++++++- pkg/exchange/okex/types.go | 28 +++++++++++++-------------- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index afe8d545a9..16e751bd98 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -215,11 +215,24 @@ func toLocalInterval(interval types.Interval) (string, error) { case strings.HasSuffix(i, "m"): return i, nil case strings.HasSuffix(i, "mo"): - return "1M", nil + return "1Mutc", nil default: - hdwRegex := regexp.MustCompile("\\d+[hdw]$") + hdwRegex := regexp.MustCompile("\\d+[dw]$") if hdwRegex.Match([]byte(i)) { - return strings.ToUpper(i), nil + return strings.ToUpper(i) + "utc", nil + } + hdwRegex = regexp.MustCompile("(\\d+)[h]$") + if fs := hdwRegex.FindStringSubmatch(i); len(fs) > 0 { + digits, err := strconv.ParseInt(string(fs[1]), 10, 64) + if err != nil { + return "", fmt.Errorf("interval %s is not supported", interval) + } + if digits >= 6 { + return strings.ToUpper(i) + "utc", nil + } else { + return strings.ToUpper(i), nil + } + } } return "", fmt.Errorf("interval %s is not supported", interval) diff --git a/pkg/exchange/okex/query_kline_test.go b/pkg/exchange/okex/query_kline_test.go index 0a06c07abf..c3e703b451 100644 --- a/pkg/exchange/okex/query_kline_test.go +++ b/pkg/exchange/okex/query_kline_test.go @@ -30,13 +30,20 @@ func Test_QueryKlines(t *testing.T) { if assert.NoError(t, err) { assert.NotEmpty(t, klineDetail) } - // test supported interval - hour + // test supported interval - hour - 1 hour klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1h, types.KLineQueryOptions{ Limit: 50, EndTime: &now}) if assert.NoError(t, err) { assert.NotEmpty(t, klineDetail) } + // test supported interval - hour - 6 hour to test UTC time + klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval6h, types.KLineQueryOptions{ + Limit: 50, + EndTime: &now}) + if assert.NoError(t, err) { + assert.NotEmpty(t, klineDetail) + } // test supported interval - day klineDetail, err = e.QueryKLines(context.Background(), queryOrder.Symbol, types.Interval1d, types.KLineQueryOptions{ Limit: 50, diff --git a/pkg/exchange/okex/types.go b/pkg/exchange/okex/types.go index f09b78eb36..d4d0f00293 100644 --- a/pkg/exchange/okex/types.go +++ b/pkg/exchange/okex/types.go @@ -22,19 +22,19 @@ var ( } ToGlobalInterval = map[string]types.Interval{ - "1m": types.Interval1m, - "3m": types.Interval3m, - "5m": types.Interval5m, - "15m": types.Interval15m, - "30m": types.Interval30m, - "1H": types.Interval1h, - "2H": types.Interval2h, - "4H": types.Interval4h, - "6H": types.Interval6h, - "12H": types.Interval12h, - "1D": types.Interval1d, - "3D": types.Interval3d, - "1W": types.Interval1w, - "1M": types.Interval1mo, + "1m": types.Interval1m, + "3m": types.Interval3m, + "5m": types.Interval5m, + "15m": types.Interval15m, + "30m": types.Interval30m, + "1H": types.Interval1h, + "2H": types.Interval2h, + "4H": types.Interval4h, + "6Hutc": types.Interval6h, + "12Hutc": types.Interval12h, + "1Dutc": types.Interval1d, + "3Dutc": types.Interval3d, + "1Wutc": types.Interval1w, + "1Mutc": types.Interval1mo, } ) From 0b5ce231ffb129b87d38a9c6264eca581b8ad945 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Wed, 4 Oct 2023 12:36:17 +0800 Subject: [PATCH 058/422] fix lint and rename i with in --- pkg/exchange/okex/convert.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 16e751bd98..1d1713a84f 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -211,26 +211,26 @@ func toLocalInterval(interval types.Interval) (string, error) { return "", fmt.Errorf("interval %s is not supported", interval) } - switch i := interval.String(); { - case strings.HasSuffix(i, "m"): - return i, nil - case strings.HasSuffix(i, "mo"): + switch in := interval.String(); { + case strings.HasSuffix(in, "m"): + return in, nil + case strings.HasSuffix(in, "mo"): return "1Mutc", nil default: - hdwRegex := regexp.MustCompile("\\d+[dw]$") - if hdwRegex.Match([]byte(i)) { - return strings.ToUpper(i) + "utc", nil + hdwRegex := regexp.MustCompile(`\d+[dw]$`) + if hdwRegex.Match([]byte(in)) { + return strings.ToUpper(in) + "utc", nil } - hdwRegex = regexp.MustCompile("(\\d+)[h]$") - if fs := hdwRegex.FindStringSubmatch(i); len(fs) > 0 { + hdwRegex = regexp.MustCompile(`(\d+)[h]$`) + if fs := hdwRegex.FindStringSubmatch(in); len(fs) > 0 { digits, err := strconv.ParseInt(string(fs[1]), 10, 64) if err != nil { return "", fmt.Errorf("interval %s is not supported", interval) } if digits >= 6 { - return strings.ToUpper(i) + "utc", nil + return strings.ToUpper(in) + "utc", nil } else { - return strings.ToUpper(i), nil + return strings.ToUpper(in), nil } } From 3b793b79b69225721b0475e9d748d27b8e3046d9 Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Wed, 4 Oct 2023 14:23:13 +0800 Subject: [PATCH 059/422] turn ToGlobalInterval to ToLocalInterval, use Map to turn to local interval --- pkg/exchange/okex/convert.go | 25 ++----------------------- pkg/exchange/okex/types.go | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 1d1713a84f..36f599c216 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -2,7 +2,6 @@ package okex import ( "fmt" - "regexp" "strconv" "strings" @@ -211,30 +210,10 @@ func toLocalInterval(interval types.Interval) (string, error) { return "", fmt.Errorf("interval %s is not supported", interval) } - switch in := interval.String(); { - case strings.HasSuffix(in, "m"): + if in, ok := ToLocalInterval[interval]; ok { return in, nil - case strings.HasSuffix(in, "mo"): - return "1Mutc", nil - default: - hdwRegex := regexp.MustCompile(`\d+[dw]$`) - if hdwRegex.Match([]byte(in)) { - return strings.ToUpper(in) + "utc", nil - } - hdwRegex = regexp.MustCompile(`(\d+)[h]$`) - if fs := hdwRegex.FindStringSubmatch(in); len(fs) > 0 { - digits, err := strconv.ParseInt(string(fs[1]), 10, 64) - if err != nil { - return "", fmt.Errorf("interval %s is not supported", interval) - } - if digits >= 6 { - return strings.ToUpper(in) + "utc", nil - } else { - return strings.ToUpper(in), nil - } - - } } + return "", fmt.Errorf("interval %s is not supported", interval) } diff --git a/pkg/exchange/okex/types.go b/pkg/exchange/okex/types.go index d4d0f00293..0217c1ee81 100644 --- a/pkg/exchange/okex/types.go +++ b/pkg/exchange/okex/types.go @@ -21,20 +21,20 @@ var ( types.Interval1mo: 60 * 60 * 24 * 30, } - ToGlobalInterval = map[string]types.Interval{ - "1m": types.Interval1m, - "3m": types.Interval3m, - "5m": types.Interval5m, - "15m": types.Interval15m, - "30m": types.Interval30m, - "1H": types.Interval1h, - "2H": types.Interval2h, - "4H": types.Interval4h, - "6Hutc": types.Interval6h, - "12Hutc": types.Interval12h, - "1Dutc": types.Interval1d, - "3Dutc": types.Interval3d, - "1Wutc": types.Interval1w, - "1Mutc": types.Interval1mo, + ToLocalInterval = map[types.Interval]string{ + types.Interval1m: "1m", + types.Interval3m: "3m", + types.Interval5m: "5m", + types.Interval15m: "15m", + types.Interval30m: "30m", + types.Interval1h: "1H", + types.Interval2h: "2H", + types.Interval4h: "4H", + types.Interval6h: "6Hutc", + types.Interval12h: "12Hutc", + types.Interval1d: "1Dutc", + types.Interval3d: "3Dutc", + types.Interval1w: "1Wutc", + types.Interval1mo: "1Mutc", } ) From 4700e754a8aa0ba29992e4ed8d67857bdf2c2c85 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 4 Oct 2023 15:17:22 +0800 Subject: [PATCH 060/422] maxapi: change default http transport settings --- pkg/exchange/max/maxapi/restapi.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/max/maxapi/restapi.go b/pkg/exchange/max/maxapi/restapi.go index 1dcdc4039e..598c6f5c5e 100644 --- a/pkg/exchange/max/maxapi/restapi.go +++ b/pkg/exchange/max/maxapi/restapi.go @@ -44,7 +44,7 @@ const ( var httpTransportMaxIdleConnsPerHost = http.DefaultMaxIdleConnsPerHost var httpTransportMaxIdleConns = 100 -var httpTransportIdleConnTimeout = 90 * time.Second +var httpTransportIdleConnTimeout = 85 * time.Second var disableUserAgentHeader = false func init() { @@ -87,7 +87,7 @@ var nonceOnce sync.Once var httpTransport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ - Timeout: 10 * time.Second, + Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, @@ -204,12 +204,16 @@ func (c *RestClient) getNonce() int64 { return (seconds+offset)*1000 - 1 + int64(math.Mod(float64(rc), 1000.0)) } -func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, m string, refURL string, params url.Values, payload interface{}) (*http.Request, error) { +func (c *RestClient) NewAuthenticatedRequest( + ctx context.Context, m string, refURL string, params url.Values, payload interface{}, +) (*http.Request, error) { return c.newAuthenticatedRequest(ctx, m, refURL, params, payload, nil) } // newAuthenticatedRequest creates new http request for authenticated routes. -func (c *RestClient) newAuthenticatedRequest(ctx context.Context, m string, refURL string, params url.Values, data interface{}, rel *url.URL) (*http.Request, error) { +func (c *RestClient) newAuthenticatedRequest( + ctx context.Context, m string, refURL string, params url.Values, data interface{}, rel *url.URL, +) (*http.Request, error) { if len(c.APIKey) == 0 { return nil, errors.New("empty api key") } From 2309bbdee82c9df553b2728de35d09a012339e9a Mon Sep 17 00:00:00 2001 From: "Alan.sung" Date: Wed, 4 Oct 2023 16:24:32 +0800 Subject: [PATCH 061/422] print local interval in error message --- pkg/exchange/okex/convert.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 36f599c216..742ad56f3a 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -210,11 +210,12 @@ func toLocalInterval(interval types.Interval) (string, error) { return "", fmt.Errorf("interval %s is not supported", interval) } - if in, ok := ToLocalInterval[interval]; ok { - return in, nil + in, ok := ToLocalInterval[interval] + if !ok { + return "", fmt.Errorf("interval %s is not supported, got local interval %s", interval, in) } - return "", fmt.Errorf("interval %s is not supported", interval) + return in, nil } func toGlobalSide(side okexapi.SideType) (s types.SideType) { From 78ea940569ba11b8cdcf8b85822f13c08d5f5493 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 17:22:03 +0800 Subject: [PATCH 062/422] max: support private channel setter --- pkg/bbgo/session.go | 15 +++++++++++++-- pkg/exchange/max/stream.go | 19 ++++++++++++++++--- pkg/types/stream.go | 4 ++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 1343eb5d85..80d24d5c6e 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -51,7 +51,13 @@ type ExchangeSession struct { TakerFeeRate fixedpoint.Value `json:"takerFeeRate" yaml:"takerFeeRate"` ModifyOrderAmountForFee bool `json:"modifyOrderAmountForFee" yaml:"modifyOrderAmountForFee"` - PublicOnly bool `json:"publicOnly,omitempty" yaml:"publicOnly"` + // PublicOnly is used for setting the session to public only (without authentication, no private user data) + PublicOnly bool `json:"publicOnly,omitempty" yaml:"publicOnly"` + + // PrivateChannels is used for filtering the private user data channel, .e.g, orders, trades, balances.. etc + // This option is exchange specific + PrivateChannels []string `json:"privateChannels,omitempty" yaml:"privateChannels,omitempty"` + Margin bool `json:"margin,omitempty" yaml:"margin"` IsolatedMargin bool `json:"isolatedMargin,omitempty" yaml:"isolatedMargin,omitempty"` IsolatedMarginSymbol string `json:"isolatedMarginSymbol,omitempty" yaml:"isolatedMarginSymbol,omitempty"` @@ -237,8 +243,13 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) // query and initialize the balances if !session.PublicOnly { - logger.Infof("querying account balances...") + if len(session.PrivateChannels) > 0 { + if setter, ok := session.UserDataStream.(types.PrivateChannelSetter); ok { + setter.SetPrivateChannels(session.PrivateChannels) + } + } + logger.Infof("querying account balances...") account, err := session.Exchange.QueryAccount(ctx) if err != nil { return err diff --git a/pkg/exchange/max/stream.go b/pkg/exchange/max/stream.go index bd02663b12..72bece012d 100644 --- a/pkg/exchange/max/stream.go +++ b/pkg/exchange/max/stream.go @@ -5,8 +5,8 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "fmt" "os" + "strconv" "time" "github.com/google/uuid" @@ -22,6 +22,8 @@ type Stream struct { key, secret string + privateChannels []string + authEventCallbacks []func(e max.AuthEvent) bookEventCallbacks []func(e max.BookEvent) tradeEventCallbacks []func(e max.PublicTradeEvent) @@ -55,6 +57,7 @@ func NewStream(key, secret string) *Stream { log.Infof("max websocket connection authenticated: %+v", e) stream.EmitAuth() }) + stream.OnKLineEvent(stream.handleKLineEvent) stream.OnOrderSnapshotEvent(stream.handleOrderSnapshotEvent) stream.OnOrderUpdateEvent(stream.handleOrderUpdateEvent) @@ -73,6 +76,10 @@ func (s *Stream) getEndpoint(ctx context.Context) (string, error) { return url, nil } +func (s *Stream) SetPrivateChannels(channels []string) { + s.privateChannels = channels +} + func (s *Stream) handleConnect() { if s.PublicOnly { cmd := &max.WebsocketCommand{ @@ -109,7 +116,11 @@ func (s *Stream) handleConnect() { } else { var filters []string - if s.MarginSettings.IsMargin { + + if len(s.privateChannels) > 0 { + // TODO: maybe check the valid private channels + filters = s.privateChannels + } else if s.MarginSettings.IsMargin { filters = []string{ "mwallet_order", "mwallet_trade", @@ -119,6 +130,8 @@ func (s *Stream) handleConnect() { } } + log.Debugf("user data websocket filters: %v", filters) + nonce := time.Now().UnixNano() / int64(time.Millisecond) auth := &max.AuthMessage{ // pragma: allowlist nextline secret @@ -126,7 +139,7 @@ func (s *Stream) handleConnect() { // pragma: allowlist nextline secret APIKey: s.key, Nonce: nonce, - Signature: signPayload(fmt.Sprintf("%d", nonce), s.secret), + Signature: signPayload(strconv.FormatInt(nonce, 10), s.secret), ID: uuid.New().String(), Filters: filters, } diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 73bd448567..51337f2602 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -42,6 +42,10 @@ type Stream interface { Close() error } +type PrivateChannelSetter interface { + SetPrivateChannels(channels []string) +} + type Unsubscriber interface { // Unsubscribe unsubscribes the all subscriptions. Unsubscribe() From 378425a3aa22d72acecb2ee3620e4bf7f2a9bfa6 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 2 Oct 2023 17:35:10 +0800 Subject: [PATCH 063/422] bbgo: add balance logger support --- pkg/bbgo/config.go | 1 + pkg/bbgo/session.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index 6abcff37be..4cbc230489 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -94,6 +94,7 @@ type NotificationConfig struct { type LoggingConfig struct { Trade bool `json:"trade,omitempty"` Order bool `json:"order,omitempty"` + Balance bool `json:"balance,omitempty"` FilledOrderOnly bool `json:"filledOrder,omitempty"` Fields map[string]interface{} `json:"fields,omitempty"` } diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 80d24d5c6e..3d2cc82e9c 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -287,6 +287,15 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) } if environ.loggingConfig != nil { + if environ.loggingConfig.Balance { + session.UserDataStream.OnBalanceSnapshot(func(balances types.BalanceMap) { + logger.Info(balances.String()) + }) + session.UserDataStream.OnBalanceUpdate(func(balances types.BalanceMap) { + logger.Info(balances.String()) + }) + } + if environ.loggingConfig.Trade { session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { logger.Info(trade.String()) From 590e1648ebb6090455e039379b4869655ebb8d2c Mon Sep 17 00:00:00 2001 From: zenix Date: Thu, 5 Oct 2023 16:16:27 +0900 Subject: [PATCH 064/422] fix: use MillisecondTimestamp instead --- pkg/exchange/binance/parse.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/exchange/binance/parse.go b/pkg/exchange/binance/parse.go index 0d2f2f431e..ac3956503d 100644 --- a/pkg/exchange/binance/parse.go +++ b/pkg/exchange/binance/parse.go @@ -534,17 +534,17 @@ func parseDepthEvent(val *fastjson.Value) (*DepthEvent, error) { } type ForceOrderEventInner struct { - Symbol string `json:"s"` - TradeTime int64 `json:"T"` - Side string `json:"S"` - OrderType string `json:"o"` - TimeInForce string `json:"f"` - Quantity fixedpoint.Value `json:"q"` - Price fixedpoint.Value `json:"p"` - AveragePrice fixedpoint.Value `json:"ap"` - OrderStatus string `json:"X"` - LastFilledQuantity fixedpoint.Value `json:"l"` - LastFilledAccQuantity fixedpoint.Value `json:"z"` + Symbol string `json:"s"` + TradeTime types.MillisecondTimestamp `json:"T"` + Side string `json:"S"` + OrderType string `json:"o"` + TimeInForce string `json:"f"` + Quantity fixedpoint.Value `json:"q"` + Price fixedpoint.Value `json:"p"` + AveragePrice fixedpoint.Value `json:"ap"` + OrderStatus string `json:"X"` + LastFilledQuantity fixedpoint.Value `json:"l"` + LastFilledAccQuantity fixedpoint.Value `json:"z"` } type ForceOrderEvent struct { @@ -554,7 +554,6 @@ type ForceOrderEvent struct { func (e *ForceOrderEvent) LiquidationInfo() types.LiquidationInfo { o := e.Order - tt := time.Unix(0, o.TradeTime*int64(time.Millisecond)) return types.LiquidationInfo{ Symbol: o.Symbol, Side: types.SideType(o.Side), @@ -564,7 +563,7 @@ func (e *ForceOrderEvent) LiquidationInfo() types.LiquidationInfo { Price: o.Price, AveragePrice: o.AveragePrice, OrderStatus: types.OrderStatus(o.OrderStatus), - TradeTime: types.Time(tt), + TradeTime: types.Time(o.TradeTime), } } From a40488b0a31e252aebe980a10a563ffeebc0c118 Mon Sep 17 00:00:00 2001 From: narumi Date: Thu, 5 Oct 2023 14:36:08 +0800 Subject: [PATCH 065/422] add xfixedmaker strategy --- config/xfixedmaker.yaml | 24 ++ pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/fixedmaker/strategy.go | 45 ++-- pkg/strategy/xfixedmaker/order_price_risk.go | 34 +++ .../xfixedmaker/order_price_risk_test.go | 63 +++++ pkg/strategy/xfixedmaker/strategy.go | 250 ++++++++++++++++++ 6 files changed, 402 insertions(+), 15 deletions(-) create mode 100644 config/xfixedmaker.yaml create mode 100644 pkg/strategy/xfixedmaker/order_price_risk.go create mode 100644 pkg/strategy/xfixedmaker/order_price_risk_test.go create mode 100644 pkg/strategy/xfixedmaker/strategy.go diff --git a/config/xfixedmaker.yaml b/config/xfixedmaker.yaml new file mode 100644 index 0000000000..c6c6563435 --- /dev/null +++ b/config/xfixedmaker.yaml @@ -0,0 +1,24 @@ +--- +sessions: + max: + exchange: max + envVarPrefix: max + binance: + exchange: binance + envVarPrefix: binance + +crossExchangeStrategies: + - xfixedmaker: + tradingExchange: max + symbol: BTCUSDT + interval: 5m + halfSpread: 0.05% + quantity: 0.005 + orderType: LIMIT_MAKER + dryRun: true + + referenceExchange: binance + referencePriceEMA: + interval: 1m + window: 14 + orderPriceLossThreshold: -10 diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 05b652a25c..571eb53d4f 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -43,6 +43,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/wall" _ "github.com/c9s/bbgo/pkg/strategy/xalign" _ "github.com/c9s/bbgo/pkg/strategy/xbalance" + _ "github.com/c9s/bbgo/pkg/strategy/xfixedmaker" _ "github.com/c9s/bbgo/pkg/strategy/xfunding" _ "github.com/c9s/bbgo/pkg/strategy/xgap" _ "github.com/c9s/bbgo/pkg/strategy/xmaker" diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 28de0a9179..ccee7c0900 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "sync" - "time" "github.com/sirupsen/logrus" @@ -71,6 +70,10 @@ func (s *Strategy) Validate() error { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) + + if !s.CircuitBreakLossThreshold.IsZero() { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.CircuitBreakEMA.Interval}) + } } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { @@ -81,15 +84,29 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.activeOrderBook.BindStream(session.UserDataStream) s.activeOrderBook.OnFilled(func(order types.Order) { + if s.IsHalted(order.UpdateTime.Time()) { + log.Infof("circuit break halted") + return + } + if s.activeOrderBook.NumOfOrders() == 0 { - log.Infof("no active orders, replenish") - s.replenish(ctx, order.UpdateTime.Time()) + log.Infof("no active orders, placing orders...") + s.placeOrders(ctx) } }) session.MarketDataStream.OnKLineClosed(func(kline types.KLine) { log.Infof("%s", kline.String()) - s.replenish(ctx, kline.EndTime.Time()) + + if s.IsHalted(kline.EndTime.Time()) { + log.Infof("circuit break halted") + return + } + + if kline.Interval == s.Interval { + s.cancelOrders(ctx) + s.placeOrders(ctx) + } }) // the shutdown handler, you can cancel all orders @@ -97,32 +114,30 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. defer wg.Done() _ = s.OrderExecutor.GracefulCancel(ctx) }) + return nil } -func (s *Strategy) replenish(ctx context.Context, t time.Time) { +func (s *Strategy) cancelOrders(ctx context.Context) { if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { log.WithError(err).Errorf("failed to cancel orders") } +} - if s.IsHalted(t) { - log.Infof("circuit break halted, not replenishing") - return - } - - submitOrders, err := s.generateSubmitOrders(ctx) +func (s *Strategy) placeOrders(ctx context.Context) { + orders, err := s.generateOrders(ctx) if err != nil { - log.WithError(err).Error("failed to generate submit orders") + log.WithError(err).Error("failed to generate orders") return } - log.Infof("submit orders: %+v", submitOrders) + log.Infof("orders: %+v", orders) if s.DryRun { log.Infof("dry run, not submitting orders") return } - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, submitOrders...) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orders...) if err != nil { log.WithError(err).Error("failed to submit orders") return @@ -132,7 +147,7 @@ func (s *Strategy) replenish(ctx context.Context, t time.Time) { s.activeOrderBook.Add(createdOrders...) } -func (s *Strategy) generateSubmitOrders(ctx context.Context) ([]types.SubmitOrder, error) { +func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, error) { orders := []types.SubmitOrder{} baseBalance, ok := s.Session.GetAccount().Balance(s.Market.BaseCurrency) diff --git a/pkg/strategy/xfixedmaker/order_price_risk.go b/pkg/strategy/xfixedmaker/order_price_risk.go new file mode 100644 index 0000000000..e7ec81b1c9 --- /dev/null +++ b/pkg/strategy/xfixedmaker/order_price_risk.go @@ -0,0 +1,34 @@ +package xfixedmaker + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/types" +) + +type OrderPriceRiskControl struct { + referencePrice *indicatorv2.EWMAStream + lossThreshold fixedpoint.Value +} + +func NewOrderPriceRiskControl(referencePrice *indicatorv2.EWMAStream, threshold fixedpoint.Value) *OrderPriceRiskControl { + return &OrderPriceRiskControl{ + referencePrice: referencePrice, + lossThreshold: threshold, + } +} + +func (r *OrderPriceRiskControl) IsSafe(side types.SideType, price fixedpoint.Value, quantity fixedpoint.Value) bool { + refPrice := fixedpoint.NewFromFloat(r.referencePrice.Last(0)) + // calculate profit + var profit fixedpoint.Value + if side == types.SideTypeBuy { + profit = refPrice.Sub(price).Mul(quantity) + } else if side == types.SideTypeSell { + profit = price.Sub(refPrice).Mul(quantity) + } else { + log.Warnf("OrderPriceRiskControl: unsupported side type: %s", side) + return false + } + return profit.Compare(r.lossThreshold) > 0 +} diff --git a/pkg/strategy/xfixedmaker/order_price_risk_test.go b/pkg/strategy/xfixedmaker/order_price_risk_test.go new file mode 100644 index 0000000000..da023c733e --- /dev/null +++ b/pkg/strategy/xfixedmaker/order_price_risk_test.go @@ -0,0 +1,63 @@ +package xfixedmaker + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/types" +) + +func Test_OrderPriceRiskControl_IsSafe(t *testing.T) { + refPrice := 30000.00 + lossThreshold := fixedpoint.NewFromFloat(-100) + + window := types.IntervalWindow{Window: 30, Interval: types.Interval1m} + refPriceEWMA := indicatorv2.EWMA2(nil, window.Window) + refPriceEWMA.PushAndEmit(refPrice) + + cases := []struct { + name string + side types.SideType + price fixedpoint.Value + quantity fixedpoint.Value + isSafe bool + }{ + { + name: "BuyingHighSafe", + side: types.SideTypeBuy, + price: fixedpoint.NewFromFloat(30040.0), + quantity: fixedpoint.NewFromFloat(1.0), + isSafe: true, + }, + { + name: "SellingLowSafe", + side: types.SideTypeSell, + price: fixedpoint.NewFromFloat(29960.0), + quantity: fixedpoint.NewFromFloat(1.0), + isSafe: true, + }, + { + name: "BuyingHighLoss", + side: types.SideTypeBuy, + price: fixedpoint.NewFromFloat(30040.0), + quantity: fixedpoint.NewFromFloat(10.0), + isSafe: false, + }, + { + name: "SellingLowLoss", + side: types.SideTypeSell, + price: fixedpoint.NewFromFloat(29960.0), + quantity: fixedpoint.NewFromFloat(10.0), + isSafe: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var riskControl = NewOrderPriceRiskControl(refPriceEWMA, lossThreshold) + assert.Equal(t, tc.isSafe, riskControl.IsSafe(tc.side, tc.price, tc.quantity)) + }) + } +} diff --git a/pkg/strategy/xfixedmaker/strategy.go b/pkg/strategy/xfixedmaker/strategy.go new file mode 100644 index 0000000000..2e6f927f14 --- /dev/null +++ b/pkg/strategy/xfixedmaker/strategy.go @@ -0,0 +1,250 @@ +package xfixedmaker + +import ( + "context" + "fmt" + "sync" + + "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "xfixedmaker" + +var log = logrus.WithField("strategy", ID) + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +// Fixed spread market making strategy +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + + TradingExchange string `json:"tradingExchange"` + Symbol string `json:"symbol"` + Interval types.Interval `json:"interval"` + Quantity fixedpoint.Value `json:"quantity"` + HalfSpread fixedpoint.Value `json:"halfSpread"` + OrderType types.OrderType `json:"orderType"` + DryRun bool `json:"dryRun"` + + ReferenceExchange string `json:"referenceExchange"` + ReferencePriceEMA types.IntervalWindow `json:"referencePriceEMA"` + OrderPriceLossThreshold fixedpoint.Value `json:"orderPriceLossThreshold"` + + market types.Market + activeOrderBook *bbgo.ActiveOrderBook + orderPriceRiskControl *OrderPriceRiskControl +} + +func (s *Strategy) Defaults() error { + if s.OrderType == "" { + log.Infof("order type is not set, using limit maker order type") + s.OrderType = types.OrderTypeLimitMaker + } + return nil +} +func (s *Strategy) Initialize() error { + return nil +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) Validate() error { + if s.Quantity.Float64() <= 0 { + return fmt.Errorf("quantity should be positive") + } + + if s.HalfSpread.Float64() <= 0 { + return fmt.Errorf("halfSpread should be positive") + } + return nil +} + +func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { + tradingSession, ok := sessions[s.TradingExchange] + if !ok { + log.Errorf("trading session %s is not defined", s.TradingExchange) + return + } + tradingSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) + if !s.CircuitBreakLossThreshold.IsZero() { + tradingSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.CircuitBreakEMA.Interval}) + } + + referenceSession, ok := sessions[s.ReferenceExchange] + if !ok { + log.Errorf("reference session %s is not defined", s.ReferenceExchange) + } + referenceSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.ReferencePriceEMA.Interval}) +} + +func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession) error { + tradingSession, ok := sessions[s.TradingExchange] + if !ok { + return fmt.Errorf("trading session %s is not defined", s.TradingExchange) + } + + referenceSession, ok := sessions[s.ReferenceExchange] + if !ok { + return fmt.Errorf("reference session %s is not defined", s.ReferenceExchange) + } + + market, ok := tradingSession.Market(s.Symbol) + if !ok { + return fmt.Errorf("market %s not found", s.Symbol) + } + s.market = market + + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, tradingSession, s.market, ID, s.InstanceID()) + + s.orderPriceRiskControl = NewOrderPriceRiskControl( + referenceSession.Indicators(s.Symbol).EMA(s.ReferencePriceEMA), + s.OrderPriceLossThreshold, + ) + + s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.activeOrderBook.BindStream(tradingSession.UserDataStream) + s.activeOrderBook.OnFilled(func(order types.Order) { + if s.IsHalted(order.UpdateTime.Time()) { + log.Infof("circuit break halted") + return + } + + if s.activeOrderBook.NumOfOrders() == 0 { + log.Infof("no active orders, placing orders...") + s.placeOrders(ctx) + } + }) + + tradingSession.MarketDataStream.OnKLineClosed(func(kline types.KLine) { + log.Infof("kline: %s", kline.String()) + + if s.IsHalted(kline.EndTime.Time()) { + log.Infof("circuit break halted") + return + } + + if kline.Interval == s.Interval { + s.cancelOrders(ctx) + s.placeOrders(ctx) + } + }) + + // the shutdown handler, you can cancel all orders + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + _ = s.OrderExecutor.GracefulCancel(ctx) + }) + return nil +} + +func (s *Strategy) cancelOrders(ctx context.Context) { + if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + log.WithError(err).Errorf("failed to cancel orders") + } +} + +func (s *Strategy) placeOrders(ctx context.Context) { + submitOrders, err := s.generateOrders(ctx) + if err != nil { + log.WithError(err).Error("failed to generate orders") + return + } + log.Infof("submit orders: %+v", submitOrders) + + if s.DryRun { + log.Infof("dry run, not submitting orders") + return + } + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, submitOrders...) + if err != nil { + log.WithError(err).Error("failed to submit orders") + return + } + log.Infof("created orders: %+v", createdOrders) + + s.activeOrderBook.Add(createdOrders...) +} + +func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, error) { + orders := []types.SubmitOrder{} + + baseBalance, ok := s.Session.GetAccount().Balance(s.market.BaseCurrency) + if !ok { + return nil, fmt.Errorf("base currency %s balance not found", s.market.BaseCurrency) + } + log.Infof("base balance: %s", baseBalance.String()) + + quoteBalance, ok := s.Session.GetAccount().Balance(s.market.QuoteCurrency) + if !ok { + return nil, fmt.Errorf("quote currency %s balance not found", s.market.QuoteCurrency) + } + log.Infof("quote balance: %s", quoteBalance.String()) + + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if err != nil { + return nil, err + } + midPrice := ticker.Buy.Add(ticker.Sell).Div(fixedpoint.NewFromFloat(2.0)) + log.Infof("mid price: %s", midPrice.String()) + + // calculate bid and ask price + // sell price = mid price * (1 + r)) + // buy price = mid price * (1 - r)) + sellPrice := midPrice.Mul(fixedpoint.One.Add(s.HalfSpread)).Round(s.market.PricePrecision, fixedpoint.Up) + buyPrice := midPrice.Mul(fixedpoint.One.Sub(s.HalfSpread)).Round(s.market.PricePrecision, fixedpoint.Down) + log.Infof("sell price: %s, buy price: %s", sellPrice.String(), buyPrice.String()) + + // check balance and generate orders + amount := s.Quantity.Mul(buyPrice) + if quoteBalance.Available.Compare(amount) > 0 { + if s.orderPriceRiskControl.IsSafe(types.SideTypeBuy, buyPrice, s.Quantity) { + orders = append(orders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: s.OrderType, + Price: buyPrice, + Quantity: s.Quantity, + }) + + } else { + log.Infof("ref price risk control triggered, not placing buy order") + } + } else { + log.Infof("not enough quote balance to buy, available: %s, amount: %s", quoteBalance.Available, amount) + } + + if baseBalance.Available.Compare(s.Quantity) > 0 { + if s.orderPriceRiskControl.IsSafe(types.SideTypeSell, sellPrice, s.Quantity) { + orders = append(orders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: s.OrderType, + Price: sellPrice, + Quantity: s.Quantity, + }) + } else { + log.Infof("ref price risk control triggered, not placing sell order") + } + } else { + log.Infof("not enough base balance to sell, available: %s, quantity: %s", baseBalance.Available, s.Quantity) + } + + return orders, nil +} From a0efa2769db901eb39f076e07f3de8f96d25dfa6 Mon Sep 17 00:00:00 2001 From: narumi Date: Thu, 5 Oct 2023 02:19:09 +0800 Subject: [PATCH 066/422] add randtrader strategy --- config/randomtrader.yaml | 9 ++ pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/randomtrader/strategy.go | 117 ++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 config/randomtrader.yaml create mode 100644 pkg/strategy/randomtrader/strategy.go diff --git a/config/randomtrader.yaml b/config/randomtrader.yaml new file mode 100644 index 0000000000..80503bf6e4 --- /dev/null +++ b/config/randomtrader.yaml @@ -0,0 +1,9 @@ +--- +exchangeStrategies: + - on: max + randomtrader: + symbol: USDCUSDT + cronExpression: "@every 8h" # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules + quantity: 8 + onStart: true + dryRun: true diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 05b652a25c..2586b48be5 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -29,6 +29,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/pivotshort" _ "github.com/c9s/bbgo/pkg/strategy/pricealert" _ "github.com/c9s/bbgo/pkg/strategy/pricedrop" + _ "github.com/c9s/bbgo/pkg/strategy/randomtrader" _ "github.com/c9s/bbgo/pkg/strategy/rebalance" _ "github.com/c9s/bbgo/pkg/strategy/rsicross" _ "github.com/c9s/bbgo/pkg/strategy/rsmaker" diff --git a/pkg/strategy/randomtrader/strategy.go b/pkg/strategy/randomtrader/strategy.go new file mode 100644 index 0000000000..1bca02727c --- /dev/null +++ b/pkg/strategy/randomtrader/strategy.go @@ -0,0 +1,117 @@ +package randomtrader + +import ( + "context" + "fmt" + "math/rand" + "sync" + + "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "randomtrader" + +var log = logrus.WithField("strategy", ID) + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + CronExpression string `json:"cronExpression"` + Quantity fixedpoint.Value `json:"quantity"` + OnStart bool `json:"onStart"` + DryRun bool `json:"dryRun"` +} + +func (s *Strategy) Defaults() error { + return nil +} + +func (s *Strategy) Initialize() error { + return nil +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) Validate() error { + if s.CronExpression == "" { + return fmt.Errorf("cronExpression is required") + } + return nil +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, s.ID(), s.InstanceID()) + + session.UserDataStream.OnStart(func() { + if s.OnStart { + s.trade(ctx) + } + }) + + // the shutdown handler, you can cancel all orders + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + _ = s.OrderExecutor.GracefulCancel(ctx) + }) + + cron := cron.New() + cron.AddFunc(s.CronExpression, func() { + s.trade(ctx) + }) + cron.Start() + + return nil +} + +func (s *Strategy) trade(ctx context.Context) { + orderForm := []types.SubmitOrder{ + { + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeMarket, + Quantity: s.Quantity, + }, { + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeMarket, + Quantity: s.Quantity, + }, + } + + submitOrder := orderForm[rand.Intn(2)] + log.Infof("submit order: %s", submitOrder.String()) + + if s.DryRun { + log.Infof("dry run, skip submit order") + return + } + + _, err := s.OrderExecutor.SubmitOrders(ctx, submitOrder) + if err != nil { + log.WithError(err).Error("submit order error") + return + } +} From 81ea074b4f63be8138318cc4ec17e0e014cdcc12 Mon Sep 17 00:00:00 2001 From: narumi Date: Sat, 7 Oct 2023 13:01:45 +0800 Subject: [PATCH 067/422] check balances --- config/randomtrader.yaml | 5 +- pkg/strategy/randomtrader/strategy.go | 78 ++++++++++++++++++++------- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/config/randomtrader.yaml b/config/randomtrader.yaml index 80503bf6e4..9212165cb1 100644 --- a/config/randomtrader.yaml +++ b/config/randomtrader.yaml @@ -3,7 +3,10 @@ exchangeStrategies: - on: max randomtrader: symbol: USDCUSDT - cronExpression: "@every 8h" # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules + # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules + cronExpression: "@every 1m" quantity: 8 + # adjust quantity by minimal notional and minimal quantity + adjustQuantity: true onStart: true dryRun: true diff --git a/pkg/strategy/randomtrader/strategy.go b/pkg/strategy/randomtrader/strategy.go index 1bca02727c..5402cfd56b 100644 --- a/pkg/strategy/randomtrader/strategy.go +++ b/pkg/strategy/randomtrader/strategy.go @@ -32,8 +32,11 @@ type Strategy struct { Symbol string `json:"symbol"` CronExpression string `json:"cronExpression"` Quantity fixedpoint.Value `json:"quantity"` + AdjustQuantity bool `json:"adjustQuantity"` OnStart bool `json:"onStart"` DryRun bool `json:"dryRun"` + + cron *cron.Cron } func (s *Strategy) Defaults() error { @@ -67,7 +70,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. session.UserDataStream.OnStart(func() { if s.OnStart { - s.trade(ctx) + s.placeOrder() } }) @@ -77,39 +80,78 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. _ = s.OrderExecutor.GracefulCancel(ctx) }) - cron := cron.New() - cron.AddFunc(s.CronExpression, func() { - s.trade(ctx) - }) - cron.Start() + s.cron = cron.New() + s.cron.AddFunc(s.CronExpression, s.placeOrder) + s.cron.Start() return nil } -func (s *Strategy) trade(ctx context.Context) { - orderForm := []types.SubmitOrder{ - { +func (s *Strategy) placeOrder() { + ctx := context.Background() + + baseBalance, ok := s.Session.GetAccount().Balance(s.Market.BaseCurrency) + if !ok { + log.Errorf("base balance not found") + return + } + quoteBalance, ok := s.Session.GetAccount().Balance(s.Market.QuoteCurrency) + if !ok { + log.Errorf("quote balance not found") + return + } + + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if err != nil { + log.WithError(err).Error("query ticker error") + return + } + + sellQuantity := s.Quantity + buyQuantity := s.Quantity + if s.AdjustQuantity { + sellQuantity = s.Market.AdjustQuantityByMinNotional(s.Quantity, ticker.Sell) + buyQuantity = fixedpoint.Max(s.Quantity, s.Market.MinQuantity) + } + + orderForm := []types.SubmitOrder{} + if baseBalance.Available.Compare(sellQuantity) > 0 { + orderForm = append(orderForm, types.SubmitOrder{ Symbol: s.Symbol, - Side: types.SideTypeBuy, + Side: types.SideTypeSell, Type: types.OrderTypeMarket, - Quantity: s.Quantity, - }, { + Quantity: sellQuantity, + }) + } else { + log.Infof("base balance: %s is not enough", baseBalance.Available.String()) + } + + if quoteBalance.Available.Div(ticker.Buy).Compare(buyQuantity) > 0 { + orderForm = append(orderForm, types.SubmitOrder{ Symbol: s.Symbol, - Side: types.SideTypeSell, + Side: types.SideTypeBuy, Type: types.OrderTypeMarket, - Quantity: s.Quantity, - }, + Quantity: buyQuantity, + }) + } else { + log.Infof("quote balance: %s is not enough", quoteBalance.Available.String()) } - submitOrder := orderForm[rand.Intn(2)] - log.Infof("submit order: %s", submitOrder.String()) + var order types.SubmitOrder + if len(orderForm) == 0 { + log.Infof("both base and quote balance are not enough, skip submit order") + return + } else { + order = orderForm[rand.Intn(len(orderForm))] + } + log.Infof("submit order: %s", order.String()) if s.DryRun { log.Infof("dry run, skip submit order") return } - _, err := s.OrderExecutor.SubmitOrders(ctx, submitOrder) + _, err = s.OrderExecutor.SubmitOrders(ctx, order) if err != nil { log.WithError(err).Error("submit order error") return From d8ff42d531bfe7917a9d178c998b2d4fb6d94c1e Mon Sep 17 00:00:00 2001 From: narumi Date: Wed, 27 Sep 2023 11:45:31 +0800 Subject: [PATCH 068/422] Fix duplicate orders caused by position risk control --- pkg/risk/riskcontrol/position.go | 46 +++++++++++++++++++++----------- pkg/strategy/common/strategy.go | 1 + 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/pkg/risk/riskcontrol/position.go b/pkg/risk/riskcontrol/position.go index 851c6f5664..8a8fd86d8c 100644 --- a/pkg/risk/riskcontrol/position.go +++ b/pkg/risk/riskcontrol/position.go @@ -24,6 +24,11 @@ type PositionRiskControl struct { // only used in the ModifiedQuantity method sliceQuantity fixedpoint.Value + // activeOrderBook is used to store orders created by the risk control. + // This allows us to cancel them before submitting the position release + // orders, preventing duplicate orders. + activeOrderBook *bbgo.ActiveOrderBook + releasePositionCallbacks []func(quantity fixedpoint.Value, side types.SideType) } @@ -34,8 +39,30 @@ func NewPositionRiskControl(orderExecutor bbgo.OrderExecutorExtended, hardLimit, sliceQuantity: quantity, } - control.OnReleasePosition(func(quantity fixedpoint.Value, side types.SideType) { - pos := orderExecutor.Position() + // register position update handler: check if position is over the hard limit + orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + if fixedpoint.Compare(position.Base, hardLimit) > 0 { + log.Infof("position %f is over hardlimit %f, releasing position...", position.Base.Float64(), hardLimit.Float64()) + control.EmitReleasePosition(position.Base.Sub(hardLimit), types.SideTypeSell) + } else if fixedpoint.Compare(position.Base, hardLimit.Neg()) < 0 { + log.Infof("position %f is over hardlimit %f, releasing position...", position.Base.Float64(), hardLimit.Float64()) + control.EmitReleasePosition(position.Base.Neg().Sub(hardLimit), types.SideTypeBuy) + } + }) + + return control +} + +func (p *PositionRiskControl) Initialize(ctx context.Context, session *bbgo.ExchangeSession) { + p.activeOrderBook = bbgo.NewActiveOrderBook("") + p.activeOrderBook.BindStream(session.UserDataStream) + + p.OnReleasePosition(func(quantity fixedpoint.Value, side types.SideType) { + if err := p.activeOrderBook.GracefulCancel(ctx, session.Exchange); err != nil { + log.WithError(err).Errorf("failed to cancel orders") + } + + pos := p.orderExecutor.Position() submitOrder := types.SubmitOrder{ Symbol: pos.Symbol, Market: pos.Market, @@ -45,27 +72,16 @@ func NewPositionRiskControl(orderExecutor bbgo.OrderExecutorExtended, hardLimit, } log.Infof("RiskControl: position limit exceeded, submitting order to reduce position: %+v", submitOrder) - createdOrders, err := orderExecutor.SubmitOrders(context.Background(), submitOrder) + createdOrders, err := p.orderExecutor.SubmitOrders(ctx, submitOrder) if err != nil { log.WithError(err).Errorf("failed to submit orders") return } log.Infof("created position release orders: %+v", createdOrders) - }) - // register position update handler: check if position is over the hard limit - orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - if fixedpoint.Compare(position.Base, hardLimit) > 0 { - log.Infof("position %f is over hardlimit %f, releasing position...", position.Base.Float64(), hardLimit.Float64()) - control.EmitReleasePosition(position.Base.Sub(hardLimit), types.SideTypeSell) - } else if fixedpoint.Compare(position.Base, hardLimit.Neg()) < 0 { - log.Infof("position %f is over hardlimit %f, releasing position...", position.Base.Float64(), hardLimit.Float64()) - control.EmitReleasePosition(position.Base.Neg().Sub(hardLimit), types.SideTypeBuy) - } + p.activeOrderBook.Add(createdOrders...) }) - - return control } // ModifiedQuantity returns sliceQuantity controlled by position risks diff --git a/pkg/strategy/common/strategy.go b/pkg/strategy/common/strategy.go index 1ed590a84a..991b6b8c58 100644 --- a/pkg/strategy/common/strategy.go +++ b/pkg/strategy/common/strategy.go @@ -76,6 +76,7 @@ func (s *Strategy) Initialize(ctx context.Context, environ *bbgo.Environment, se if !s.PositionHardLimit.IsZero() && !s.MaxPositionQuantity.IsZero() { log.Infof("positionHardLimit and maxPositionQuantity are configured, setting up PositionRiskControl...") s.positionRiskControl = riskcontrol.NewPositionRiskControl(s.OrderExecutor, s.PositionHardLimit, s.MaxPositionQuantity) + s.positionRiskControl.Initialize(ctx, session) } if !s.CircuitBreakLossThreshold.IsZero() { From 4a6f6f7a5ac2c3bd25931f832834462400bf3d22 Mon Sep 17 00:00:00 2001 From: narumi Date: Wed, 11 Oct 2023 12:12:57 +0800 Subject: [PATCH 069/422] add backtest config --- config/fixedmaker.yaml | 23 ++++++++++++++++++++--- pkg/strategy/fixedmaker/strategy.go | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/config/fixedmaker.yaml b/config/fixedmaker.yaml index 98f2130297..63d3ee058d 100644 --- a/config/fixedmaker.yaml +++ b/config/fixedmaker.yaml @@ -1,10 +1,27 @@ --- +backtest: + startTime: "2023-01-01" + endTime: "2023-05-31" + symbols: + - USDCUSDT + sessions: + - max + accounts: + max: + balances: + USDC: 500.0 + USDT: 500.0 + exchangeStrategies: - on: max fixedmaker: - symbol: BTCUSDT + symbol: USDCUSDT interval: 5m halfSpread: 0.05% - quantity: 0.005 + quantity: 15 orderType: LIMIT_MAKER - dryRun: true + dryRun: false + + positionHardLimit: 1500 + maxPositionQuantity: 1500 + circuitBreakLossThreshold: -0.15 diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 28de0a9179..08a7267c62 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -64,7 +64,7 @@ func (s *Strategy) Validate() error { } if s.HalfSpread.Float64() <= 0 { - return fmt.Errorf("halfSpreadRatio should be positive") + return fmt.Errorf("halfSpread should be positive") } return nil } From a8d678a544f90a3e67cfd7a3a90ac15c0d407b5d Mon Sep 17 00:00:00 2001 From: narumi Date: Wed, 11 Oct 2023 15:50:44 +0800 Subject: [PATCH 070/422] rename randomtrader to random --- config/{randomtrader.yaml => random.yaml} | 4 ++-- pkg/cmd/strategy/builtin.go | 2 +- pkg/strategy/{randomtrader => random}/strategy.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename config/{randomtrader.yaml => random.yaml} (84%) rename pkg/strategy/{randomtrader => random}/strategy.go (98%) diff --git a/config/randomtrader.yaml b/config/random.yaml similarity index 84% rename from config/randomtrader.yaml rename to config/random.yaml index 9212165cb1..551e397fc9 100644 --- a/config/randomtrader.yaml +++ b/config/random.yaml @@ -1,10 +1,10 @@ --- exchangeStrategies: - on: max - randomtrader: + random: symbol: USDCUSDT # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules - cronExpression: "@every 1m" + cronExpression: "@every 8h" quantity: 8 # adjust quantity by minimal notional and minimal quantity adjustQuantity: true diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 2124200990..d868e926ab 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -29,7 +29,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/pivotshort" _ "github.com/c9s/bbgo/pkg/strategy/pricealert" _ "github.com/c9s/bbgo/pkg/strategy/pricedrop" - _ "github.com/c9s/bbgo/pkg/strategy/randomtrader" + _ "github.com/c9s/bbgo/pkg/strategy/random" _ "github.com/c9s/bbgo/pkg/strategy/rebalance" _ "github.com/c9s/bbgo/pkg/strategy/rsicross" _ "github.com/c9s/bbgo/pkg/strategy/rsmaker" diff --git a/pkg/strategy/randomtrader/strategy.go b/pkg/strategy/random/strategy.go similarity index 98% rename from pkg/strategy/randomtrader/strategy.go rename to pkg/strategy/random/strategy.go index 5402cfd56b..f1bae583bf 100644 --- a/pkg/strategy/randomtrader/strategy.go +++ b/pkg/strategy/random/strategy.go @@ -1,4 +1,4 @@ -package randomtrader +package random import ( "context" @@ -15,7 +15,7 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -const ID = "randomtrader" +const ID = "random" var log = logrus.WithField("strategy", ID) From a0a7b0ffdcd709aa0370a73bf16ecdbbce9f7e76 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 11 Oct 2023 17:33:07 +0800 Subject: [PATCH 071/422] grid2: set max retries --- pkg/strategy/grid2/strategy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 90af91cba0..32c3dc528b 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1888,6 +1888,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. bbgo.Sync(ctx, s) }) orderExecutor.ActiveMakerOrders().OnFilled(s.newOrderUpdateHandler(ctx, session)) + orderExecutor.SetMaxRetries(5) if s.logger != nil { orderExecutor.SetLogger(s.logger) From ef582f6e52715d6640e8105fdfa86b963a3d02b6 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 12 Oct 2023 11:11:26 +0800 Subject: [PATCH 072/422] pkg/exchange: support order book depth 200 on bybit --- pkg/exchange/bybit/stream.go | 8 ++++++-- pkg/exchange/bybit/stream_test.go | 22 ++++++++++++++++++++++ pkg/types/stream.go | 1 + 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 145591cac0..c6b42cb9b1 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -318,8 +318,12 @@ func (s *Stream) convertSubscription(sub types.Subscription) (string, error) { case types.BookChannel: depth := types.DepthLevel1 - if len(sub.Options.Depth) > 0 && sub.Options.Depth == types.DepthLevel50 { - depth = types.DepthLevel50 + + switch sub.Options.Depth { + case types.DepthLevel50: + depth = sub.Options.Depth + case types.DepthLevel200: + depth = sub.Options.Depth } return genTopic(TopicTypeOrderBook, depth, sub.Symbol), nil diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index 7cfc4d99b6..e1fec2af80 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -395,6 +395,28 @@ func Test_convertSubscription(t *testing.T) { assert.NoError(t, err) assert.Equal(t, genTopic(TopicTypeOrderBook, types.DepthLevel1, "BTCUSDT"), res) }) + t.Run("BookChannel.DepthLevel50", func(t *testing.T) { + res, err := s.convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.BookChannel, + Options: types.SubscribeOptions{ + Depth: types.DepthLevel50, + }, + }) + assert.NoError(t, err) + assert.Equal(t, genTopic(TopicTypeOrderBook, types.DepthLevel50, "BTCUSDT"), res) + }) + t.Run("BookChannel.DepthLevel200", func(t *testing.T) { + res, err := s.convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.BookChannel, + Options: types.SubscribeOptions{ + Depth: types.DepthLevel200, + }, + }) + assert.NoError(t, err) + assert.Equal(t, genTopic(TopicTypeOrderBook, types.DepthLevel200, "BTCUSDT"), res) + }) t.Run("BookChannel. with default depth", func(t *testing.T) { res, err := s.convertSubscription(types.Subscription{ Symbol: "BTCUSDT", diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 50cbf59a03..9574b2b64c 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -521,6 +521,7 @@ const ( DepthLevel5 Depth = "5" DepthLevel20 Depth = "20" DepthLevel50 Depth = "50" + DepthLevel200 Depth = "200" ) type Speed string From 69b38fc3045b0da19e0f82396b9ace98fc395e83 Mon Sep 17 00:00:00 2001 From: narumi Date: Thu, 12 Oct 2023 14:31:20 +0800 Subject: [PATCH 073/422] update xfixedmaker config --- config/xfixedmaker.yaml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/config/xfixedmaker.yaml b/config/xfixedmaker.yaml index c6c6563435..bfee2e7711 100644 --- a/config/xfixedmaker.yaml +++ b/config/xfixedmaker.yaml @@ -7,18 +7,40 @@ sessions: exchange: binance envVarPrefix: binance +backtest: + startTime: "2023-01-01" + endTime: "2023-01-02" + symbols: + - BTCUSDT + sessions: + - max + - binance + accounts: + max: + balances: + BTC: 0.5 + USDT: 15000.0 + crossExchangeStrategies: - xfixedmaker: tradingExchange: max symbol: BTCUSDT interval: 5m - halfSpread: 0.05% + halfSpread: 0.01% quantity: 0.005 orderType: LIMIT_MAKER - dryRun: true + dryRun: false referenceExchange: binance referencePriceEMA: interval: 1m window: 14 orderPriceLossThreshold: -10 + + positionHardLimit: 200 + maxPositionQuantity: 0.005 + + circuitBreakLossThreshold: -10 + circuitBreakEMA: + interval: 1m + window: 14 From ca80bdb2828aef70909283a2392a826fa42211ad Mon Sep 17 00:00:00 2001 From: chiahung Date: Thu, 28 Sep 2023 16:15:31 +0800 Subject: [PATCH 074/422] FEATURE: recover active orders with open orders periodically --- pkg/strategy/grid2/checker.go | 105 +++++++++++++++++++++++++++++++++ pkg/strategy/grid2/strategy.go | 27 +++++---- 2 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 pkg/strategy/grid2/checker.go diff --git a/pkg/strategy/grid2/checker.go b/pkg/strategy/grid2/checker.go new file mode 100644 index 0000000000..2a91b34d6c --- /dev/null +++ b/pkg/strategy/grid2/checker.go @@ -0,0 +1,105 @@ +package grid2 + +import ( + "context" + "strconv" + "sync/atomic" + "time" + + "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" + "github.com/pkg/errors" +) + +type ActiveOrderRecover struct { + strategy *Strategy + + interval time.Duration +} + +func NewActiveOrderRecover(strategy *Strategy, interval time.Duration) *ActiveOrderRecover { + return &ActiveOrderRecover{ + strategy: strategy, + interval: interval, + } +} + +func (c *ActiveOrderRecover) Run(ctx context.Context) { + // sleep for a while to wait for recovered + time.Sleep(util.MillisecondsJitter(5*time.Second, 1000*10)) + + if err := c.recover(ctx); err != nil { + c.strategy.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + } + + ticker := time.NewTicker(util.MillisecondsJitter(c.interval, 30*60*1000)) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := c.recover(ctx); err != nil { + c.strategy.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + } + case <-ctx.Done(): + return + } + } +} + +func (c *ActiveOrderRecover) recover(ctx context.Context) error { + recovered := atomic.LoadInt32(&c.strategy.recovered) + if recovered == 0 { + c.strategy.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") + return nil + } + + c.strategy.logger.Infof("[ActiveOrderRecover] recovering active orders with open orders") + + if c.strategy.getGrid() == nil { + return nil + } + + c.strategy.mu.Lock() + defer c.strategy.mu.Unlock() + + openOrders, err := c.strategy.session.Exchange.QueryOpenOrders(ctx, c.strategy.Symbol) + if err != nil { + return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders") + } + + activeOrderBook := c.strategy.orderExecutor.ActiveMakerOrders() + activeOrders := activeOrderBook.Orders() + + openOrdersMap := make(map[uint64]types.Order) + for _, openOrder := range openOrders { + openOrders[openOrder.OrderID] = openOrder + } + + // update active orders not in open orders + for _, activeOrder := range activeOrders { + if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { + c.strategy.logger.Infof("find active order (%d) not in open orders, updating...", activeOrder.OrderID) + delete(openOrdersMap, activeOrder.OrderID) + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, c.strategy.orderQueryService, types.OrderQuery{ + Symbol: activeOrder.Symbol, + OrderID: strconv.FormatUint(activeOrder.OrderID, 10), + }) + + if err != nil { + c.strategy.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order (%d)", activeOrder.OrderID) + continue + } + + activeOrderBook.Update(*updatedOrder) + } + } + + // update open orders not in active orders + for _, openOrders := range openOrdersMap { + activeOrderBook.Update(openOrders) + } + + return nil +} diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 32c3dc528b..71ca97e104 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -171,6 +171,9 @@ type Strategy struct { RecoverGridByScanningTrades bool `json:"recoverGridByScanningTrades"` RecoverGridWithin time.Duration `json:"recoverGridWithin"` + // activeOrderRecover periodically check the open orders is the same as active orderbook and recover it + activeOrderRecover *ActiveOrderRecover + EnableProfitFixer bool `json:"enableProfitFixer"` FixProfitSince *types.Time `json:"fixProfitSince"` @@ -1986,18 +1989,22 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) } + s.activeOrderRecover = NewActiveOrderRecover(s, 40*time.Minute) session.UserDataStream.OnAuth(func() { if !bbgo.IsBackTesting { - // callback may block the stream execution, so we spawn the recover function to the background - // add (5 seconds + random <10 seconds jitter) delay - go time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { - recovered := atomic.LoadInt32(&s.recovered) - if recovered == 0 { - return - } - - s.recoverActiveOrders(ctx, session) - }) + go s.activeOrderRecover.Run(ctx) + /* + // callback may block the stream execution, so we spawn the recover function to the background + // add (5 seconds + random <10 seconds jitter) delay + go time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { + recovered := atomic.LoadInt32(&s.recovered) + if recovered == 0 { + return + } + + s.recoverActiveOrders(ctx, session) + }) + */ } }) From 4c9b1e78fe4c19d27d5375b02b4cb35a1a239188 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 2 Oct 2023 11:56:15 +0800 Subject: [PATCH 075/422] remove checker --- pkg/strategy/grid2/active_order_recover.go | 94 ++++++++++++++++++ pkg/strategy/grid2/checker.go | 105 --------------------- pkg/strategy/grid2/strategy.go | 6 +- 3 files changed, 95 insertions(+), 110 deletions(-) create mode 100644 pkg/strategy/grid2/active_order_recover.go delete mode 100644 pkg/strategy/grid2/checker.go diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go new file mode 100644 index 0000000000..dfbf3e7b5e --- /dev/null +++ b/pkg/strategy/grid2/active_order_recover.go @@ -0,0 +1,94 @@ +package grid2 + +import ( + "context" + "strconv" + "sync/atomic" + "time" + + "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" +) + +func (s *Strategy) recoverActiveOrdersWithOpenOrdersPeriodically(ctx context.Context) { + // sleep for a while to wait for recovered + time.Sleep(util.MillisecondsJitter(5*time.Second, 1000*10)) + + if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") + } else { + if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + } + } + + ticker := time.NewTicker(util.MillisecondsJitter(40*time.Minute, 30*60*1000)) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") + } else { + if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + } + } + case <-ctx.Done(): + return + } + } +} + +func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOrders []types.Order) error { + recovered := atomic.LoadInt32(&s.recovered) + if recovered == 0 { + s.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") + return nil + } + + s.logger.Infof("[ActiveOrderRecover] recovering active orders with open orders") + + if s.getGrid() == nil { + return nil + } + + s.mu.Lock() + defer s.mu.Unlock() + + activeOrderBook := s.orderExecutor.ActiveMakerOrders() + activeOrders := activeOrderBook.Orders() + + openOrdersMap := make(map[uint64]types.Order) + for _, openOrder := range openOrders { + openOrders[openOrder.OrderID] = openOrder + } + + // update active orders not in open orders + for _, activeOrder := range activeOrders { + if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { + s.logger.Infof("find active order (%d) not in open orders, updating...", activeOrder.OrderID) + delete(openOrdersMap, activeOrder.OrderID) + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ + Symbol: activeOrder.Symbol, + OrderID: strconv.FormatUint(activeOrder.OrderID, 10), + }) + + if err != nil { + s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order (%d)", activeOrder.OrderID) + continue + } + + activeOrderBook.Update(*updatedOrder) + } + } + + // update open orders not in active orders + for _, openOrders := range openOrdersMap { + activeOrderBook.Update(openOrders) + } + + return nil +} diff --git a/pkg/strategy/grid2/checker.go b/pkg/strategy/grid2/checker.go deleted file mode 100644 index 2a91b34d6c..0000000000 --- a/pkg/strategy/grid2/checker.go +++ /dev/null @@ -1,105 +0,0 @@ -package grid2 - -import ( - "context" - "strconv" - "sync/atomic" - "time" - - "github.com/c9s/bbgo/pkg/exchange/retry" - "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" - "github.com/pkg/errors" -) - -type ActiveOrderRecover struct { - strategy *Strategy - - interval time.Duration -} - -func NewActiveOrderRecover(strategy *Strategy, interval time.Duration) *ActiveOrderRecover { - return &ActiveOrderRecover{ - strategy: strategy, - interval: interval, - } -} - -func (c *ActiveOrderRecover) Run(ctx context.Context) { - // sleep for a while to wait for recovered - time.Sleep(util.MillisecondsJitter(5*time.Second, 1000*10)) - - if err := c.recover(ctx); err != nil { - c.strategy.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") - } - - ticker := time.NewTicker(util.MillisecondsJitter(c.interval, 30*60*1000)) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := c.recover(ctx); err != nil { - c.strategy.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") - } - case <-ctx.Done(): - return - } - } -} - -func (c *ActiveOrderRecover) recover(ctx context.Context) error { - recovered := atomic.LoadInt32(&c.strategy.recovered) - if recovered == 0 { - c.strategy.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") - return nil - } - - c.strategy.logger.Infof("[ActiveOrderRecover] recovering active orders with open orders") - - if c.strategy.getGrid() == nil { - return nil - } - - c.strategy.mu.Lock() - defer c.strategy.mu.Unlock() - - openOrders, err := c.strategy.session.Exchange.QueryOpenOrders(ctx, c.strategy.Symbol) - if err != nil { - return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders") - } - - activeOrderBook := c.strategy.orderExecutor.ActiveMakerOrders() - activeOrders := activeOrderBook.Orders() - - openOrdersMap := make(map[uint64]types.Order) - for _, openOrder := range openOrders { - openOrders[openOrder.OrderID] = openOrder - } - - // update active orders not in open orders - for _, activeOrder := range activeOrders { - if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { - c.strategy.logger.Infof("find active order (%d) not in open orders, updating...", activeOrder.OrderID) - delete(openOrdersMap, activeOrder.OrderID) - updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, c.strategy.orderQueryService, types.OrderQuery{ - Symbol: activeOrder.Symbol, - OrderID: strconv.FormatUint(activeOrder.OrderID, 10), - }) - - if err != nil { - c.strategy.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order (%d)", activeOrder.OrderID) - continue - } - - activeOrderBook.Update(*updatedOrder) - } - } - - // update open orders not in active orders - for _, openOrders := range openOrdersMap { - activeOrderBook.Update(openOrders) - } - - return nil -} diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 71ca97e104..6a8f6574b5 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -171,9 +171,6 @@ type Strategy struct { RecoverGridByScanningTrades bool `json:"recoverGridByScanningTrades"` RecoverGridWithin time.Duration `json:"recoverGridWithin"` - // activeOrderRecover periodically check the open orders is the same as active orderbook and recover it - activeOrderRecover *ActiveOrderRecover - EnableProfitFixer bool `json:"enableProfitFixer"` FixProfitSince *types.Time `json:"fixProfitSince"` @@ -1989,10 +1986,9 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) } - s.activeOrderRecover = NewActiveOrderRecover(s, 40*time.Minute) session.UserDataStream.OnAuth(func() { if !bbgo.IsBackTesting { - go s.activeOrderRecover.Run(ctx) + go s.recoverActiveOrdersWithOpenOrdersPeriodically(ctx) /* // callback may block the stream execution, so we spawn the recover function to the background // add (5 seconds + random <10 seconds jitter) delay From 27294ac9b632bf6e57ee83ebceeb4f682c151d72 Mon Sep 17 00:00:00 2001 From: chiahung Date: Fri, 6 Oct 2023 18:04:57 +0800 Subject: [PATCH 076/422] FIX: fix some error and use chan to trigger active orders recover when on auth --- pkg/strategy/grid2/active_order_recover.go | 47 ++++++++++++---------- pkg/strategy/grid2/strategy.go | 42 +++++++++---------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index dfbf3e7b5e..e14e5c0ffa 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -12,36 +12,37 @@ import ( ) func (s *Strategy) recoverActiveOrdersWithOpenOrdersPeriodically(ctx context.Context) { - // sleep for a while to wait for recovered - time.Sleep(util.MillisecondsJitter(5*time.Second, 1000*10)) + // every time we activeOrdersRecoverCh receive signal, do active orders recover + s.activeOrdersRecoverCh = make(chan struct{}, 1) - if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") - } else { - if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") - } - } - - ticker := time.NewTicker(util.MillisecondsJitter(40*time.Minute, 30*60*1000)) + // make ticker's interval random in 40 min ~ 70 min + interval := util.MillisecondsJitter(40*time.Minute, 30*60*1000) + s.logger.Infof("[ActiveOrderRecover] interval: %s", interval) + ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ticker.C: - if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") - } else { - if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") - } - } + s.queryOpenOrdersThenRecoverActiveOrders(ctx) + case <-s.activeOrdersRecoverCh: + s.queryOpenOrdersThenRecoverActiveOrders(ctx) case <-ctx.Done(): return } } } +func (s *Strategy) queryOpenOrdersThenRecoverActiveOrders(ctx context.Context) { + if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") + } else { + if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + } + } +} + func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOrders []types.Order) error { recovered := atomic.LoadInt32(&s.recovered) if recovered == 0 { @@ -63,14 +64,13 @@ func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOr openOrdersMap := make(map[uint64]types.Order) for _, openOrder := range openOrders { - openOrders[openOrder.OrderID] = openOrder + openOrdersMap[openOrder.OrderID] = openOrder } // update active orders not in open orders for _, activeOrder := range activeOrders { if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { s.logger.Infof("find active order (%d) not in open orders, updating...", activeOrder.OrderID) - delete(openOrdersMap, activeOrder.OrderID) updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ Symbol: activeOrder.Symbol, OrderID: strconv.FormatUint(activeOrder.OrderID, 10), @@ -82,12 +82,15 @@ func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOr } activeOrderBook.Update(*updatedOrder) + } else { + delete(openOrdersMap, activeOrder.OrderID) } } + // TODO: should we add open orders back into active orderbook ? // update open orders not in active orders - for _, openOrders := range openOrdersMap { - activeOrderBook.Update(openOrders) + for _, openOrder := range openOrdersMap { + activeOrderBook.Update(openOrder) } return nil diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 6a8f6574b5..88ed84dc5e 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -206,7 +206,8 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc - recovered int32 + recovered int32 + activeOrdersRecoverCh chan struct{} // this ensures that bbgo.Sync to lock the object sync.Mutex @@ -1977,8 +1978,12 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Infof("user data stream started, initializing grid...") if !bbgo.IsBackTesting { - go time.AfterFunc(3*time.Second, func() { - s.startProcess(ctx, session) + time.AfterFunc(3*time.Second, func() { + if err := s.startProcess(ctx, session); err != nil { + return + } + + s.recoverActiveOrdersWithOpenOrdersPeriodically(ctx) }) } else { s.startProcess(ctx, session) @@ -1987,27 +1992,20 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } session.UserDataStream.OnAuth(func() { - if !bbgo.IsBackTesting { - go s.recoverActiveOrdersWithOpenOrdersPeriodically(ctx) - /* - // callback may block the stream execution, so we spawn the recover function to the background - // add (5 seconds + random <10 seconds jitter) delay - go time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { - recovered := atomic.LoadInt32(&s.recovered) - if recovered == 0 { - return - } - - s.recoverActiveOrders(ctx, session) - }) - */ - } + time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { + select { + case s.activeOrdersRecoverCh <- struct{}{}: + s.logger.Info("trigger active orders recover when on auth") + default: + s.logger.Warn("failed to trigger active orders recover when on auth") + } + }) }) return nil } -func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSession) { +func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSession) error { s.debugGridProfitStats("startProcess") if s.RecoverOrdersWhenStart { // do recover only when triggerPrice is not set and not in the back-test mode @@ -2016,15 +2014,17 @@ func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSessi // if recover fail, return and do not open grid s.logger.WithError(err).Error("failed to start process, recover error") s.EmitGridError(errors.Wrapf(err, "failed to start process, recover error")) - return + return err } } // avoid using goroutine here for back-test if err := s.openGrid(ctx, session); err != nil { s.EmitGridError(errors.Wrapf(err, "failed to start process, setup grid orders error")) - return + return err } + + return nil } func (s *Strategy) recoverGrid(ctx context.Context, session *bbgo.ExchangeSession) error { From 1347c8ef87e768b3a9a87fb8d958af2bc0d43de7 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 9 Oct 2023 17:06:22 +0800 Subject: [PATCH 077/422] grid2: refactor recoverActiveOrdersPeriodically --- pkg/strategy/grid2/active_order_recover.go | 41 +++++++++++++--------- pkg/strategy/grid2/strategy.go | 2 +- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index e14e5c0ffa..1c10849b0e 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -11,7 +11,7 @@ import ( "github.com/c9s/bbgo/pkg/util" ) -func (s *Strategy) recoverActiveOrdersWithOpenOrdersPeriodically(ctx context.Context) { +func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverCh receive signal, do active orders recover s.activeOrdersRecoverCh = make(chan struct{}, 1) @@ -23,27 +23,25 @@ func (s *Strategy) recoverActiveOrdersWithOpenOrdersPeriodically(ctx context.Con for { select { - case <-ticker.C: - s.queryOpenOrdersThenRecoverActiveOrders(ctx) - case <-s.activeOrdersRecoverCh: - s.queryOpenOrdersThenRecoverActiveOrders(ctx) + case <-ctx.Done(): return - } - } -} -func (s *Strategy) queryOpenOrdersThenRecoverActiveOrders(ctx context.Context) { - if openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") - } else { - if err := s.recoverActiveOrdersWithOpenOrders(ctx, openOrders); err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to recover avtive orderbook") + case <-ticker.C: + if err := s.syncActiveOrders(ctx); err != nil { + log.WithError(err).Errorf("unable to sync active orders") + } + + case <-s.activeOrdersRecoverCh: + if err := s.syncActiveOrders(ctx); err != nil { + log.WithError(err).Errorf("unable to sync active orders") + } + } } } -func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOrders []types.Order) error { +func (s *Strategy) syncActiveOrders(ctx context.Context) error { recovered := atomic.LoadInt32(&s.recovered) if recovered == 0 { s.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") @@ -56,6 +54,13 @@ func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOr return nil } + openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol) + + if err != nil { + s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") + return err + } + s.mu.Lock() defer s.mu.Unlock() @@ -70,14 +75,16 @@ func (s *Strategy) recoverActiveOrdersWithOpenOrders(ctx context.Context, openOr // update active orders not in open orders for _, activeOrder := range activeOrders { if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { - s.logger.Infof("find active order (%d) not in open orders, updating...", activeOrder.OrderID) + + s.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ Symbol: activeOrder.Symbol, OrderID: strconv.FormatUint(activeOrder.OrderID, 10), }) if err != nil { - s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order (%d)", activeOrder.OrderID) + s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) continue } diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 88ed84dc5e..46e10c050c 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1983,7 +1983,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. return } - s.recoverActiveOrdersWithOpenOrdersPeriodically(ctx) + s.recoverActiveOrdersPeriodically(ctx) }) } else { s.startProcess(ctx, session) From 5f9d020ac8a451af13225ee9ef41b92acb202330 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 9 Oct 2023 17:09:41 +0800 Subject: [PATCH 078/422] grid2: improve some logging --- pkg/strategy/grid2/active_order_recover.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 1c10849b0e..a8e7bcc265 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -42,14 +42,14 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { } func (s *Strategy) syncActiveOrders(ctx context.Context) error { + s.logger.Infof("[ActiveOrderRecover] syncActiveOrders") + recovered := atomic.LoadInt32(&s.recovered) if recovered == 0 { s.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") return nil } - s.logger.Infof("[ActiveOrderRecover] recovering active orders with open orders") - if s.getGrid() == nil { return nil } From a39925b9127d552b8eeaf38c7959751f50abf785 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 9 Oct 2023 17:10:11 +0800 Subject: [PATCH 079/422] grid2: invert if --- pkg/strategy/grid2/active_order_recover.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index a8e7bcc265..9f807dc6f4 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -74,8 +74,9 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { // update active orders not in open orders for _, activeOrder := range activeOrders { - if _, exist := openOrdersMap[activeOrder.OrderID]; !exist { - + if _, exist := openOrdersMap[activeOrder.OrderID]; exist { + delete(openOrdersMap, activeOrder.OrderID) + } else { s.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ @@ -89,8 +90,6 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { } activeOrderBook.Update(*updatedOrder) - } else { - delete(openOrdersMap, activeOrder.OrderID) } } From c6d4ebf57b87306b48db753990b38bff4485800d Mon Sep 17 00:00:00 2001 From: chiahung Date: Wed, 11 Oct 2023 17:12:11 +0800 Subject: [PATCH 080/422] also sync orders already in active orderbook if the open orders are expired --- pkg/strategy/grid2/active_order_recover.go | 44 ++++++++++++++++------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 9f807dc6f4..2b31689606 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "time" + "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" @@ -15,8 +16,8 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverCh receive signal, do active orders recover s.activeOrdersRecoverCh = make(chan struct{}, 1) - // make ticker's interval random in 40 min ~ 70 min - interval := util.MillisecondsJitter(40*time.Minute, 30*60*1000) + // make ticker's interval random in 25 min ~ 35 min + interval := util.MillisecondsJitter(25*time.Minute, 10*60*1000) s.logger.Infof("[ActiveOrderRecover] interval: %s", interval) ticker := time.NewTicker(interval) defer ticker.Stop() @@ -55,12 +56,14 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { } openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol) - if err != nil { s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") return err } + // open orders query time + 1 min means this open orders may be changed ! + openOrdersExpiredTime := time.Now().Add(1 * time.Minute) + s.mu.Lock() defer s.mu.Unlock() @@ -73,27 +76,29 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { } // update active orders not in open orders + var openOrdersExpired bool = false for _, activeOrder := range activeOrders { if _, exist := openOrdersMap[activeOrder.OrderID]; exist { + if openOrdersExpired || time.Now().After(openOrdersExpiredTime) { + openOrdersExpired = true + s.logger.Infof("active order #%d is in the open orders but the update_at is over 1 min, updating...", activeOrder.OrderID) + if err := s.syncActiveOrder(ctx, activeOrderBook, activeOrder.OrderID); err != nil { + s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + continue + } + } + delete(openOrdersMap, activeOrder.OrderID) } else { s.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ - Symbol: activeOrder.Symbol, - OrderID: strconv.FormatUint(activeOrder.OrderID, 10), - }) - - if err != nil { + if err := s.syncActiveOrder(ctx, activeOrderBook, activeOrder.OrderID); err != nil { s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) continue } - - activeOrderBook.Update(*updatedOrder) } } - // TODO: should we add open orders back into active orderbook ? // update open orders not in active orders for _, openOrder := range openOrdersMap { activeOrderBook.Update(openOrder) @@ -101,3 +106,18 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { return nil } + +func (s *Strategy) syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderID uint64) error { + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ + Symbol: s.Symbol, + OrderID: strconv.FormatUint(orderID, 10), + }) + + if err != nil { + return err + } + + activeOrderBook.Update(*updatedOrder) + + return nil +} From 136c2cd36f18a35609902dd9502c1d09bc647121 Mon Sep 17 00:00:00 2001 From: chiahung Date: Wed, 11 Oct 2023 17:36:18 +0800 Subject: [PATCH 081/422] add open orders metrics --- pkg/strategy/grid2/active_order_recover.go | 2 ++ pkg/strategy/grid2/metrics.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 2b31689606..fd1a8d648d 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -61,6 +61,8 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { return err } + metricsNumOfOpenOrders.With(s.newPrometheusLabels()).Set(float64(len(openOrders))) + // open orders query time + 1 min means this open orders may be changed ! openOrdersExpiredTime := time.Now().Add(1 * time.Minute) diff --git a/pkg/strategy/grid2/metrics.go b/pkg/strategy/grid2/metrics.go index b72bb4b3f0..747164201d 100644 --- a/pkg/strategy/grid2/metrics.go +++ b/pkg/strategy/grid2/metrics.go @@ -18,6 +18,8 @@ var ( metricsGridBaseInvestment *prometheus.GaugeVec metricsGridFilledOrderPrice *prometheus.GaugeVec + + metricsNumOfOpenOrders *prometheus.GaugeVec ) func labelKeys(labels prometheus.Labels) []string { @@ -167,6 +169,18 @@ func initMetrics(extendedLabels []string) { "side", }, extendedLabels...), ) + + metricsNumOfOpenOrders = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "bbgo_grid2_num_of_open_orders", + Help: "number of open orders", + }, + append([]string{ + "exchange", // exchange name + "symbol", // symbol of the market + }, extendedLabels...), + ) + } var metricsRegistered = false @@ -193,6 +207,7 @@ func registerMetrics() { metricsGridQuoteInvestment, metricsGridBaseInvestment, metricsGridFilledOrderPrice, + metricsNumOfOpenOrders, ) metricsRegistered = true } From de1a884153d4f70dc4cbb27626dc6be8300761bc Mon Sep 17 00:00:00 2001 From: chiahung Date: Thu, 12 Oct 2023 14:20:34 +0800 Subject: [PATCH 082/422] not add non existing open orders into active orderbook if updated in 5 min --- pkg/strategy/grid2/active_order_recover.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index fd1a8d648d..fc5e23bd84 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -45,6 +45,8 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { func (s *Strategy) syncActiveOrders(ctx context.Context) error { s.logger.Infof("[ActiveOrderRecover] syncActiveOrders") + notAddNonExistingOpenOrdersAfter := time.Now().Add(-5 * time.Minute) + recovered := atomic.LoadInt32(&s.recovered) if recovered == 0 { s.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") @@ -63,9 +65,6 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { metricsNumOfOpenOrders.With(s.newPrometheusLabels()).Set(float64(len(openOrders))) - // open orders query time + 1 min means this open orders may be changed ! - openOrdersExpiredTime := time.Now().Add(1 * time.Minute) - s.mu.Lock() defer s.mu.Unlock() @@ -78,18 +77,9 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { } // update active orders not in open orders - var openOrdersExpired bool = false for _, activeOrder := range activeOrders { if _, exist := openOrdersMap[activeOrder.OrderID]; exist { - if openOrdersExpired || time.Now().After(openOrdersExpiredTime) { - openOrdersExpired = true - s.logger.Infof("active order #%d is in the open orders but the update_at is over 1 min, updating...", activeOrder.OrderID) - if err := s.syncActiveOrder(ctx, activeOrderBook, activeOrder.OrderID); err != nil { - s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) - continue - } - } - + // no need to sync active order already in active orderbook, because we only need to know if it filled or not. delete(openOrdersMap, activeOrder.OrderID) } else { s.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) @@ -103,6 +93,11 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { // update open orders not in active orders for _, openOrder := range openOrdersMap { + // we don't add open orders into active orderbook if updated in 5 min + if openOrder.UpdateTime.After(notAddNonExistingOpenOrdersAfter) { + continue + } + activeOrderBook.Update(openOrder) } From c5449374cd8e8134261d838026ef64648f42d017 Mon Sep 17 00:00:00 2001 From: chiahung Date: Fri, 13 Oct 2023 16:50:59 +0800 Subject: [PATCH 083/422] add test and remove recovered atmoic bool --- pkg/strategy/grid2/active_order_recover.go | 72 +++---- .../grid2/active_order_recover_test.go | 176 ++++++++++++++++++ pkg/strategy/grid2/strategy.go | 6 - 3 files changed, 216 insertions(+), 38 deletions(-) create mode 100644 pkg/strategy/grid2/active_order_recover_test.go diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index fc5e23bd84..2d8dab19c7 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -3,15 +3,26 @@ package grid2 import ( "context" "strconv" - "sync/atomic" "time" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "go.uber.org/multierr" ) +type SyncActiveOrdersOpts struct { + logger *logrus.Entry + metricsLabels prometheus.Labels + activeOrderBook *bbgo.ActiveOrderBook + orderQueryService types.ExchangeOrderQueryService + exchange types.Exchange +} + func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverCh receive signal, do active orders recover s.activeOrdersRecoverCh = make(chan struct{}, 1) @@ -22,6 +33,14 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { ticker := time.NewTicker(interval) defer ticker.Stop() + opts := SyncActiveOrdersOpts{ + logger: s.logger, + metricsLabels: s.newPrometheusLabels(), + activeOrderBook: s.orderExecutor.ActiveMakerOrders(), + orderQueryService: s.orderQueryService, + exchange: s.session.Exchange, + } + for { select { @@ -29,12 +48,12 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { return case <-ticker.C: - if err := s.syncActiveOrders(ctx); err != nil { + if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") } case <-s.activeOrdersRecoverCh: - if err := s.syncActiveOrders(ctx); err != nil { + if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") } @@ -42,50 +61,38 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { } } -func (s *Strategy) syncActiveOrders(ctx context.Context) error { - s.logger.Infof("[ActiveOrderRecover] syncActiveOrders") +func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { + opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders") notAddNonExistingOpenOrdersAfter := time.Now().Add(-5 * time.Minute) - recovered := atomic.LoadInt32(&s.recovered) - if recovered == 0 { - s.logger.Infof("[ActiveOrderRecover] skip recovering active orders because recover not ready") - return nil - } - - if s.getGrid() == nil { - return nil - } - - openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol) + openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, opts.exchange, opts.activeOrderBook.Symbol) if err != nil { - s.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") - return err + opts.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") + return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders, skip this time") } - metricsNumOfOpenOrders.With(s.newPrometheusLabels()).Set(float64(len(openOrders))) + metricsNumOfOpenOrders.With(opts.metricsLabels).Set(float64(len(openOrders))) - s.mu.Lock() - defer s.mu.Unlock() - - activeOrderBook := s.orderExecutor.ActiveMakerOrders() - activeOrders := activeOrderBook.Orders() + activeOrders := opts.activeOrderBook.Orders() openOrdersMap := make(map[uint64]types.Order) for _, openOrder := range openOrders { openOrdersMap[openOrder.OrderID] = openOrder } + var errs error // update active orders not in open orders for _, activeOrder := range activeOrders { if _, exist := openOrdersMap[activeOrder.OrderID]; exist { // no need to sync active order already in active orderbook, because we only need to know if it filled or not. delete(openOrdersMap, activeOrder.OrderID) } else { - s.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if err := s.syncActiveOrder(ctx, activeOrderBook, activeOrder.OrderID); err != nil { - s.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID); err != nil { + opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + errs = multierr.Append(errs, err) continue } } @@ -98,15 +105,16 @@ func (s *Strategy) syncActiveOrders(ctx context.Context) error { continue } - activeOrderBook.Update(openOrder) + opts.activeOrderBook.Add(openOrder) + // opts.activeOrderBook.Update(openOrder) } - return nil + return errs } -func (s *Strategy) syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderID uint64) error { - updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, s.orderQueryService, types.OrderQuery{ - Symbol: s.Symbol, +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ + Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), }) diff --git a/pkg/strategy/grid2/active_order_recover_test.go b/pkg/strategy/grid2/active_order_recover_test.go new file mode 100644 index 0000000000..dffdccc388 --- /dev/null +++ b/pkg/strategy/grid2/active_order_recover_test.go @@ -0,0 +1,176 @@ +package grid2 + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/types/mocks" + "github.com/golang/mock/gomock" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" +) + +func TestSyncActiveOrders(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + labels := prometheus.Labels{ + "exchange": "default", + "symbol": symbol, + } + t.Run("all open orders are match with active orderbook", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + mockExchange := mocks.NewMockExchange(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + opts := SyncActiveOrdersOpts{ + logger: log, + metricsLabels: labels, + activeOrderBook: activeOrderbook, + orderQueryService: mockOrderQueryService, + exchange: mockExchange, + } + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + } + order.Symbol = symbol + + activeOrderbook.Add(order) + mockExchange.EXPECT().QueryOpenOrders(ctx, symbol).Return([]types.Order{order}, nil) + + assert.NoError(syncActiveOrders(ctx, opts)) + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(uint64(1), activeOrders[0].OrderID) + assert.Equal(types.OrderStatusNew, activeOrders[0].Status) + }) + + t.Run("there is order in active orderbook but not in open orders", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + mockExchange := mocks.NewMockExchange(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + opts := SyncActiveOrdersOpts{ + logger: log, + metricsLabels: labels, + activeOrderBook: activeOrderbook, + orderQueryService: mockOrderQueryService, + exchange: mockExchange, + } + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + updatedOrder := order + updatedOrder.Status = types.OrderStatusFilled + + activeOrderbook.Add(order) + mockExchange.EXPECT().QueryOpenOrders(ctx, symbol).Return(nil, nil) + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + assert.NoError(syncActiveOrders(ctx, opts)) + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(0, len(activeOrders)) + }) + + t.Run("there is order on open orders but not in active orderbook", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + mockExchange := mocks.NewMockExchange(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + opts := SyncActiveOrdersOpts{ + logger: log, + metricsLabels: labels, + activeOrderBook: activeOrderbook, + orderQueryService: mockOrderQueryService, + exchange: mockExchange, + } + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + CreationTime: types.Time(time.Now()), + } + + mockExchange.EXPECT().QueryOpenOrders(ctx, symbol).Return([]types.Order{order}, nil) + assert.NoError(syncActiveOrders(ctx, opts)) + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(uint64(1), activeOrders[0].OrderID) + assert.Equal(types.OrderStatusNew, activeOrders[0].Status) + }) + + t.Run("there is order on open order but not in active orderbook also order in active orderbook but not on open orders", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + mockExchange := mocks.NewMockExchange(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + opts := SyncActiveOrdersOpts{ + logger: log, + metricsLabels: labels, + activeOrderBook: activeOrderbook, + orderQueryService: mockOrderQueryService, + exchange: mockExchange, + } + + order1 := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + updatedOrder1 := order1 + updatedOrder1.Status = types.OrderStatusFilled + order2 := types.Order{ + OrderID: 2, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + + activeOrderbook.Add(order1) + mockExchange.EXPECT().QueryOpenOrders(ctx, symbol).Return([]types.Order{order2}, nil) + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order1.OrderID, 10), + }).Return(&updatedOrder1, nil) + + assert.NoError(syncActiveOrders(ctx, opts)) + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(uint64(2), activeOrders[0].OrderID) + assert.Equal(types.OrderStatusNew, activeOrders[0].Status) + }) +} diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 46e10c050c..cb4ef25a36 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "time" "github.com/google/uuid" @@ -206,7 +205,6 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc - recovered int32 activeOrdersRecoverCh chan struct{} // this ensures that bbgo.Sync to lock the object @@ -2028,10 +2026,6 @@ func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSessi } func (s *Strategy) recoverGrid(ctx context.Context, session *bbgo.ExchangeSession) error { - defer func() { - atomic.AddInt32(&s.recovered, 1) - }() - if s.RecoverGridByScanningTrades { s.debugLog("recovering grid by scanning trades") return s.recoverByScanningTrades(ctx, session) From badadafa2d0dbc1df8389bfbc68fe92d68e10591 Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 13 Oct 2023 18:11:21 +0800 Subject: [PATCH 084/422] remove adjustQuantity from config --- config/random.yaml | 2 -- pkg/strategy/random/strategy.go | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/config/random.yaml b/config/random.yaml index 551e397fc9..269c5d8a8b 100644 --- a/config/random.yaml +++ b/config/random.yaml @@ -6,7 +6,5 @@ exchangeStrategies: # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules cronExpression: "@every 8h" quantity: 8 - # adjust quantity by minimal notional and minimal quantity - adjustQuantity: true onStart: true dryRun: true diff --git a/pkg/strategy/random/strategy.go b/pkg/strategy/random/strategy.go index f1bae583bf..96b56cb5a6 100644 --- a/pkg/strategy/random/strategy.go +++ b/pkg/strategy/random/strategy.go @@ -10,7 +10,6 @@ import ( "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/bbgo" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -29,13 +28,12 @@ type Strategy struct { Environment *bbgo.Environment Market types.Market - Symbol string `json:"symbol"` - CronExpression string `json:"cronExpression"` - Quantity fixedpoint.Value `json:"quantity"` - AdjustQuantity bool `json:"adjustQuantity"` - OnStart bool `json:"onStart"` - DryRun bool `json:"dryRun"` + Symbol string `json:"symbol"` + CronExpression string `json:"cronExpression"` + OnStart bool `json:"onStart"` + DryRun bool `json:"dryRun"` + bbgo.QuantityOrAmount cron *cron.Cron } @@ -59,6 +57,10 @@ func (s *Strategy) Validate() error { if s.CronExpression == "" { return fmt.Errorf("cronExpression is required") } + + if err := s.QuantityOrAmount.Validate(); err != nil { + return err + } return nil } @@ -107,12 +109,10 @@ func (s *Strategy) placeOrder() { return } - sellQuantity := s.Quantity - buyQuantity := s.Quantity - if s.AdjustQuantity { - sellQuantity = s.Market.AdjustQuantityByMinNotional(s.Quantity, ticker.Sell) - buyQuantity = fixedpoint.Max(s.Quantity, s.Market.MinQuantity) - } + sellQuantity := s.CalculateQuantity(ticker.Sell) + buyQuantity := s.CalculateQuantity(ticker.Buy) + sellQuantity = s.Market.AdjustQuantityByMinNotional(sellQuantity, ticker.Sell) + buyQuantity = s.Market.AdjustQuantityByMinNotional(buyQuantity, ticker.Buy) orderForm := []types.SubmitOrder{} if baseBalance.Available.Compare(sellQuantity) > 0 { From 4c69dccf094965bd0b92c454a4e9309bd76f6823 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 16 Oct 2023 12:36:52 +0800 Subject: [PATCH 085/422] make rightWindow possible to be set as zero --- pkg/datatype/floats/pivot.go | 4 ---- pkg/datatype/floats/pivot_test.go | 29 +++++++++++++++++++++++++++++ pkg/indicator/pivothigh.go | 6 +++++- pkg/indicator/pivotlow.go | 6 +++++- pkg/indicator/pivotlow_test.go | 4 ++-- pkg/strategy/trendtrader/trend.go | 20 ++++++++++++++------ pkg/types/interval.go | 2 +- 7 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 pkg/datatype/floats/pivot_test.go diff --git a/pkg/datatype/floats/pivot.go b/pkg/datatype/floats/pivot.go index b7536cbe3e..00bad94270 100644 --- a/pkg/datatype/floats/pivot.go +++ b/pkg/datatype/floats/pivot.go @@ -7,10 +7,6 @@ func (s Slice) Pivot(left, right int, f func(a, pivot float64) bool) (float64, b func FindPivot(values Slice, left, right int, f func(a, pivot float64) bool) (float64, bool) { length := len(values) - if right == 0 { - right = left - } - if length == 0 || length < left+right+1 { return 0.0, false } diff --git a/pkg/datatype/floats/pivot_test.go b/pkg/datatype/floats/pivot_test.go new file mode 100644 index 0000000000..5228882ca7 --- /dev/null +++ b/pkg/datatype/floats/pivot_test.go @@ -0,0 +1,29 @@ +package floats + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFindPivot(t *testing.T) { + + t.Run("middle", func(t *testing.T) { + pv, ok := FindPivot(Slice{10, 20, 30, 40, 30, 20}, 2, 2, func(a, pivot float64) bool { + return a < pivot + }) + if assert.True(t, ok) { + assert.Equal(t, 40., pv) + } + }) + + t.Run("last", func(t *testing.T) { + pv, ok := FindPivot(Slice{10, 20, 30, 40, 30, 45}, 2, 0, func(a, pivot float64) bool { + return a < pivot + }) + if assert.True(t, ok) { + assert.Equal(t, 45., pv) + } + }) + +} diff --git a/pkg/indicator/pivothigh.go b/pkg/indicator/pivothigh.go index ec90f57f79..2569e31fbb 100644 --- a/pkg/indicator/pivothigh.go +++ b/pkg/indicator/pivothigh.go @@ -39,7 +39,11 @@ func (inc *PivotHigh) Update(value float64) { return } - high, ok := calculatePivotHigh(inc.Highs, inc.Window, inc.RightWindow) + if inc.RightWindow == nil { + inc.RightWindow = &inc.Window + } + + high, ok := calculatePivotHigh(inc.Highs, inc.Window, *inc.RightWindow) if !ok { return } diff --git a/pkg/indicator/pivotlow.go b/pkg/indicator/pivotlow.go index 2023fc9417..021a432ef2 100644 --- a/pkg/indicator/pivotlow.go +++ b/pkg/indicator/pivotlow.go @@ -39,7 +39,11 @@ func (inc *PivotLow) Update(value float64) { return } - low, ok := calculatePivotLow(inc.Lows, inc.Window, inc.RightWindow) + if inc.RightWindow == nil { + inc.RightWindow = &inc.Window + } + + low, ok := calculatePivotLow(inc.Lows, inc.Window, *inc.RightWindow) if !ok { return } diff --git a/pkg/indicator/pivotlow_test.go b/pkg/indicator/pivotlow_test.go index 318df37a7c..2425f530e4 100644 --- a/pkg/indicator/pivotlow_test.go +++ b/pkg/indicator/pivotlow_test.go @@ -36,8 +36,8 @@ func Test_calculatePivotLow(t *testing.T) { assert.Equal(t, 0.0, low) }) - t.Run("right window 0", func(t *testing.T) { - low, ok := calculatePivotLow([]float64{15.0, 13.0, 12.0, 10.0, 14.0, 15.0}, 2, 0) + t.Run("right window same", func(t *testing.T) { + low, ok := calculatePivotLow([]float64{15.0, 13.0, 12.0, 10.0, 14.0, 15.0}, 2, 2) assert.True(t, ok) assert.Equal(t, 10.0, low) }) diff --git a/pkg/strategy/trendtrader/trend.go b/pkg/strategy/trendtrader/trend.go index 81876ef30a..d562e83443 100644 --- a/pkg/strategy/trendtrader/trend.go +++ b/pkg/strategy/trendtrader/trend.go @@ -14,7 +14,7 @@ type TrendLine struct { Market types.Market `json:"-"` types.IntervalWindow - PivotRightWindow fixedpoint.Value `json:"pivotRightWindow"` + PivotRightWindow int `json:"pivotRightWindow"` // MarketOrder is the option to enable market order short. MarketOrder bool `json:"marketOrder"` @@ -35,9 +35,9 @@ func (s *TrendLine) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) - //if s.pivot != nil { + // if s.pivot != nil { // session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) - //} + // } } func (s *TrendLine) Bind(session *bbgo.ExchangeSession, orderExecutor *bbgo.GeneralOrderExecutor) { @@ -47,8 +47,14 @@ func (s *TrendLine) Bind(session *bbgo.ExchangeSession, orderExecutor *bbgo.Gene position := orderExecutor.Position() symbol := position.Symbol standardIndicator := session.StandardIndicatorSet(s.Symbol) - s.pivotHigh = standardIndicator.PivotHigh(types.IntervalWindow{s.Interval, int(3. * s.PivotRightWindow.Float64()), int(s.PivotRightWindow.Float64())}) - s.pivotLow = standardIndicator.PivotLow(types.IntervalWindow{s.Interval, int(3. * s.PivotRightWindow.Float64()), int(s.PivotRightWindow.Float64())}) + + s.pivotHigh = standardIndicator.PivotHigh(types.IntervalWindow{ + Interval: s.Interval, + Window: int(3. * s.PivotRightWindow), RightWindow: &s.PivotRightWindow}) + + s.pivotLow = standardIndicator.PivotLow(types.IntervalWindow{ + Interval: s.Interval, + Window: int(3. * s.PivotRightWindow), RightWindow: &s.PivotRightWindow}) resistancePrices := types.NewQueue(3) pivotHighDurationCounter := 0. @@ -124,7 +130,9 @@ func (s *TrendLine) Bind(session *bbgo.ExchangeSession, orderExecutor *bbgo.Gene } } -func (s *TrendLine) placeOrder(ctx context.Context, side types.SideType, quantity fixedpoint.Value, symbol string) error { +func (s *TrendLine) placeOrder( + ctx context.Context, side types.SideType, quantity fixedpoint.Value, symbol string, +) error { market, _ := s.session.Market(symbol) _, err := s.orderExecutor.SubmitOrders(ctx, types.SubmitOrder{ Symbol: symbol, diff --git a/pkg/types/interval.go b/pkg/types/interval.go index 3410b0657e..689da68e71 100644 --- a/pkg/types/interval.go +++ b/pkg/types/interval.go @@ -179,7 +179,7 @@ type IntervalWindow struct { Window int `json:"window"` // RightWindow is used by the pivot indicator - RightWindow int `json:"rightWindow"` + RightWindow *int `json:"rightWindow"` } type IntervalWindowBandWidth struct { From dfa3f7d4c43ba7868185fb8c86bd2a60eadbbcbc Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 16 Oct 2023 12:40:44 +0800 Subject: [PATCH 086/422] indicator: make right window optional --- pkg/indicator/v2/pivothigh.go | 7 ++++++- pkg/indicator/v2/pivotlow.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/indicator/v2/pivothigh.go b/pkg/indicator/v2/pivothigh.go index 74267ab885..020edef7e3 100644 --- a/pkg/indicator/v2/pivothigh.go +++ b/pkg/indicator/v2/pivothigh.go @@ -11,7 +11,12 @@ type PivotHighStream struct { window, rightWindow int } -func PivotHigh2(source types.Float64Source, window, rightWindow int) *PivotHighStream { +func PivotHigh(source types.Float64Source, window int, args ...int) *PivotHighStream { + rightWindow := window + if len(args) > 0 { + rightWindow = args[0] + } + s := &PivotHighStream{ Float64Series: types.NewFloat64Series(), window: window, diff --git a/pkg/indicator/v2/pivotlow.go b/pkg/indicator/v2/pivotlow.go index cdd7aadded..1bbb1977cb 100644 --- a/pkg/indicator/v2/pivotlow.go +++ b/pkg/indicator/v2/pivotlow.go @@ -11,7 +11,12 @@ type PivotLowStream struct { window, rightWindow int } -func PivotLow(source types.Float64Source, window, rightWindow int) *PivotLowStream { +func PivotLow(source types.Float64Source, window int, args ...int) *PivotLowStream { + rightWindow := window + if len(args) > 0 { + rightWindow = args[0] + } + s := &PivotLowStream{ Float64Series: types.NewFloat64Series(), window: window, From 5ff3828ec1ac57fa99b51cd41eaab279bb090abc Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 16 Oct 2023 16:02:43 +0800 Subject: [PATCH 087/422] move to onAuth --- pkg/strategy/grid2/active_order_recover.go | 22 +++++++++++++++++++++- pkg/strategy/grid2/strategy.go | 21 ++------------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 2d8dab19c7..60774d2290 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -23,9 +23,29 @@ type SyncActiveOrdersOpts struct { exchange types.Exchange } +func (s *Strategy) initializeRecoverCh() bool { + s.mu.Lock() + defer s.mu.Unlock() + + alreadyInitialize := false + + if s.activeOrdersRecoverCh == nil { + s.logger.Info("initialize recover channel") + s.activeOrdersRecoverCh = make(chan struct{}, 1) + } else { + s.logger.Info("already initialize recover channel, trigger active orders recover") + alreadyInitialize = true + s.activeOrdersRecoverCh <- struct{}{} + } + + return alreadyInitialize +} + func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverCh receive signal, do active orders recover - s.activeOrdersRecoverCh = make(chan struct{}, 1) + if alreadyInitialize := s.initializeRecoverCh(); alreadyInitialize { + return + } // make ticker's interval random in 25 min ~ 35 min interval := util.MillisecondsJitter(25*time.Minute, 10*60*1000) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index cb4ef25a36..8c389cd5b9 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1967,14 +1967,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. // if TriggerPrice is zero, that means we need to open the grid when start up if s.TriggerPrice.IsZero() { - // must call the openGrid method inside the OnStart callback because - // it needs to receive the trades from the user data stream - // - // should try to avoid blocking the user data stream - // callbacks are blocking operation - session.UserDataStream.OnStart(func() { - s.logger.Infof("user data stream started, initializing grid...") - + session.UserDataStream.OnAuth(func() { + s.logger.Infof("user data stream authenticated, start the process") if !bbgo.IsBackTesting { time.AfterFunc(3*time.Second, func() { if err := s.startProcess(ctx, session); err != nil { @@ -1989,17 +1983,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) } - session.UserDataStream.OnAuth(func() { - time.AfterFunc(util.MillisecondsJitter(5*time.Second, 1000*10), func() { - select { - case s.activeOrdersRecoverCh <- struct{}{}: - s.logger.Info("trigger active orders recover when on auth") - default: - s.logger.Warn("failed to trigger active orders recover when on auth") - } - }) - }) - return nil } From c257bc8ccfdbce85a0653073fe63afa0dbd9b594 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 17 Oct 2023 13:51:51 +0800 Subject: [PATCH 088/422] sleep 100ms to avoid DDOS --- pkg/strategy/grid2/active_order_recover.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 60774d2290..9ad16a78c5 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -110,6 +110,9 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + // sleep 100ms to avoid DDOS + time.Sleep(100 * time.Millisecond) + if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID); err != nil { opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) errs = multierr.Append(errs, err) From 243b90aaf92dc32a09f94f9addcb9495dea3e803 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 17 Oct 2023 15:20:28 +0800 Subject: [PATCH 089/422] fix nil metrics error --- pkg/strategy/grid2/active_order_recover.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 9ad16a78c5..32b911681e 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -92,7 +92,9 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders, skip this time") } - metricsNumOfOpenOrders.With(opts.metricsLabels).Set(float64(len(openOrders))) + if metricsNumOfOpenOrders != nil { + metricsNumOfOpenOrders.With(opts.metricsLabels).Set(float64(len(openOrders))) + } activeOrders := opts.activeOrderBook.Orders() From 10daeab1cb3c12b2403241fd2be2d848550226b4 Mon Sep 17 00:00:00 2001 From: gx578007 Date: Tue, 17 Oct 2023 15:45:55 +0800 Subject: [PATCH 090/422] FIX: [max] remove outdated margin fields --- pkg/exchange/max/exchange.go | 4 ++-- pkg/exchange/max/maxapi/account.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 0a894d4fa4..4c095a5c3c 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -664,8 +664,8 @@ func (e *Exchange) queryBalances(ctx context.Context, walletType maxapi.WalletTy Currency: cur, Available: b.Balance, Locked: b.Locked, - NetAsset: b.Balance.Add(b.Locked).Sub(b.Debt), - Borrowed: b.Borrowed, + NetAsset: b.Balance.Add(b.Locked).Sub(b.Principal).Sub(b.Interest), + Borrowed: b.Principal, Interest: b.Interest, } } diff --git a/pkg/exchange/max/maxapi/account.go b/pkg/exchange/max/maxapi/account.go index 17d4e29d47..678de81f72 100644 --- a/pkg/exchange/max/maxapi/account.go +++ b/pkg/exchange/max/maxapi/account.go @@ -25,9 +25,7 @@ type Account struct { Locked fixedpoint.Value `json:"locked"` // v3 fields for M wallet - Debt fixedpoint.Value `json:"debt"` Principal fixedpoint.Value `json:"principal"` - Borrowed fixedpoint.Value `json:"borrowed"` Interest fixedpoint.Value `json:"interest"` // v2 fields From ccb7308263053baea0fec75c06c0178ece1056b0 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 17 Oct 2023 16:13:05 +0800 Subject: [PATCH 091/422] fix --- pkg/strategy/grid2/active_order_recover.go | 28 +++++++++++++--------- pkg/strategy/grid2/strategy.go | 25 +------------------ 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 32b911681e..e93e722f01 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -27,23 +27,29 @@ func (s *Strategy) initializeRecoverCh() bool { s.mu.Lock() defer s.mu.Unlock() - alreadyInitialize := false + isInitialize := false - if s.activeOrdersRecoverCh == nil { - s.logger.Info("initialize recover channel") - s.activeOrdersRecoverCh = make(chan struct{}, 1) + if s.activeOrdersRecoverC == nil { + s.logger.Info("initializing recover channel") + s.activeOrdersRecoverC = make(chan struct{}, 1) } else { - s.logger.Info("already initialize recover channel, trigger active orders recover") - alreadyInitialize = true - s.activeOrdersRecoverCh <- struct{}{} + s.logger.Info("recover channel is already initialized, trigger active orders recover") + isInitialize = true + + select { + case s.activeOrdersRecoverC <- struct{}{}: + s.logger.Info("trigger active orders recover") + default: + s.logger.Info("activeOrdersRecoverC is full") + } } - return alreadyInitialize + return isInitialize } func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { - // every time we activeOrdersRecoverCh receive signal, do active orders recover - if alreadyInitialize := s.initializeRecoverCh(); alreadyInitialize { + // every time we activeOrdersRecoverC receive signal, do active orders recover + if isInitialize := s.initializeRecoverCh(); isInitialize { return } @@ -72,7 +78,7 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { log.WithError(err).Errorf("unable to sync active orders") } - case <-s.activeOrdersRecoverCh: + case <-s.activeOrdersRecoverC: if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") } diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 8c389cd5b9..736b1ad67c 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -2,7 +2,6 @@ package grid2 import ( "context" - "encoding/json" "fmt" "math" "sort" @@ -205,7 +204,7 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc - activeOrdersRecoverCh chan struct{} + activeOrdersRecoverC chan struct{} // this ensures that bbgo.Sync to lock the object sync.Mutex @@ -899,7 +898,6 @@ func (s *Strategy) newOrderUpdateHandler(ctx context.Context, session *bbgo.Exch s.handleOrderFilled(o) // sync the profits to redis - s.debugGridProfitStats("OrderUpdate") bbgo.Sync(ctx, s) s.updateGridNumOfOrdersMetricsWithLock() @@ -1018,7 +1016,6 @@ func (s *Strategy) CloseGrid(ctx context.Context) error { defer s.EmitGridClosed() - s.debugGridProfitStats("CloseGrid") bbgo.Sync(ctx, s) // now we can cancel the open orders @@ -1171,7 +1168,6 @@ func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession) if len(orderIds) > 0 { s.GridProfitStats.InitialOrderID = orderIds[0] - s.debugGridProfitStats("openGrid") bbgo.Sync(ctx, s) } @@ -1272,23 +1268,6 @@ func (s *Strategy) debugOrders(desc string, orders []types.Order) { s.logger.Infof(sb.String()) } -func (s *Strategy) debugGridProfitStats(trigger string) { - if !s.Debug { - return - } - - stats := *s.GridProfitStats - // ProfitEntries may have too many profits, make it nil to readable - stats.ProfitEntries = nil - b, err := json.Marshal(stats) - if err != nil { - s.logger.WithError(err).Errorf("[%s] failed to debug grid profit stats", trigger) - return - } - - s.logger.Infof("trigger %s => grid profit stats : %s", trigger, string(b)) -} - func (s *Strategy) debugLog(format string, args ...interface{}) { if !s.Debug { return @@ -1883,7 +1862,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.GridProfitStats.AddTrade(trade) }) orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - s.debugGridProfitStats("OnPositionUpdate") bbgo.Sync(ctx, s) }) orderExecutor.ActiveMakerOrders().OnFilled(s.newOrderUpdateHandler(ctx, session)) @@ -1987,7 +1965,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } func (s *Strategy) startProcess(ctx context.Context, session *bbgo.ExchangeSession) error { - s.debugGridProfitStats("startProcess") if s.RecoverOrdersWhenStart { // do recover only when triggerPrice is not set and not in the back-test mode s.logger.Infof("recoverWhenStart is set, trying to recover grid orders...") From 92396cae5e383652446c33ca1bec93c58bc5cfe5 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 18 Oct 2023 15:36:53 +0800 Subject: [PATCH 092/422] bbgo: check symbol length for injection --- pkg/bbgo/trader.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index edc58588a1..6c21f1a4d7 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -168,7 +168,9 @@ func (trader *Trader) SetRiskControls(riskControls *RiskControls) { trader.riskControls = riskControls } -func (trader *Trader) RunSingleExchangeStrategy(ctx context.Context, strategy SingleExchangeStrategy, session *ExchangeSession, orderExecutor OrderExecutor) error { +func (trader *Trader) RunSingleExchangeStrategy( + ctx context.Context, strategy SingleExchangeStrategy, session *ExchangeSession, orderExecutor OrderExecutor, +) error { if v, ok := strategy.(StrategyValidator); ok { if err := v.Validate(); err != nil { return fmt.Errorf("failed to validate the config: %w", err) @@ -254,7 +256,7 @@ func (trader *Trader) injectFieldsAndSubscribe(ctx context.Context) error { log.Errorf("strategy %s does not implement ExchangeSessionSubscriber", strategy.ID()) } - if symbol, ok := dynamic.LookupSymbolField(rs); ok { + if symbol, ok := dynamic.LookupSymbolField(rs); ok && symbol != "" { log.Infof("found symbol %s based strategy from %s", symbol, rs.Type()) if err := session.initSymbol(ctx, trader.environment, symbol); err != nil { From 900db74fb93f4728f80926a5399656ad0527e993 Mon Sep 17 00:00:00 2001 From: narumi Date: Thu, 19 Oct 2023 15:14:28 +0800 Subject: [PATCH 093/422] skip public session --- pkg/strategy/xnav/strategy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/strategy/xnav/strategy.go b/pkg/strategy/xnav/strategy.go index b4b33cab70..91e3bc22fe 100644 --- a/pkg/strategy/xnav/strategy.go +++ b/pkg/strategy/xnav/strategy.go @@ -82,6 +82,11 @@ func (s *Strategy) recordNetAssetValue(ctx context.Context, sessions map[string] // iterate the sessions and record them quoteCurrency := "USDT" for sessionName, session := range sessions { + if session.PublicOnly { + log.Infof("session %s is public only, skip", sessionName) + continue + } + // update the account balances and the margin information if _, err := session.UpdateAccount(ctx); err != nil { log.WithError(err).Errorf("can not update account") From 51d86ca0593281e4536664bfee48b7c848b53b71 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 18 Oct 2023 11:25:50 +0800 Subject: [PATCH 094/422] pkg/exchange, types: support book stream on bitget --- pkg/exchange/bitget/stream.go | 165 +++++++++++++ pkg/exchange/bitget/stream_callbacks.go | 15 ++ pkg/exchange/bitget/stream_test.go | 313 ++++++++++++++++++++++++ pkg/exchange/bitget/types.go | 138 +++++++++++ pkg/types/stream.go | 1 + 5 files changed, 632 insertions(+) create mode 100644 pkg/exchange/bitget/stream.go create mode 100644 pkg/exchange/bitget/stream_callbacks.go create mode 100644 pkg/exchange/bitget/stream_test.go create mode 100644 pkg/exchange/bitget/types.go diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go new file mode 100644 index 0000000000..c29d89aaa2 --- /dev/null +++ b/pkg/exchange/bitget/stream.go @@ -0,0 +1,165 @@ +package bitget + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate callbackgen -type Stream +type Stream struct { + types.StandardStream + + bookEventCallbacks []func(o BookEvent) +} + +func NewStream() *Stream { + stream := &Stream{ + StandardStream: types.NewStandardStream(), + } + + stream.SetEndpointCreator(stream.createEndpoint) + stream.SetParser(parseWebSocketEvent) + stream.SetDispatcher(stream.dispatchEvent) + stream.OnConnect(stream.handlerConnect) + + stream.OnBookEvent(stream.handleBookEvent) + return stream +} + +func (s *Stream) syncSubscriptions(opType WsEventType) error { + if opType != WsEventUnsubscribe && opType != WsEventSubscribe { + return fmt.Errorf("unexpected subscription type: %v", opType) + } + + logger := log.WithField("opType", opType) + args := []WsArg{} + for _, subscription := range s.Subscriptions { + arg, err := convertSubscription(subscription) + if err != nil { + logger.WithError(err).Errorf("convert error, subscription: %+v", subscription) + return err + } + + args = append(args, arg) + } + + logger.Infof("%s channels: %+v", opType, args) + if err := s.Conn.WriteJSON(WsOp{ + Op: opType, + Args: args, + }); err != nil { + logger.WithError(err).Error("failed to send request") + return err + } + + return nil +} + +func (s *Stream) Unsubscribe() { + // errors are handled in the syncSubscriptions, so they are skipped here. + _ = s.syncSubscriptions(WsEventUnsubscribe) + s.Resubscribe(func(old []types.Subscription) (new []types.Subscription, err error) { + // clear the subscriptions + return []types.Subscription{}, nil + }) +} + +func (s *Stream) createEndpoint(_ context.Context) (string, error) { + var url string + if s.PublicOnly { + url = bitgetapi.PublicWebSocketURL + } else { + url = bitgetapi.PrivateWebSocketURL + } + return url, nil +} + +func (s *Stream) dispatchEvent(event interface{}) { + switch e := event.(type) { + case *WsEvent: + if err := e.IsValid(); err != nil { + log.Errorf("invalid event: %v", err) + } + + case *BookEvent: + s.EmitBookEvent(*e) + } +} + +func (s *Stream) handlerConnect() { + if s.PublicOnly { + // errors are handled in the syncSubscriptions, so they are skipped here. + _ = s.syncSubscriptions(WsEventSubscribe) + } else { + log.Error("*** PRIVATE API NOT IMPLEMENTED ***") + } +} + +func (s *Stream) handleBookEvent(o BookEvent) { + for _, book := range o.ToGlobalOrderBooks() { + switch o.Type { + case ActionTypeSnapshot: + s.EmitBookSnapshot(book) + + case ActionTypeUpdate: + s.EmitBookUpdate(book) + } + } +} + +func convertSubscription(sub types.Subscription) (WsArg, error) { + arg := WsArg{ + // support spot only + InstType: instSp, + Channel: "", + InstId: sub.Symbol, + } + + switch sub.Channel { + case types.BookChannel: + arg.Channel = ChannelOrderBook5 + + switch sub.Options.Depth { + case types.DepthLevel15: + arg.Channel = ChannelOrderBook15 + case types.DepthLevel200: + log.Warn("*** The subscription events for the order book may return fewer than 200 bids/asks at a depth of 200. ***") + arg.Channel = ChannelOrderBook + } + return arg, nil + } + + return arg, fmt.Errorf("unsupported stream channel: %s", sub.Channel) +} + +func parseWebSocketEvent(in []byte) (interface{}, error) { + var event WsEvent + + err := json.Unmarshal(in, &event) + if err != nil { + return nil, err + } + + if event.IsOp() { + return &event, nil + } + + switch event.Arg.Channel { + case ChannelOrderBook, ChannelOrderBook5, ChannelOrderBook15: + var book BookEvent + err = json.Unmarshal(event.Data, &book.Events) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into BookEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + book.Type = event.Action + book.InstId = event.Arg.InstId + return &book, nil + } + + return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go new file mode 100644 index 0000000000..3908bfac8c --- /dev/null +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -0,0 +1,15 @@ +// Code generated by "callbackgen -type Stream"; DO NOT EDIT. + +package bitget + +import () + +func (s *Stream) OnBookEvent(cb func(o BookEvent)) { + s.bookEventCallbacks = append(s.bookEventCallbacks, cb) +} + +func (s *Stream) EmitBookEvent(o BookEvent) { + for _, cb := range s.bookEventCallbacks { + cb(o) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go new file mode 100644 index 0000000000..477922afda --- /dev/null +++ b/pkg/exchange/bitget/stream_test.go @@ -0,0 +1,313 @@ +package bitget + +import ( + "context" + "fmt" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func getTestClientOrSkip(t *testing.T) *Stream { + if b, _ := strconv.ParseBool(os.Getenv("CI")); b { + t.Skip("skip test for CI") + } + + return NewStream() +} + +func TestStream(t *testing.T) { + t.Skip() + s := getTestClientOrSkip(t) + + symbols := []string{ + "BTCUSDT", + "ETHUSDT", + "DOTUSDT", + "ADAUSDT", + "AAVEUSDT", + "APTUSDT", + "ATOMUSDT", + "AXSUSDT", + "BNBUSDT", + "SOLUSDT", + "DOGEUSDT", + } + + t.Run("book test", func(t *testing.T) { + s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ + Depth: types.DepthLevel5, + }) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBookSnapshot(func(book types.SliceOrderBook) { + t.Log("got snapshot", len(book.Bids), len(book.Asks), book.Symbol, book.Time, book) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", len(book.Bids), len(book.Asks), book.Symbol, book.Time, book) + }) + c := make(chan struct{}) + <-c + }) + + t.Run("book test on unsubscribe and reconnect", func(t *testing.T) { + for _, symbol := range symbols { + s.Subscribe(types.BookChannel, symbol, types.SubscribeOptions{ + Depth: types.DepthLevel200, + }) + } + + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBookSnapshot(func(book types.SliceOrderBook) { + t.Log("got snapshot", book) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", book) + }) + + <-time.After(2 * time.Second) + + s.Unsubscribe() + for _, symbol := range symbols { + s.Subscribe(types.BookChannel, symbol, types.SubscribeOptions{ + Depth: types.DepthLevel200, + }) + } + + <-time.After(2 * time.Second) + + s.Reconnect() + + c := make(chan struct{}) + <-c + }) +} + +func TestStream_parseWebSocketEvent(t *testing.T) { + t.Run("op subscribe event", func(t *testing.T) { + input := `{ + "event":"subscribe", + "arg":{ + "instType":"sp", + "channel":"books5", + "instId":"BTCUSDT" + } + }` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WsEvent) + assert.True(t, ok) + assert.Equal(t, WsEvent{ + Event: WsEventSubscribe, + Arg: WsArg{ + InstType: instSp, + Channel: ChannelOrderBook5, + InstId: "BTCUSDT", + }, + }, *opEvent) + + assert.NoError(t, opEvent.IsValid()) + }) + + t.Run("op unsubscribe event", func(t *testing.T) { + input := `{ + "event":"unsubscribe", + "arg":{ + "instType":"sp", + "channel":"books5", + "instId":"BTCUSDT" + } + }` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WsEvent) + assert.True(t, ok) + assert.Equal(t, WsEvent{ + Event: WsEventUnsubscribe, + Arg: WsArg{ + InstType: instSp, + Channel: ChannelOrderBook5, + InstId: "BTCUSDT", + }, + }, *opEvent) + }) + + t.Run("op error event", func(t *testing.T) { + input := `{ + "event":"error", + "arg":{ + "instType":"sp", + "channel":"books5", + "instId":"BTCUSDT-" + }, + "code":30001, + "msg":"instType:sp,channel:books5,instId:BTCUSDT- doesn't exist", + "op":"subscribe" + }` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WsEvent) + assert.True(t, ok) + assert.Equal(t, WsEvent{ + Event: WsEventError, + Code: 30001, + Msg: "instType:sp,channel:books5,instId:BTCUSDT- doesn't exist", + Op: "subscribe", + Arg: WsArg{ + InstType: instSp, + Channel: ChannelOrderBook5, + InstId: "BTCUSDT-", + }, + }, *opEvent) + }) + + t.Run("Orderbook event", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"books5", + "instId":"BTCUSDT" + }, + "data":[ + { + "asks":[ + [ + "28350.78", + "0.2082" + ], + [ + "28350.80", + "0.2081" + ] + ], + "bids":[ + [ + "28350.70", + "0.5585" + ], + [ + "28350.67", + "6.8175" + ] + ], + "checksum":0, + "ts":"1697593934630" + } + ], + "ts":1697593934630 + }` + + eventFn := func(in string, actionType ActionType) { + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + book, ok := res.(*BookEvent) + assert.True(t, ok) + assert.Equal(t, BookEvent{ + Events: []struct { + Asks types.PriceVolumeSlice `json:"asks"` + // Order book on buy side, descending order + Bids types.PriceVolumeSlice `json:"bids"` + Ts types.MillisecondTimestamp `json:"ts"` + Checksum int `json:"checksum"` + }{ + { + Asks: []types.PriceVolume{ + { + Price: fixedpoint.NewFromFloat(28350.78), + Volume: fixedpoint.NewFromFloat(0.2082), + }, + { + Price: fixedpoint.NewFromFloat(28350.80), + Volume: fixedpoint.NewFromFloat(0.2081), + }, + }, + Bids: []types.PriceVolume{ + { + Price: fixedpoint.NewFromFloat(28350.70), + Volume: fixedpoint.NewFromFloat(0.5585), + }, + { + Price: fixedpoint.NewFromFloat(28350.67), + Volume: fixedpoint.NewFromFloat(6.8175), + }, + }, + Ts: types.NewMillisecondTimestampFromInt(1697593934630), + Checksum: 0, + }, + }, + Type: actionType, + InstId: "BTCUSDT", + }, *book) + } + + t.Run("snapshot type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeSnapshot) + eventFn(snapshotInput, ActionTypeSnapshot) + }) + + t.Run("update type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeUpdate) + eventFn(snapshotInput, ActionTypeUpdate) + }) + }) +} + +func Test_convertSubscription(t *testing.T) { + t.Run("BookChannel.ChannelOrderBook5", func(t *testing.T) { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.BookChannel, + Options: types.SubscribeOptions{ + Depth: types.DepthLevel5, + }, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelOrderBook5, + InstId: "BTCUSDT", + }, res) + }) + t.Run("BookChannel.DepthLevel15", func(t *testing.T) { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.BookChannel, + Options: types.SubscribeOptions{ + Depth: types.DepthLevel15, + }, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelOrderBook15, + InstId: "BTCUSDT", + }, res) + }) + t.Run("BookChannel.DepthLevel200", func(t *testing.T) { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.BookChannel, + Options: types.SubscribeOptions{ + Depth: types.DepthLevel200, + }, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelOrderBook, + InstId: "BTCUSDT", + }, res) + }) +} diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go new file mode 100644 index 0000000000..79bd95da26 --- /dev/null +++ b/pkg/exchange/bitget/types.go @@ -0,0 +1,138 @@ +package bitget + +import ( + "encoding/json" + "fmt" + + "github.com/c9s/bbgo/pkg/types" +) + +type InstType string + +const ( + instSp InstType = "sp" +) + +type ChannelType string + +const ( + // ChannelOrderBook snapshot and update might return less than 200 bids/asks as per symbol's orderbook various from + // each other; The number of bids/asks is not a fixed value and may vary in the future + ChannelOrderBook ChannelType = "books" + // ChannelOrderBook5 top 5 order book of "books" that begins from bid1/ask1 + ChannelOrderBook5 ChannelType = "books5" + // ChannelOrderBook15 top 15 order book of "books" that begins from bid1/ask1 + ChannelOrderBook15 ChannelType = "books15" +) + +type WsArg struct { + InstType InstType `json:"instType"` + Channel ChannelType `json:"channel"` + // InstId Instrument ID. e.q. BTCUSDT, ETHUSDT + InstId string `json:"instId"` +} + +type WsEventType string + +const ( + WsEventSubscribe WsEventType = "subscribe" + WsEventUnsubscribe WsEventType = "unsubscribe" + WsEventError WsEventType = "error" +) + +type WsOp struct { + Op WsEventType `json:"op"` + Args []WsArg `json:"args"` +} + +// WsEvent is the lowest level of event type. We use this struct to convert the received data, so that we will know +// whether the event belongs to `op` or `data`. +type WsEvent struct { + // for comment event + Arg WsArg `json:"arg"` + + // for op event + Event WsEventType `json:"event"` + Code int `json:"code"` + Msg string `json:"msg"` + Op string `json:"op"` + + // for data event + Action ActionType `json:"action"` + Data json.RawMessage `json:"data"` +} + +// IsOp represents the data event will be empty +func (w *WsEvent) IsOp() bool { + return w.Action == "" && len(w.Data) == 0 +} + +func (w *WsEvent) IsValid() error { + switch w.Event { + case WsEventError: + return fmt.Errorf("websocket request error, op: %s, code: %d, msg: %s", w.Op, w.Code, w.Msg) + + case WsEventSubscribe, WsEventUnsubscribe: + // Actually, this code is unnecessary because the events are either `Subscribe` or `Unsubscribe`, But to avoid bugs + // in the exchange, we still check. + if w.Code != 0 || len(w.Msg) != 0 { + return fmt.Errorf("unexpected websocket %s event, code: %d, msg: %s", w.Event, w.Code, w.Msg) + } + return nil + + default: + return fmt.Errorf("unexpected event type: %+v", w) + } +} + +type ActionType string + +const ( + ActionTypeSnapshot ActionType = "snapshot" + ActionTypeUpdate ActionType = "update" +) + +// { +// "asks":[ +// [ +// "28350.78", +// "0.2082" +// ], +// ], +// "bids":[ +// [ +// "28350.70", +// "0.5585" +// ], +// ], +// "checksum":0, +// "ts":"1697593934630" +// } +type BookEvent struct { + Events []struct { + // Order book on sell side, ascending order + Asks types.PriceVolumeSlice `json:"asks"` + // Order book on buy side, descending order + Bids types.PriceVolumeSlice `json:"bids"` + Ts types.MillisecondTimestamp `json:"ts"` + Checksum int `json:"checksum"` + } + + // internal use + Type ActionType + InstId string +} + +func (e *BookEvent) ToGlobalOrderBooks() []types.SliceOrderBook { + books := make([]types.SliceOrderBook, len(e.Events)) + for i, event := range e.Events { + books[i] = types.SliceOrderBook{ + Symbol: e.InstId, + Bids: event.Bids, + Asks: event.Asks, + Time: event.Ts.Time(), + } + } + + return books +} diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 4553183be1..f65927b615 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -523,6 +523,7 @@ const ( DepthLevelMedium Depth = "MEDIUM" DepthLevel1 Depth = "1" DepthLevel5 Depth = "5" + DepthLevel15 Depth = "15" DepthLevel20 Depth = "20" DepthLevel50 Depth = "50" DepthLevel200 Depth = "200" From c26804179db66f3993d171341f7eb99efce28330 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 20 Oct 2023 10:11:22 +0800 Subject: [PATCH 095/422] config: add logging sample config --- config/xmaker.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/xmaker.yaml b/config/xmaker.yaml index 323bf130d0..e9d10dc9ac 100644 --- a/config/xmaker.yaml +++ b/config/xmaker.yaml @@ -17,6 +17,12 @@ persistence: port: 6379 db: 0 +logging: + trade: true + order: true + fields: + env: staging + sessions: max: exchange: max From c9fca567235c6f3a4f226a1ecc438aa3dd74fdcb Mon Sep 17 00:00:00 2001 From: chiahung Date: Fri, 20 Oct 2023 15:17:31 +0800 Subject: [PATCH 096/422] MINOR: remove profit entries from profit stats --- pkg/strategy/grid2/profit_stats.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index 5f47effc6a..cd8367c23c 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -22,7 +22,6 @@ type GridProfitStats struct { TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` Volume fixedpoint.Value `json:"volume,omitempty"` Market types.Market `json:"market,omitempty"` - ProfitEntries []*GridProfit `json:"profitEntries,omitempty"` Since *time.Time `json:"since,omitempty"` InitialOrderID uint64 `json:"initialOrderID"` } @@ -38,7 +37,6 @@ func newGridProfitStats(market types.Market) *GridProfitStats { TotalFee: make(map[string]fixedpoint.Value), Volume: fixedpoint.Zero, Market: market, - ProfitEntries: nil, } } @@ -69,8 +67,6 @@ func (s *GridProfitStats) AddProfit(profit *GridProfit) { case s.Market.BaseCurrency: s.TotalBaseProfit = s.TotalBaseProfit.Add(profit.Profit) } - - s.ProfitEntries = append(s.ProfitEntries, profit) } func (s *GridProfitStats) SlackAttachment() slack.Attachment { From e9078a71c8c62e4c7f568ed83d37f8c564123129 Mon Sep 17 00:00:00 2001 From: chiahung Date: Fri, 20 Oct 2023 16:23:18 +0800 Subject: [PATCH 097/422] FEATURE: twin orderbook --- pkg/strategy/grid2/twin_order.go | 125 ++++++++++++++++++++++++++ pkg/strategy/grid2/twin_order_test.go | 68 ++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 pkg/strategy/grid2/twin_order_test.go diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go index adeeb52634..9b9cf7e166 100644 --- a/pkg/strategy/grid2/twin_order.go +++ b/pkg/strategy/grid2/twin_order.go @@ -111,3 +111,128 @@ func (m TwinOrderMap) String() string { sb.WriteString("================== END OF PIN ORDER MAP ==================\n") return sb.String() } + +type TwinOrderBook struct { + // sort in asc order + pins []fixedpoint.Value + + // pin index, use to find the next or last pin in desc order + pinIdx map[fixedpoint.Value]int + + // orderbook + m map[fixedpoint.Value]*TwinOrder + + size int +} + +func NewTwinOrderBook(pins []Pin) *TwinOrderBook { + var v []fixedpoint.Value + for _, pin := range pins { + v = append(v, fixedpoint.Value(pin)) + } + + // sort it in asc order + sort.Slice(v, func(i, j int) bool { + return v[j].Compare(v[i]) > 0 + }) + + pinIdx := make(map[fixedpoint.Value]int) + m := make(map[fixedpoint.Value]*TwinOrder) + for i, pin := range v { + m[pin] = &TwinOrder{} + pinIdx[pin] = i + } + + ob := TwinOrderBook{ + pins: v, + pinIdx: pinIdx, + m: m, + size: 0, + } + + return &ob +} + +func (book *TwinOrderBook) String() string { + var sb strings.Builder + + sb.WriteString("================== TWIN ORDERBOOK ==================\n") + for _, pin := range book.pins { + twin := book.m[fixedpoint.Value(pin)] + twinOrder := twin.GetOrder() + sb.WriteString(fmt.Sprintf("-> %8s) %s\n", pin, twinOrder.String())) + } + sb.WriteString("================== END OF TWINORDERBOOK ==================\n") + return sb.String() +} + +func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) { + idx, exist := book.pinIdx[order.Price] + if !exist { + return fixedpoint.Zero, fmt.Errorf("the order's (%d) price (%s) is not in pins", order.OrderID, order.Price) + } + + if order.Side == types.SideTypeBuy { + idx++ + if idx >= len(book.pins) { + return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order) + } + } else if order.Side == types.SideTypeSell { + if idx == 0 { + return fixedpoint.Zero, fmt.Errorf("this order's twin order price is at zero index, %+v", order) + } + // do nothing + } else { + // should not happen + return fixedpoint.Zero, fmt.Errorf("the order's (%d) side (%s) is not supported", order.OrderID, order.Side) + } + + return book.pins[idx], nil +} + +func (book *TwinOrderBook) AddOrder(order types.Order) error { + pin, err := book.GetTwinOrderPin(order) + if err != nil { + return err + } + + twinOrder, exist := book.m[pin] + if !exist { + // should not happen + return fmt.Errorf("no any empty twin order at pins, should not happen, check it") + } + + if !twinOrder.Exist() { + book.size++ + } + twinOrder.SetOrder(order) + + return nil +} + +func (book *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder { + return book.m[pin] +} + +func (book *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) { + book.m[pin] = order +} + +func (book *TwinOrderBook) Size() int { + return book.size +} + +func (book *TwinOrderBook) EmptyTwinOrderSize() int { + return len(book.pins) - 1 - book.size +} + +func (book *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap { + orderMap := types.NewSyncOrderMap() + for _, twin := range book.m { + if twin.Exist() { + orderMap.Add(twin.GetOrder()) + } + } + + return orderMap +} diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go new file mode 100644 index 0000000000..eb75e93a17 --- /dev/null +++ b/pkg/strategy/grid2/twin_order_test.go @@ -0,0 +1,68 @@ +package grid2 + +import ( + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestTwinOrderBook(t *testing.T) { + assert := assert.New(t) + pins := []Pin{ + Pin(fixedpoint.NewFromInt(3)), + Pin(fixedpoint.NewFromInt(4)), + Pin(fixedpoint.NewFromInt(1)), + Pin(fixedpoint.NewFromInt(5)), + Pin(fixedpoint.NewFromInt(2)), + } + + book := NewTwinOrderBook(pins) + assert.Equal(0, book.Size()) + assert.Equal(4, book.EmptyTwinOrderSize()) + for _, pin := range pins { + twinOrder := book.GetTwinOrder(fixedpoint.Value(pin)) + if !assert.NotNil(twinOrder) { + continue + } + + assert.False(twinOrder.Exist()) + } + + orders := []types.Order{ + { + OrderID: 1, + SubmitOrder: types.SubmitOrder{ + Price: fixedpoint.NewFromInt(2), + Side: types.SideTypeBuy, + }, + }, + { + OrderID: 2, + SubmitOrder: types.SubmitOrder{ + Price: fixedpoint.NewFromInt(4), + Side: types.SideTypeSell, + }, + }, + } + + for _, order := range orders { + assert.NoError(book.AddOrder(order)) + } + assert.Equal(2, book.Size()) + assert.Equal(2, book.EmptyTwinOrderSize()) + + for _, order := range orders { + pin, err := book.GetTwinOrderPin(order) + if !assert.NoError(err) { + continue + } + twinOrder := book.GetTwinOrder(pin) + if !assert.True(twinOrder.Exist()) { + continue + } + + assert.Equal(order.OrderID, twinOrder.GetOrder().OrderID) + } +} From a18b1be44e20df4762ffbf7bbb9e53608821b344 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 19 Oct 2023 18:12:05 +0800 Subject: [PATCH 098/422] pkg/exchange: support market trade stream on bitget --- pkg/exchange/bitget/stream.go | 44 ++++- pkg/exchange/bitget/stream_callbacks.go | 10 ++ pkg/exchange/bitget/stream_test.go | 206 +++++++++++++++++++++++- pkg/exchange/bitget/types.go | 130 ++++++++++++++- 4 files changed, 381 insertions(+), 9 deletions(-) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index c29d89aaa2..eadf25b48a 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -13,7 +13,8 @@ import ( type Stream struct { types.StandardStream - bookEventCallbacks []func(o BookEvent) + bookEventCallbacks []func(o BookEvent) + marketTradeEventCallbacks []func(o MarketTradeEvent) } func NewStream() *Stream { @@ -27,6 +28,7 @@ func NewStream() *Stream { stream.OnConnect(stream.handlerConnect) stream.OnBookEvent(stream.handleBookEvent) + stream.OnMarketTradeEvent(stream.handleMaretTradeEvent) return stream } @@ -87,6 +89,9 @@ func (s *Stream) dispatchEvent(event interface{}) { case *BookEvent: s.EmitBookEvent(*e) + + case *MarketTradeEvent: + s.EmitMarketTradeEvent(*e) } } @@ -101,7 +106,7 @@ func (s *Stream) handlerConnect() { func (s *Stream) handleBookEvent(o BookEvent) { for _, book := range o.ToGlobalOrderBooks() { - switch o.Type { + switch o.actionType { case ActionTypeSnapshot: s.EmitBookSnapshot(book) @@ -131,6 +136,10 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { arg.Channel = ChannelOrderBook } return arg, nil + + case types.MarketTradeChannel: + arg.Channel = ChannelTrade + return arg, nil } return arg, fmt.Errorf("unsupported stream channel: %s", sub.Channel) @@ -156,10 +165,37 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { return nil, fmt.Errorf("failed to unmarshal data into BookEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) } - book.Type = event.Action - book.InstId = event.Arg.InstId + book.actionType = event.Action + book.instId = event.Arg.InstId return &book, nil + + case ChannelTrade: + var trade MarketTradeEvent + err = json.Unmarshal(event.Data, &trade.Events) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into MarketTradeEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + trade.actionType = event.Action + trade.instId = event.Arg.InstId + return &trade, nil } return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) } + +func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { + if m.actionType == ActionTypeSnapshot { + // we don't support snapshot event + return + } + for _, trade := range m.Events { + globalTrade, err := trade.ToGlobal(m.instId) + if err != nil { + log.WithError(err).Error("failed to convert to market trade") + return + } + + s.EmitMarketTrade(globalTrade) + } +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go index 3908bfac8c..01da4388f8 100644 --- a/pkg/exchange/bitget/stream_callbacks.go +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -13,3 +13,13 @@ func (s *Stream) EmitBookEvent(o BookEvent) { cb(o) } } + +func (s *Stream) OnMarketTradeEvent(cb func(o MarketTradeEvent)) { + s.marketTradeEventCallbacks = append(s.marketTradeEventCallbacks, cb) +} + +func (s *Stream) EmitMarketTradeEvent(o MarketTradeEvent) { + for _, cb := range s.marketTradeEventCallbacks { + cb(o) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index 477922afda..c25fcd9426 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -92,6 +92,20 @@ func TestStream(t *testing.T) { c := make(chan struct{}) <-c }) + + t.Run("trade test", func(t *testing.T) { + s.Subscribe(types.MarketTradeChannel, "BTCUSDT", types.SubscribeOptions{}) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnMarketTrade(func(trade types.Trade) { + t.Log("got update", trade) + }) + c := make(chan struct{}) + <-c + }) + } func TestStream_parseWebSocketEvent(t *testing.T) { @@ -247,8 +261,8 @@ func TestStream_parseWebSocketEvent(t *testing.T) { Checksum: 0, }, }, - Type: actionType, - InstId: "BTCUSDT", + actionType: actionType, + instId: "BTCUSDT", }, *book) } @@ -264,6 +278,181 @@ func TestStream_parseWebSocketEvent(t *testing.T) { }) } +func Test_parseWebSocketEvent_MarketTrade(t *testing.T) { + t.Run("MarketTrade event", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "1697697791663", + "28303.43", + "0.0452", + "sell" + ], + [ + "1697697794663", + "28345.67", + "0.1234", + "sell" + ] + ], + "ts":1697697791670 + }` + + eventFn := func(in string, actionType ActionType) { + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + book, ok := res.(*MarketTradeEvent) + assert.True(t, ok) + assert.Equal(t, MarketTradeEvent{ + Events: MarketTradeSlice{ + { + Ts: types.NewMillisecondTimestampFromInt(1697697791663), + Price: fixedpoint.NewFromFloat(28303.43), + Size: fixedpoint.NewFromFloat(0.0452), + Side: "sell", + }, + + { + Ts: types.NewMillisecondTimestampFromInt(1697697794663), + Price: fixedpoint.NewFromFloat(28345.67), + Size: fixedpoint.NewFromFloat(0.1234), + Side: "sell", + }, + }, + actionType: actionType, + instId: "BTCUSDT", + }, *book) + } + + t.Run("snapshot type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeSnapshot) + eventFn(snapshotInput, ActionTypeSnapshot) + }) + + t.Run("update type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeUpdate) + eventFn(snapshotInput, ActionTypeUpdate) + }) + }) + + t.Run("Unexpected length of market trade", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "1697697791663", + "28303.43", + "28303.43", + "0.0452", + "sell" + ] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "unexpected trades length") + }) + + t.Run("Unexpected timestamp", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "TIMESTAMP", + "28303.43", + "0.0452", + "sell" + ] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "timestamp") + }) + + t.Run("Unexpected price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "1697697791663", + "1p", + "0.0452", + "sell" + ] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "price") + }) + + t.Run("Unexpected size", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "1697697791663", + "28303.43", + "2v", + "sell" + ] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "size") + }) + + t.Run("Unexpected side", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"trade", + "instId":"BTCUSDT" + }, + "data":[ + [ + "1697697791663", + "28303.43", + "0.0452", + 12345 + ] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "side") + }) +} + func Test_convertSubscription(t *testing.T) { t.Run("BookChannel.ChannelOrderBook5", func(t *testing.T) { res, err := convertSubscription(types.Subscription{ @@ -310,4 +499,17 @@ func Test_convertSubscription(t *testing.T) { InstId: "BTCUSDT", }, res) }) + t.Run("TradeChannel", func(t *testing.T) { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.MarketTradeChannel, + Options: types.SubscribeOptions{}, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelTrade, + InstId: "BTCUSDT", + }, res) + }) } diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index 79bd95da26..df75f5bf25 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -2,8 +2,10 @@ package bitget import ( "encoding/json" + "errors" "fmt" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -23,6 +25,7 @@ const ( ChannelOrderBook5 ChannelType = "books5" // ChannelOrderBook15 top 15 order book of "books" that begins from bid1/ask1 ChannelOrderBook15 ChannelType = "books15" + ChannelTrade ChannelType = "trade" ) type WsArg struct { @@ -119,15 +122,15 @@ type BookEvent struct { } // internal use - Type ActionType - InstId string + actionType ActionType + instId string } func (e *BookEvent) ToGlobalOrderBooks() []types.SliceOrderBook { books := make([]types.SliceOrderBook, len(e.Events)) for i, event := range e.Events { books[i] = types.SliceOrderBook{ - Symbol: e.InstId, + Symbol: e.instId, Bids: event.Bids, Asks: event.Asks, Time: event.Ts.Time(), @@ -136,3 +139,124 @@ func (e *BookEvent) ToGlobalOrderBooks() []types.SliceOrderBook { return books } + +type SideType string + +const ( + SideBuy SideType = "buy" + SideSell SideType = "sell" +) + +func (s SideType) ToGlobal() (types.SideType, error) { + switch s { + case SideBuy: + return types.SideTypeBuy, nil + case SideSell: + return types.SideTypeSell, nil + default: + return "", fmt.Errorf("unexpceted side type: %s", s) + } +} + +type MarketTrade struct { + Ts types.MillisecondTimestamp + Price fixedpoint.Value + Size fixedpoint.Value + Side SideType +} + +type MarketTradeSlice []MarketTrade + +func (m *MarketTradeSlice) UnmarshalJSON(b []byte) error { + if m == nil { + return errors.New("nil pointer of market trade slice") + } + s, err := parseMarketTradeSliceJSON(b) + if err != nil { + return err + } + + *m = s + return nil +} + +// ParseMarketTradeSliceJSON tries to parse a 2 dimensional string array into a MarketTradeSlice +// +// [ +// +// [ +// "1697694819663", +// "28312.97", +// "0.1653", +// "sell" +// ], +// [ +// "1697694818663", +// "28313", +// "0.1598", +// "buy" +// ] +// +// ] +func parseMarketTradeSliceJSON(in []byte) (slice MarketTradeSlice, err error) { + var rawTrades [][]json.RawMessage + + err = json.Unmarshal(in, &rawTrades) + if err != nil { + return slice, err + } + + for _, raw := range rawTrades { + if len(raw) != 4 { + return nil, fmt.Errorf("unexpected trades length: %d, data: %q", len(raw), raw) + } + var trade MarketTrade + if err = json.Unmarshal(raw[0], &trade.Ts); err != nil { + return nil, fmt.Errorf("failed to unmarshal into timestamp: %q", raw[0]) + } + if err = json.Unmarshal(raw[1], &trade.Price); err != nil { + return nil, fmt.Errorf("failed to unmarshal into price: %q", raw[1]) + } + if err = json.Unmarshal(raw[2], &trade.Size); err != nil { + return nil, fmt.Errorf("failed to unmarshal into size: %q", raw[2]) + } + if err = json.Unmarshal(raw[3], &trade.Side); err != nil { + return nil, fmt.Errorf("failed to unmarshal into side: %q", raw[3]) + } + + slice = append(slice, trade) + } + + return slice, nil +} + +func (m MarketTrade) ToGlobal(symbol string) (types.Trade, error) { + side, err := m.Side.ToGlobal() + if err != nil { + return types.Trade{}, err + } + + return types.Trade{ + ID: 0, // not supported + OrderID: 0, // not supported + Exchange: types.ExchangeBitget, + Price: m.Price, + Quantity: m.Size, + QuoteQuantity: m.Price.Mul(m.Size), + Symbol: symbol, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: false, // not supported + Time: types.Time(m.Ts.Time()), + Fee: fixedpoint.Zero, // not supported + FeeCurrency: "", // not supported + }, nil +} + +type MarketTradeEvent struct { + Events MarketTradeSlice + + // internal use + actionType ActionType + instId string +} From 2ef86e4e7194ee631b5f896f702b4bd56e735db9 Mon Sep 17 00:00:00 2001 From: Samiksha Mishra <38784342+mishrasamiksha@users.noreply.github.com> Date: Sat, 21 Oct 2023 17:51:33 +0530 Subject: [PATCH 099/422] Update README.md Corrected typo error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b947f5acad..eb0011a006 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ the implementation. | irr | this strategy opens the position based on the predicated return rate | long/short | | | bollmaker | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker | | | wall | this strategy creates wall (large amount order) on the order book | maker | no | -| scmaker | this market making strategy is desgiend for stable coin markets, like USDC/USDT | maker | | +| scmaker | this market making strategy is designed for stable coin markets, like USDC/USDT | maker | | | drift | | long/short | | | rsicross | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator | long/short | | | marketcap | this strategy implements a strategy that rebalances the portfolio based on the market capitalization | rebalance | no | From da5f8f57f36155a4005d406c4a2c69c702c0c9be Mon Sep 17 00:00:00 2001 From: Sakshi Umredkar <128240967+saakshii12@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:36:08 +0530 Subject: [PATCH 100/422] Fixed a typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b947f5acad..7f84135aee 100644 --- a/README.md +++ b/README.md @@ -488,7 +488,7 @@ See also: ## Command Usages -### Submitting Orders to a specific exchagne session +### Submitting Orders to a specific exchange session ```shell bbgo submit-order --session=okex --symbol=OKBUSDT --side=buy --price=10.0 --quantity=1 From 3150f6b3f57504486560632b58a864584b3658b0 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 23 Oct 2023 13:00:17 +0800 Subject: [PATCH 101/422] fix --- pkg/strategy/grid2/twin_order.go | 48 +++++++++++++-------------- pkg/strategy/grid2/twin_order_test.go | 2 +- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go index 9b9cf7e166..bfdf577214 100644 --- a/pkg/strategy/grid2/twin_order.go +++ b/pkg/strategy/grid2/twin_order.go @@ -125,7 +125,7 @@ type TwinOrderBook struct { size int } -func NewTwinOrderBook(pins []Pin) *TwinOrderBook { +func newTwinOrderBook(pins []Pin) *TwinOrderBook { var v []fixedpoint.Value for _, pin := range pins { v = append(v, fixedpoint.Value(pin)) @@ -143,22 +143,20 @@ func NewTwinOrderBook(pins []Pin) *TwinOrderBook { pinIdx[pin] = i } - ob := TwinOrderBook{ + return &TwinOrderBook{ pins: v, pinIdx: pinIdx, m: m, size: 0, } - - return &ob } -func (book *TwinOrderBook) String() string { +func (b *TwinOrderBook) String() string { var sb strings.Builder sb.WriteString("================== TWIN ORDERBOOK ==================\n") - for _, pin := range book.pins { - twin := book.m[fixedpoint.Value(pin)] + for _, pin := range b.pins { + twin := b.m[fixedpoint.Value(pin)] twinOrder := twin.GetOrder() sb.WriteString(fmt.Sprintf("-> %8s) %s\n", pin, twinOrder.String())) } @@ -166,15 +164,15 @@ func (book *TwinOrderBook) String() string { return sb.String() } -func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) { - idx, exist := book.pinIdx[order.Price] +func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) { + idx, exist := b.pinIdx[order.Price] if !exist { return fixedpoint.Zero, fmt.Errorf("the order's (%d) price (%s) is not in pins", order.OrderID, order.Price) } if order.Side == types.SideTypeBuy { idx++ - if idx >= len(book.pins) { + if idx >= len(b.pins) { return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order) } } else if order.Side == types.SideTypeSell { @@ -187,48 +185,48 @@ func (book *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, return fixedpoint.Zero, fmt.Errorf("the order's (%d) side (%s) is not supported", order.OrderID, order.Side) } - return book.pins[idx], nil + return b.pins[idx], nil } -func (book *TwinOrderBook) AddOrder(order types.Order) error { - pin, err := book.GetTwinOrderPin(order) +func (b *TwinOrderBook) AddOrder(order types.Order) error { + pin, err := b.GetTwinOrderPin(order) if err != nil { return err } - twinOrder, exist := book.m[pin] + twinOrder, exist := b.m[pin] if !exist { // should not happen return fmt.Errorf("no any empty twin order at pins, should not happen, check it") } if !twinOrder.Exist() { - book.size++ + b.size++ } twinOrder.SetOrder(order) return nil } -func (book *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder { - return book.m[pin] +func (b *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder { + return b.m[pin] } -func (book *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) { - book.m[pin] = order +func (b *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) { + b.m[pin] = order } -func (book *TwinOrderBook) Size() int { - return book.size +func (b *TwinOrderBook) Size() int { + return b.size } -func (book *TwinOrderBook) EmptyTwinOrderSize() int { - return len(book.pins) - 1 - book.size +func (b *TwinOrderBook) EmptyTwinOrderSize() int { + return len(b.pins) - 1 - b.size } -func (book *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap { +func (b *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap { orderMap := types.NewSyncOrderMap() - for _, twin := range book.m { + for _, twin := range b.m { if twin.Exist() { orderMap.Add(twin.GetOrder()) } diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go index eb75e93a17..d6204ee941 100644 --- a/pkg/strategy/grid2/twin_order_test.go +++ b/pkg/strategy/grid2/twin_order_test.go @@ -18,7 +18,7 @@ func TestTwinOrderBook(t *testing.T) { Pin(fixedpoint.NewFromInt(2)), } - book := NewTwinOrderBook(pins) + book := newTwinOrderBook(pins) assert.Equal(0, book.Size()) assert.Equal(4, book.EmptyTwinOrderSize()) for _, pin := range pins { From 6b5dd653aa69aabfee67d72df12c76256e369d17 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 23 Oct 2023 14:38:56 +0800 Subject: [PATCH 102/422] go: update requestgen to v1.3.5 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4ebe211fa6..91eca5f8be 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/squirrel v1.5.3 github.com/adshao/go-binance/v2 v2.4.2 github.com/c-bata/goptuna v0.8.1 - github.com/c9s/requestgen v1.3.4 + github.com/c9s/requestgen v1.3.5 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b github.com/cenkalti/backoff/v4 v4.2.0 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index 448c0a2145..710eba662d 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/c-bata/goptuna v0.8.1 h1:25+n1MLv0yvCsD56xv4nqIus3oLHL9GuPAZDLIqmX1U= github.com/c-bata/goptuna v0.8.1/go.mod h1:knmS8+Iyq5PPy1YUeIEq0pMFR4Y6x7z/CySc9HlZTCY= github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY= github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4= +github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs= +github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b/go.mod h1:EKObf66Cp7erWxym2de+07qNN5T1N9PXxHdh97N44EQ= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= From c977b8e295cf85a190df22bd72719447280f27e9 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 23 Oct 2023 17:42:39 +0800 Subject: [PATCH 103/422] add lock to protect twin orderbook and add more comments --- pkg/strategy/grid2/twin_order.go | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go index bfdf577214..ccc9bfa4fd 100644 --- a/pkg/strategy/grid2/twin_order.go +++ b/pkg/strategy/grid2/twin_order.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -112,7 +113,20 @@ func (m TwinOrderMap) String() string { return sb.String() } +// TwinOrderBook is to verify grid +// For grid trading, there are twin orders between a grid +// e.g. 100, 200, 300, 400, 500 +// BUY 100 and SELL 200 are a twin. +// BUY 200 and SELL 300 are a twin. +// Because they can't be placed on orderbook at the same time. +// We use sell price to be the twin orderbook's key +// New the twin orderbook with pins, and it will sort the pins in asc order. +// There must be a non nil TwinOrder on the every pin (except the first one). +// But the TwinOrder.Exist() may be false. It means there is no twin order on this grid type TwinOrderBook struct { + // used to protect orderbook update + mu sync.Mutex + // sort in asc order pins []fixedpoint.Value @@ -122,6 +136,7 @@ type TwinOrderBook struct { // orderbook m map[fixedpoint.Value]*TwinOrder + // size is the amount on twin orderbook size int } @@ -171,6 +186,17 @@ func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, er } if order.Side == types.SideTypeBuy { + // we use sell price as twin orderbook's key, so if the order's side is buy. + // we need to find its next price on grid. + // e.g. + // BUY 100 <- twin -> SELL 200 + // BUY 200 <- twin -> SELL 300 + // BUY 300 <- twin -> SELL 400 + // BUY 400 <- twin -> SELL 500 + // if the order is BUY 100, we need to find its twin order's price to be the twin orderbook's key + // so we plus 1 here and use sorted pins to find the next price (200) + // there must no BUY 500 in the grid, so we need to make sure the idx should always not over the len(pins) + // also, there must no SELL 100 in the grid, so we need to make sure the idx should always not be 0 idx++ if idx >= len(b.pins) { return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order) @@ -189,20 +215,30 @@ func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, er } func (b *TwinOrderBook) AddOrder(order types.Order) error { + b.mu.Lock() + defer b.mu.Unlock() + pin, err := b.GetTwinOrderPin(order) if err != nil { return err } + // At all the pins, we already create the empty TwinOrder{} + // As a result,if the exist is false, it means the pin is not in the twin orderbook. + // That's invalid pin, or we have something wrong when new TwinOrderBook twinOrder, exist := b.m[pin] if !exist { // should not happen return fmt.Errorf("no any empty twin order at pins, should not happen, check it") } + // Exist == false means there is no twin order on this pin if !twinOrder.Exist() { b.size++ } + if b.size >= len(b.pins) { + return fmt.Errorf("the maximum size of twin orderbook is len(pins) - 1, need to check it") + } twinOrder.SetOrder(order) return nil @@ -213,14 +249,20 @@ func (b *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder { } func (b *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) { + b.mu.Lock() + defer b.mu.Unlock() + b.m[pin] = order } +// Size is the valid twin order on grid. func (b *TwinOrderBook) Size() int { return b.size } +// EmptyTwinOrderSize is the amount of grid there is no twin order on it. func (b *TwinOrderBook) EmptyTwinOrderSize() int { + // for grid, there is only pins - 1 order on the grid, so we need to minus 1. return len(b.pins) - 1 - b.size } From 3710c33670509d0dfcda6309b6a9aec3907bcfcc Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 23 Oct 2023 13:17:20 +0800 Subject: [PATCH 104/422] REFACTOR: rename file and variable --- pkg/strategy/grid2/active_order_recover.go | 12 +-- .../grid2/active_order_recover_test.go | 74 +++++++++++++++++++ .../grid2/{recover.go => grid_recover.go} | 0 .../{recover_test.go => grid_recover_test.go} | 0 pkg/strategy/grid2/strategy.go | 2 +- 5 files changed, 81 insertions(+), 7 deletions(-) rename pkg/strategy/grid2/{recover.go => grid_recover.go} (100%) rename pkg/strategy/grid2/{recover_test.go => grid_recover_test.go} (100%) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index e93e722f01..2ffdbb43c1 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -23,21 +23,21 @@ type SyncActiveOrdersOpts struct { exchange types.Exchange } -func (s *Strategy) initializeRecoverCh() bool { +func (s *Strategy) initializeRecoverC() bool { s.mu.Lock() defer s.mu.Unlock() isInitialize := false - if s.activeOrdersRecoverC == nil { + if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.activeOrdersRecoverC = make(chan struct{}, 1) + s.recoverC = make(chan struct{}, 1) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true select { - case s.activeOrdersRecoverC <- struct{}{}: + case s.recoverC <- struct{}{}: s.logger.Info("trigger active orders recover") default: s.logger.Info("activeOrdersRecoverC is full") @@ -49,7 +49,7 @@ func (s *Strategy) initializeRecoverCh() bool { func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverC receive signal, do active orders recover - if isInitialize := s.initializeRecoverCh(); isInitialize { + if isInitialize := s.initializeRecoverC(); isInitialize { return } @@ -78,7 +78,7 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { log.WithError(err).Errorf("unable to sync active orders") } - case <-s.activeOrdersRecoverC: + case <-s.recoverC: if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") } diff --git a/pkg/strategy/grid2/active_order_recover_test.go b/pkg/strategy/grid2/active_order_recover_test.go index dffdccc388..5a72c05c03 100644 --- a/pkg/strategy/grid2/active_order_recover_test.go +++ b/pkg/strategy/grid2/active_order_recover_test.go @@ -174,3 +174,77 @@ func TestSyncActiveOrders(t *testing.T) { assert.Equal(types.OrderStatusNew, activeOrders[0].Status) }) } + +func TestSyncActiveOrder(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + + t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(0, len(activeOrders)) + }) + + t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusPartiallyFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(order.OrderID, activeOrders[0].OrderID) + assert.Equal(updatedOrder.Status, activeOrders[0].Status) + }) +} diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/grid_recover.go similarity index 100% rename from pkg/strategy/grid2/recover.go rename to pkg/strategy/grid2/grid_recover.go diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/grid_recover_test.go similarity index 100% rename from pkg/strategy/grid2/recover_test.go rename to pkg/strategy/grid2/grid_recover_test.go diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 736b1ad67c..ce3f77dc2f 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -204,7 +204,7 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc - activeOrdersRecoverC chan struct{} + recoverC chan struct{} // this ensures that bbgo.Sync to lock the object sync.Mutex From a9d9ef379268f6f1a841d3626a73d89612097276 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 24 Oct 2023 13:44:25 +0800 Subject: [PATCH 105/422] Add AddSubscriber method on Float64Series --- pkg/types/series_float64.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/types/series_float64.go b/pkg/types/series_float64.go index f12e05d584..8c07b4a13d 100644 --- a/pkg/types/series_float64.go +++ b/pkg/types/series_float64.go @@ -46,6 +46,21 @@ func (f *Float64Series) Subscribe(source Float64Source, c func(x float64)) { } } +// AddSubscriber adds the subscriber function and push historical data to the subscriber +func (f *Float64Series) AddSubscriber(fn func(v float64)) { + f.OnUpdate(fn) + + if f.Length() == 0 { + return + } + + // push historical values to the subscriber + for _, vv := range f.Slice { + fn(vv) + } +} + + // Bind binds the source event to the target (Float64Calculator) // A Float64Calculator should be able to calculate the float64 result from a single float64 argument input func (f *Float64Series) Bind(source Float64Source, target Float64Calculator) { From 4c1654652eb20eec8237e51b0c7c92fc3e921524 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 24 Oct 2023 13:44:49 +0800 Subject: [PATCH 106/422] indicator: remove unnecessary zero value push --- pkg/indicator/v2/rma.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go index 1aa08ccdc0..4a6c8b327a 100644 --- a/pkg/indicator/v2/rma.go +++ b/pkg/indicator/v2/rma.go @@ -48,11 +48,6 @@ func (s *RMAStream) Calculate(x float64) float64 { } s.counter++ - if s.counter < s.window { - // we can use x, but we need to use 0. to make the same behavior as the result from python pandas_ta - s.Slice.Push(0) - } - s.Slice.Push(tmp) s.previous = tmp return tmp From 22a7232e8b5a657f75cbd01a2b1cef49cad7b9eb Mon Sep 17 00:00:00 2001 From: narumi Date: Tue, 24 Oct 2023 16:37:44 +0800 Subject: [PATCH 107/422] fix duplicate rma value --- pkg/indicator/v2/rma.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go index 4a6c8b327a..b943cf6f8d 100644 --- a/pkg/indicator/v2/rma.go +++ b/pkg/indicator/v2/rma.go @@ -34,23 +34,20 @@ func RMA2(source types.Float64Source, window int, adjust bool) *RMAStream { func (s *RMAStream) Calculate(x float64) float64 { lambda := 1 / float64(s.window) - tmp := 0.0 if s.counter == 0 { s.sum = 1 - tmp = x + s.previous = x } else { if s.Adjust { s.sum = s.sum*(1-lambda) + 1 - tmp = s.previous + (x-s.previous)/s.sum + s.previous = s.previous + (x-s.previous)/s.sum } else { - tmp = s.previous*(1-lambda) + x*lambda + s.previous = s.previous*(1-lambda) + x*lambda } } s.counter++ - s.Slice.Push(tmp) - s.previous = tmp - return tmp + return s.previous } func (s *RMAStream) Truncate() { From 2a9fd10716b8c345b7606fdde49ec76c5eb6364a Mon Sep 17 00:00:00 2001 From: narumi Date: Tue, 24 Oct 2023 16:38:05 +0800 Subject: [PATCH 108/422] add rma test cases --- pkg/indicator/rma_test.go | 72 ++++++++++++++++++++++++++++++++++++ pkg/indicator/v2/rma_test.go | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 pkg/indicator/rma_test.go create mode 100644 pkg/indicator/v2/rma_test.go diff --git a/pkg/indicator/rma_test.go b/pkg/indicator/rma_test.go new file mode 100644 index 0000000000..6e14bcdb16 --- /dev/null +++ b/pkg/indicator/rma_test.go @@ -0,0 +1,72 @@ +package indicator + +import ( + "encoding/json" + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +/* +python + +import pandas as pd +import pandas_ta as ta + +data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84] + +close = pd.Series(data) +result = ta.rma(close, length=14) +print(result) +*/ +func Test_RMA(t *testing.T) { + var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`) + var values []fixedpoint.Value + err := json.Unmarshal(bytes, &values) + assert.NoError(t, err) + + var kLines []types.KLine + for _, p := range values { + kLines = append(kLines, types.KLine{High: p, Low: p, Close: p}) + } + + tests := []struct { + name string + window int + want []float64 + }{ + { + name: "test_binance_btcusdt_1h", + window: 14, + want: []float64{ + 40129.841000, + 40041.830291, + 39988.157743, + 39958.803719, + 39946.115094, + 39958.296741, + 39967.681562, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rma := RMA{ + IntervalWindow: types.IntervalWindow{Window: tt.window}, + Adjust: true, + } + rma.CalculateAndUpdate(kLines) + + if assert.Equal(t, len(tt.want), len(rma.Values)-tt.window+1) { + for i, v := range tt.want { + j := tt.window - 1 + i + got := rma.Values[j] + assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got) + } + } + }) + } +} diff --git a/pkg/indicator/v2/rma_test.go b/pkg/indicator/v2/rma_test.go new file mode 100644 index 0000000000..b06605f75b --- /dev/null +++ b/pkg/indicator/v2/rma_test.go @@ -0,0 +1,65 @@ +package indicatorv2 + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +/* +python + +import pandas as pd +import pandas_ta as ta + +data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84] + +close = pd.Series(data) +result = ta.rma(close, length=14) +print(result) +*/ +func Test_RMA2(t *testing.T) { + var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`) + var values []float64 + err := json.Unmarshal(bytes, &values) + assert.NoError(t, err) + + prices := ClosePrices(nil) + for _, v := range values { + prices.Push(v) + } + + tests := []struct { + name string + window int + want []float64 + }{ + { + name: "test_binance_btcusdt_1h", + window: 14, + want: []float64{ + 40129.841000, + 40041.830291, + 39988.157743, + 39958.803719, + 39946.115094, + 39958.296741, + 39967.681562, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rma := RMA2(prices, tt.window, true) + if assert.Equal(t, len(tt.want), len(rma.Slice)-tt.window+1) { + for i, v := range tt.want { + j := tt.window - 1 + i + got := rma.Slice[j] + assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got) + } + } + }) + } +} From 3e5869cab37f0dd45e1472ff258b3104dbfd5594 Mon Sep 17 00:00:00 2001 From: narumi Date: Tue, 24 Oct 2023 17:03:40 +0800 Subject: [PATCH 109/422] remove zero padding from RMA --- pkg/indicator/rma.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/indicator/rma.go b/pkg/indicator/rma.go index a06ad18af9..69a7f4fbc9 100644 --- a/pkg/indicator/rma.go +++ b/pkg/indicator/rma.go @@ -67,11 +67,6 @@ func (inc *RMA) Update(x float64) { } inc.counter++ - if inc.counter < inc.Window { - inc.Values.Push(0) - return - } - inc.Values.Push(inc.tmp) if len(inc.Values) > MaxNumOfRMA { inc.Values = inc.Values[MaxNumOfRMATruncateSize-1:] From 75575792e706f9139b02274cd961858a8da96bc5 Mon Sep 17 00:00:00 2001 From: Saksham Bhugra <85192629+sh4d0wy@users.noreply.github.com> Date: Tue, 24 Oct 2023 22:40:47 +0530 Subject: [PATCH 110/422] Update CONTRIBUTING.md --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c92962cdcb..f25a27df09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,19 +31,19 @@ Install pre-commit to check your changes before you commit: See for more details. -For new large features, such as integrating binance futures contracts, please propose a discussion first before you start working on it. +For new large features, such as integrating Binance futures contracts, please propose a discussion first before you start working on it. For new small features, you could open a pull request directly. For each contributor, you have chance to receive the BBG token through the polygon network. -Each issue has its BBG label, by completing the issue with a pull request, you can get correspond amount of BBG. +Each issue has its BBG label, by completing the issue with a pull request, you can get corresponding amount of BBG. ## Support ### By contributing pull requests -Any pull request is welcome, documentation, format fixing, testing, features. +Any pull request is welcome, documentation, format fixing, testing, and features. ### By registering account with referral ID @@ -52,7 +52,7 @@ You may register your exchange account with my referral ID to support this proje - For MAX Exchange: (default commission rate to your account) - For Binance Exchange: (5% commission back to your account) -### By small amount cryptos +### By small amount of cryptos - BTC address `3J6XQJNWT56amqz9Hz2BEVQ7W4aNmb5kiU` - USDT ERC20 address `0xeBcf7887A5b767DEb2e0C77E46A22c6Adc64E427` From ab1bc998f9b5453d57870e79b56189b46b697532 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 23 Oct 2023 15:55:55 +0800 Subject: [PATCH 111/422] FEATURE: prepare query trades funtion for new recover --- pkg/strategy/grid2/active_order_recover.go | 16 -- .../grid2/active_order_recover_test.go | 74 ------ pkg/strategy/grid2/recover.go | 143 +++++++++++ pkg/strategy/grid2/recover_test.go | 237 ++++++++++++++++++ pkg/strategy/grid2/twin_order.go | 5 +- pkg/strategy/grid2/twin_order_test.go | 5 + 6 files changed, 389 insertions(+), 91 deletions(-) create mode 100644 pkg/strategy/grid2/recover.go create mode 100644 pkg/strategy/grid2/recover_test.go diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 2ffdbb43c1..5042084fc8 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,7 +2,6 @@ package grid2 import ( "context" - "strconv" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -142,18 +141,3 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { return errs } - -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { - updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ - Symbol: activeOrderBook.Symbol, - OrderID: strconv.FormatUint(orderID, 10), - }) - - if err != nil { - return err - } - - activeOrderBook.Update(*updatedOrder) - - return nil -} diff --git a/pkg/strategy/grid2/active_order_recover_test.go b/pkg/strategy/grid2/active_order_recover_test.go index 5a72c05c03..dffdccc388 100644 --- a/pkg/strategy/grid2/active_order_recover_test.go +++ b/pkg/strategy/grid2/active_order_recover_test.go @@ -174,77 +174,3 @@ func TestSyncActiveOrders(t *testing.T) { assert.Equal(types.OrderStatusNew, activeOrders[0].Status) }) } - -func TestSyncActiveOrder(t *testing.T) { - assert := assert.New(t) - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - symbol := "ETHUSDT" - - t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) { - mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) - activeOrderbook := bbgo.NewActiveOrderBook(symbol) - - order := types.Order{ - OrderID: 1, - Status: types.OrderStatusNew, - SubmitOrder: types.SubmitOrder{ - Symbol: symbol, - }, - } - activeOrderbook.Add(order) - - updatedOrder := order - updatedOrder.Status = types.OrderStatusFilled - - mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ - Symbol: symbol, - OrderID: strconv.FormatUint(order.OrderID, 10), - }).Return(&updatedOrder, nil) - - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { - return - } - - // verify active orderbook - activeOrders := activeOrderbook.Orders() - assert.Equal(0, len(activeOrders)) - }) - - t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) { - mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) - activeOrderbook := bbgo.NewActiveOrderBook(symbol) - - order := types.Order{ - OrderID: 1, - Status: types.OrderStatusNew, - SubmitOrder: types.SubmitOrder{ - Symbol: symbol, - }, - } - activeOrderbook.Add(order) - - updatedOrder := order - updatedOrder.Status = types.OrderStatusPartiallyFilled - - mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ - Symbol: symbol, - OrderID: strconv.FormatUint(order.OrderID, 10), - }).Return(&updatedOrder, nil) - - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { - return - } - - // verify active orderbook - activeOrders := activeOrderbook.Orders() - assert.Equal(1, len(activeOrders)) - assert.Equal(order.OrderID, activeOrders[0].OrderID) - assert.Equal(updatedOrder.Status, activeOrders[0].Status) - }) -} diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go new file mode 100644 index 0000000000..623966e607 --- /dev/null +++ b/pkg/strategy/grid2/recover.go @@ -0,0 +1,143 @@ +package grid2 + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/types" + "github.com/pkg/errors" +) + +/* + Background knowledge + 1. active orderbook add orders only when receive new order event or call Add/Update method manually + 2. active orderbook remove orders only when receive filled/cancelled event or call Remove/Update method manually + As a result + 1. at the same twin-order-price, there is order in open orders but not in active orderbook + - not receive new order event + => add order into active orderbook + 2. at the same twin-order-price, there is order in active orderbook but not in open orders + - not receive filled event + => query the filled order and call Update method + 3. at the same twin-order-price, there is no order in open orders and no order in active orderbook + - failed to create the order + => query the last order from trades to emit filled, and it will submit again + - not receive new order event and the order filled before we find it. + => query the untracked order (also is the last order) from trades to emit filled and it will submit the reversed order + 4. at the same twin-order-price, there are different orders in open orders and active orderbook + - should not happen !!! + => log error + 5. at the same twin-order-price, there is the same order in open orders and active orderbook + - normal case + => no need to do anything + After killing pod, active orderbook must be empty. we can think it is the same as not receive new event. + Process + 1. build twin orderbook with pins and open orders. + 2. build twin orderbook with pins and active orders. + 3. compare above twin orderbooks to add open orders into active orderbook and update active orders. + 4. run grid recover to make sure all the twin price has its order. +*/ + +func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error) { + book := newTwinOrderBook(pins) + + for _, order := range orders { + if err := book.AddOrder(order); err != nil { + return nil, err + } + } + + return book, nil +} + +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ + Symbol: activeOrderBook.Symbol, + OrderID: strconv.FormatUint(orderID, 10), + }) + + if err != nil { + return err + } + + activeOrderBook.Update(*updatedOrder) + + return nil +} + +func queryTradesToUpdateTwinOrderBook( + ctx context.Context, + symbol string, + twinOrderBook *TwinOrderBook, + queryTradesService types.ExchangeTradeHistoryService, + queryOrderService types.ExchangeOrderQueryService, + existedOrders *types.SyncOrderMap, + since, until time.Time, + logger func(format string, args ...interface{})) error { + if twinOrderBook == nil { + return fmt.Errorf("twin orderbook should not be nil, please check it") + } + + var fromTradeID uint64 = 0 + var limit int64 = 1000 + for { + trades, err := queryTradesService.QueryTrades(ctx, symbol, &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + LastTradeID: fromTradeID, + Limit: limit, + }) + + if err != nil { + return errors.Wrapf(err, "failed to query trades to recover the grid") + } + + if logger != nil { + logger("QueryTrades from %s <-> %s (from: %d) return %d trades", since, until, fromTradeID, len(trades)) + } + + for _, trade := range trades { + if trade.Time.After(until) { + return nil + } + + if logger != nil { + logger(trade.String()) + } + + if existedOrders.Exists(trade.OrderID) { + // already queries, skip + continue + } + order, err := retry.QueryOrderUntilSuccessful(ctx, queryOrderService, types.OrderQuery{ + Symbol: trade.Symbol, + OrderID: strconv.FormatUint(trade.OrderID, 10), + }) + + if err != nil { + return errors.Wrapf(err, "failed to query order by trade (trade id: %d, order id: %d)", trade.ID, trade.OrderID) + } + + if logger != nil { + logger(order.String()) + } + // avoid query this order again + existedOrders.Add(*order) + // add 1 to avoid duplicate + fromTradeID = trade.ID + 1 + + if err := twinOrderBook.AddOrder(*order); err != nil { + return errors.Wrapf(err, "failed to add queried order into twin orderbook") + } + } + + // stop condition + if int64(len(trades)) < limit { + return nil + } + } +} diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go new file mode 100644 index 0000000000..bdfd191eed --- /dev/null +++ b/pkg/strategy/grid2/recover_test.go @@ -0,0 +1,237 @@ +package grid2 + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/types/mocks" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func newStrategy(t *TestData) *Strategy { + s := t.Strategy + s.Debug = true + s.Initialize() + s.Market = t.Market + s.Position = types.NewPositionFromMarket(t.Market) + s.orderExecutor = bbgo.NewGeneralOrderExecutor(&bbgo.ExchangeSession{}, t.Market.Symbol, ID, s.InstanceID(), s.Position) + return &s +} + +func TestBuildTwinOrderBook(t *testing.T) { + assert := assert.New(t) + + pins := []Pin{ + Pin(fixedpoint.NewFromInt(200)), + Pin(fixedpoint.NewFromInt(300)), + Pin(fixedpoint.NewFromInt(500)), + Pin(fixedpoint.NewFromInt(400)), + Pin(fixedpoint.NewFromInt(100)), + } + t.Run("build twin orderbook with no order", func(t *testing.T) { + b, err := buildTwinOrderBook(pins, nil) + if !assert.NoError(err) { + return + } + + assert.Equal(0, b.Size()) + assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100))) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + }) + + t.Run("build twin orderbook with some valid orders", func(t *testing.T) { + orders := []types.Order{ + { + OrderID: 1, + SubmitOrder: types.SubmitOrder{ + Side: types.SideTypeBuy, + Price: fixedpoint.NewFromInt(100), + }, + }, + { + OrderID: 5, + SubmitOrder: types.SubmitOrder{ + Side: types.SideTypeSell, + Price: fixedpoint.NewFromInt(500), + }, + }, + } + b, err := buildTwinOrderBook(pins, orders) + if !assert.NoError(err) { + return + } + + assert.Equal(2, b.Size()) + assert.Equal(2, b.EmptyTwinOrderSize()) + assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100))) + assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist()) + assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + }) + + t.Run("build twin orderbook with invalid orders", func(t *testing.T) {}) +} + +func TestSyncActiveOrder(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + + t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(0, len(activeOrders)) + }) + + t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusPartiallyFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(order.OrderID, activeOrders[0].OrderID) + assert.Equal(updatedOrder.Status, activeOrders[0].Status) + }) +} + +func TestQueryTradesToUpdateTwinOrderBook(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + pins := []Pin{ + Pin(fixedpoint.NewFromInt(100)), + Pin(fixedpoint.NewFromInt(200)), + Pin(fixedpoint.NewFromInt(300)), + Pin(fixedpoint.NewFromInt(400)), + Pin(fixedpoint.NewFromInt(500)), + } + + t.Run("query trades and update twin orderbook successfully in one page", func(t *testing.T) { + book := newTwinOrderBook(pins) + mockTradeHistoryService := mocks.NewMockExchangeTradeHistoryService(mockCtrl) + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + + trades := []types.Trade{ + { + ID: 1, + OrderID: 1, + Symbol: symbol, + Time: types.Time(time.Now().Add(-2 * time.Hour)), + }, + { + ID: 2, + OrderID: 2, + Symbol: symbol, + Time: types.Time(time.Now().Add(-1 * time.Hour)), + }, + } + orders := []types.Order{ + { + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + Side: types.SideTypeBuy, + Price: fixedpoint.NewFromInt(100), + }, + }, + { + OrderID: 2, + Status: types.OrderStatusFilled, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + Side: types.SideTypeSell, + Price: fixedpoint.NewFromInt(500), + }, + }, + } + mockTradeHistoryService.EXPECT().QueryTrades(gomock.Any(), gomock.Any(), gomock.Any()).Return(trades, nil).Times(1) + mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{ + Symbol: symbol, + OrderID: "1", + }).Return(&orders[0], nil) + mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{ + Symbol: symbol, + OrderID: "2", + }).Return(&orders[1], nil) + + assert.Equal(0, book.Size()) + if !assert.NoError(queryTradesToUpdateTwinOrderBook(ctx, symbol, book, mockTradeHistoryService, mockOrderQueryService, book.SyncOrderMap(), time.Now().Add(-24*time.Hour), time.Now(), nil)) { + return + } + + assert.Equal(2, book.Size()) + assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.Equal(orders[0].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(200)).GetOrder().OrderID) + assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + assert.Equal(orders[1].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(500)).GetOrder().OrderID) + }) +} diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go index ccc9bfa4fd..f3a093b957 100644 --- a/pkg/strategy/grid2/twin_order.go +++ b/pkg/strategy/grid2/twin_order.go @@ -154,7 +154,10 @@ func newTwinOrderBook(pins []Pin) *TwinOrderBook { pinIdx := make(map[fixedpoint.Value]int) m := make(map[fixedpoint.Value]*TwinOrder) for i, pin := range v { - m[pin] = &TwinOrder{} + // we use sell price for twin orderbook's price, so we skip the first pin as price + if i > 0 { + m[pin] = &TwinOrder{} + } pinIdx[pin] = i } diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go index d6204ee941..47395303fd 100644 --- a/pkg/strategy/grid2/twin_order_test.go +++ b/pkg/strategy/grid2/twin_order_test.go @@ -23,6 +23,11 @@ func TestTwinOrderBook(t *testing.T) { assert.Equal(4, book.EmptyTwinOrderSize()) for _, pin := range pins { twinOrder := book.GetTwinOrder(fixedpoint.Value(pin)) + if fixedpoint.NewFromInt(1) == fixedpoint.Value(pin) { + assert.Nil(twinOrder) + continue + } + if !assert.NotNil(twinOrder) { continue } From 7e532f55751e975cb47791dc8509e3064cb601f2 Mon Sep 17 00:00:00 2001 From: Rohan Kumar <130988597+rohan37kumar@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:49:33 +0530 Subject: [PATCH 112/422] Updated README.md --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 85ab0f27c0..12af0d06f1 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ You can use BBGO's trading unit and back-test unit to implement your own strateg ### Trading Unit Developers 🧑‍💻 -You can use BBGO's underlying common exchange API, currently it supports 4+ major exchanges, so you don't have to repeat +You can use BBGO's underlying common exchange API, currently, it supports 4+ major exchanges, so you don't have to repeat the implementation. ## Features - Exchange abstraction interface. -- Stream integration (user data websocket, market data websocket). -- Real-time orderBook integration through websocket. +- Stream integration (user data web socket, market data web socket). +- Real-time orderBook integration through a web socket. - TWAP order execution support. See [TWAP Order Execution](./doc/topics/twap.md) - PnL calculation. - Slack/Telegram notification. @@ -177,7 +177,7 @@ bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/download. Or refer to the [Release Page](https://github.com/c9s/bbgo/releases) and download manually. -Since v2, we've added new float point implementation from dnum to support decimals with higher precision. To download & +Since v2, we've added a new float point implementation from dnum to support decimals with higher precision. To download & setup, please refer to [Dnum Installation](doc/topics/dnum-binary.md) ### One-click Linode StackScript @@ -319,7 +319,7 @@ You can only use one database driver MySQL or SQLite to store your trading data. #### Configure MySQL Database -To use MySQL database for data syncing, first you need to install your mysql server: +To use MySQL database for data syncing, first, you need to install your mysql server: ```sh # For Ubuntu Linux @@ -427,7 +427,7 @@ See [Developing Strategy](./doc/topics/developing-strategy.md) ## Write your own private strategy -Create your go package, and initialize the repository with `go mod` and add bbgo as a dependency: +Create your go package, initialize the repository with `go mod`, and add bbgo as a dependency: ```sh go mod init @@ -550,7 +550,7 @@ following types could be injected automatically: 2. Allocate and initialize exchange sessions. 3. Add exchange sessions to the environment (the data layer). 4. Use the given environment to initialize the trader object (the logic layer). -5. The trader initializes the environment and start the exchange connections. +5. The trader initializes the environment and starts the exchange connections. 6. Call strategy.Run() method sequentially. ## Exchange API Examples @@ -567,7 +567,7 @@ maxRest := maxapi.NewRestClient(maxapi.ProductionAPIURL) maxRest.Auth(key, secret) ``` -Creating user data stream to get the orderbook (depth): +Creating user data stream to get the order book (depth): ```go stream := max.NewStream(key, secret) @@ -591,7 +591,7 @@ streambook.BindStream(stream) 1. Click the "Fork" button from the GitHub repository. 2. Clone your forked repository into `$GOPATH/github.com/c9s/bbgo`. -3. Change directory into `$GOPATH/github.com/c9s/bbgo`. +3. Change the directory into `$GOPATH/github.com/c9s/bbgo`. 4. Create a branch and start your development. 5. Test your changes. 6. Push your changes to your fork. @@ -616,13 +616,13 @@ make embed && go run -tags web ./cmd/bbgo-lorca ### What's Position? - Base Currency & Quote Currency -- How to calculate average cost? +- How to calculate the average cost? ### Looking For A New Strategy? -You can write an article about BBGO in any topic, in 750-1500 words for exchange, and I can implement the strategy for -you (depends on the complexity and efforts). If you're interested in, DM me in telegram or -twitter , we can discuss. +You can write an article about BBGO on any topic, in 750-1500 words for exchange, and I can implement the strategy for +you (depending on the complexity and efforts). If you're interested in, DM me in telegram or +twitter , and we can discuss. ### Adding New Crypto Exchange support? From c611cfe73baa7025468ead2da47d8b650c26b601 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 25 Oct 2023 21:30:54 +0800 Subject: [PATCH 113/422] pkg/exchange: add a jumpIfEmpty to batch trade option --- pkg/exchange/batch/option.go | 12 ++++++++++++ pkg/exchange/batch/trade.go | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 pkg/exchange/batch/option.go diff --git a/pkg/exchange/batch/option.go b/pkg/exchange/batch/option.go new file mode 100644 index 0000000000..67f18e6087 --- /dev/null +++ b/pkg/exchange/batch/option.go @@ -0,0 +1,12 @@ +package batch + +import "time" + +type Option func(query *AsyncTimeRangedBatchQuery) + +// JumpIfEmpty jump the startTime + duration when the result is empty +func JumpIfEmpty(duration time.Duration) Option { + return func(query *AsyncTimeRangedBatchQuery) { + query.JumpIfEmpty = duration + } +} diff --git a/pkg/exchange/batch/trade.go b/pkg/exchange/batch/trade.go index 1c91da7773..4fce26b651 100644 --- a/pkg/exchange/batch/trade.go +++ b/pkg/exchange/batch/trade.go @@ -17,7 +17,7 @@ type TradeBatchQuery struct { types.ExchangeTradeHistoryService } -func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions) (c chan types.Trade, errC chan error) { +func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions, opts ...Option) (c chan types.Trade, errC chan error) { if options.EndTime == nil { now := time.Now() options.EndTime = &now @@ -45,6 +45,10 @@ func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *type JumpIfEmpty: 24 * time.Hour, } + for _, opt := range opts { + opt(query) + } + c = make(chan types.Trade, 100) errC = query.Query(ctx, c, startTime, endTime) return c, errC From 881db49b70306b1b2f7be75d032831c60ae4461b Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 25 Oct 2023 21:36:26 +0800 Subject: [PATCH 114/422] pkg/exchange: rename tradeRateLimiter to queryOrderTradeRateLimiter --- pkg/exchange/bybit/exchange.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index e1c3545c59..7cec992c68 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -26,15 +26,15 @@ const ( ) // https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit -// sharedRateLimiter indicates that the API belongs to the public API. -// -// The default order limiter apply 5 requests per second and a 5 initial bucket -// this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates +// GET/POST method (shared): 120 requests per second for 5 consecutive seconds var ( - sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) - tradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) - orderRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1) + // sharedRateLimiter indicates that the API belongs to the public API. + // The default order limiter apply 5 requests per second and a 5 initial bucket + // this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates + sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + queryOrderTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + orderRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 10) + closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1) log = logrus.WithFields(logrus.Fields{ "exchange": "bybit", @@ -159,7 +159,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ req = req.Cursor(cursor) } - if err = tradeRateLimiter.Wait(ctx); err != nil { + if err = queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("place order rate limiter wait error: %w", err) } res, err := req.Do(ctx) @@ -232,7 +232,7 @@ func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) (tr req.Symbol(q.Symbol) } - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("trade rate limiter wait error: %w", err) } response, err := req.Do(ctx) @@ -463,7 +463,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type } req.Limit(limit) - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("trade rate limiter wait error: %w", err) } response, err := req.Do(ctx) From 55d444d86a85046693403d2d4276ccc3c742e7e7 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 26 Oct 2023 09:31:25 +0800 Subject: [PATCH 115/422] pkg/exchange: add jumpIfEmpty opts to closed order batch query --- pkg/exchange/batch/closedorders.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/batch/closedorders.go b/pkg/exchange/batch/closedorders.go index 51d12f5ff1..77e37690b6 100644 --- a/pkg/exchange/batch/closedorders.go +++ b/pkg/exchange/batch/closedorders.go @@ -12,7 +12,7 @@ type ClosedOrderBatchQuery struct { types.ExchangeTradeHistoryService } -func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64) (c chan types.Order, errC chan error) { +func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64, opts ...Option) (c chan types.Order, errC chan error) { query := &AsyncTimeRangedBatchQuery{ Type: types.Order{}, Q: func(startTime, endTime time.Time) (interface{}, error) { @@ -32,6 +32,10 @@ func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startT JumpIfEmpty: 30 * 24 * time.Hour, } + for _, opt := range opts { + opt(query) + } + c = make(chan types.Order, 100) errC = query.Query(ctx, c, startTime, endTime) return c, errC From 7c27cb9801df36c2b7f5d925369a8b020e1bdf9a Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Thu, 26 Oct 2023 10:24:54 +0545 Subject: [PATCH 116/422] docs: fix typos in doc/development/adding-new-exchange.md --- doc/development/adding-new-exchange.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/development/adding-new-exchange.md b/doc/development/adding-new-exchange.md index 6f0ae80754..1eb45a2bad 100644 --- a/doc/development/adding-new-exchange.md +++ b/doc/development/adding-new-exchange.md @@ -58,7 +58,7 @@ Stream - [ ] Public trade message parser (optional) - [ ] Ticker message parser (optional) - [ ] ping/pong handling. (you can reuse the existing types.StandardStream) -- [ ] heart-beat hanlding or keep-alive handling. (already included in types.StandardStream) +- [ ] heart-beat handling or keep-alive handling. (already included in types.StandardStream) - [ ] handling reconnect. (already included in types.StandardStream) Database From 41896646270e7c699157cc52556291bb9e8b5f3a Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Thu, 26 Oct 2023 10:25:09 +0545 Subject: [PATCH 117/422] docs: fix typos in doc/development/release-process.md --- doc/development/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/development/release-process.md b/doc/development/release-process.md index 89efb1b351..eaf33d8f8a 100644 --- a/doc/development/release-process.md +++ b/doc/development/release-process.md @@ -40,7 +40,7 @@ Run the following command to create the release: make version VERSION=v1.20.2 ``` -The above command wilL: +The above command will: - Update and compile the migration scripts into go files. - Bump the version name in the go code. From fc9ce53747133fc03d3881538148b73fd3b78fe2 Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Thu, 26 Oct 2023 10:25:22 +0545 Subject: [PATCH 118/422] docs: fix typos in doc/development/series.md --- doc/development/series.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/development/series.md b/doc/development/series.md index 0f23756fc3..ec95973772 100644 --- a/doc/development/series.md +++ b/doc/development/series.md @@ -25,7 +25,7 @@ type BoolSeries interface { } ``` -Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fullfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happend on the curves at the moment. +Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fulfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happened on the curves at the moment. #### Expected Implementation @@ -44,7 +44,7 @@ and if any of the method in the interface not been implemented, this would gener #### Extended Series -Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shfit`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions. +Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shift`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions. Example: From f31d829294595f61263e1b65b3b7afb3fe66a5ec Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 24 Oct 2023 13:02:34 +0800 Subject: [PATCH 119/422] FEAUTRE: merge grid recover and active orders recover --- pkg/strategy/grid2/recover.go | 196 +++++++++++++++++++++++++++++++++- 1 file changed, 195 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 623966e607..e9be4866de 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -8,6 +8,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" "github.com/pkg/errors" ) @@ -42,6 +43,198 @@ import ( 4. run grid recover to make sure all the twin price has its order. */ +func (s *Strategy) recover(ctx context.Context) error { + historyService, implemented := s.session.Exchange.(types.ExchangeTradeHistoryService) + // if the exchange doesn't support ExchangeTradeHistoryService, do not run recover + if !implemented { + s.logger.Warn("ExchangeTradeHistoryService is not implemented, can not recover grid") + return nil + } + + activeOrderBook := s.orderExecutor.ActiveMakerOrders() + activeOrders := activeOrderBook.Orders() + + openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol) + if err != nil { + return err + } + + // check if it's new strategy or need to recover + if len(activeOrders) == 0 && len(openOrders) == 0 && s.GridProfitStats.InitialOrderID == 0 { + // even though there is no open orders and initial orderID is 0 + // we still need to query trades to make sure if we need to recover or not + trades, err := historyService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{ + // from 1, because some API will ignore 0 last trade id + LastTradeID: 1, + // if there is any trades, we need to recover. + Limit: 1, + }) + + if err != nil { + return errors.Wrapf(err, "unable to query trades when recovering") + } + + if len(trades) == 0 { + s.logger.Info("no open order, no active order, no trade, it's a new strategy so no need to recover") + return nil + } + } + + s.logger.Info("start recovering") + + if s.getGrid() == nil { + s.setGrid(s.newGrid()) + } + + s.mu.Lock() + defer s.mu.Unlock() + + pins := s.getGrid().Pins + + activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) + openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) + + s.logger.Infof("active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) + s.logger.Infof("open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) + + // remove index 0, because twin orderbook's price is from the second one + pins = pins[1:] + var noTwinOrderPins []fixedpoint.Value + + for _, pin := range pins { + v := fixedpoint.Value(pin) + activeOrder := activeOrdersInTwinOrderBook.GetTwinOrder(v) + openOrder := openOrdersInTwinOrderBook.GetTwinOrder(v) + if activeOrder == nil || openOrder == nil { + return fmt.Errorf("there is no any twin order at this pin, can not recover") + } + + var activeOrderID uint64 = 0 + if activeOrder.Exist() { + activeOrderID = activeOrder.GetOrder().OrderID + } + + var openOrderID uint64 = 0 + if openOrder.Exist() { + openOrderID = openOrder.GetOrder().OrderID + } + + // case 3 + if activeOrderID == 0 && openOrderID == 0 { + noTwinOrderPins = append(noTwinOrderPins, v) + continue + } + + // case 1 + if activeOrderID == 0 { + activeOrderBook.Add(openOrder.GetOrder()) + // also add open orders into active order's twin orderbook, we will use this active orderbook to recover empty price grid + activeOrdersInTwinOrderBook.AddTwinOrder(v, openOrder) + continue + } + + // case 2 + if openOrderID == 0 { + syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, activeOrder.GetOrder().OrderID) + continue + } + + // case 4 + if activeOrderID != openOrderID { + return fmt.Errorf("there are two different orders in the same pin, can not recover") + } + + // case 5 + // do nothing + } + + s.logger.Infof("twin orderbook after adding open orders\n%s", activeOrdersInTwinOrderBook.String()) + + if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil { + s.logger.WithError(err).Error("failed to recover empty grid") + return err + } + + s.logger.Infof("twin orderbook after recovering\n%s", activeOrdersInTwinOrderBook.String()) + + if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 { + return fmt.Errorf("there is still empty grid in twin orderbook") + } + + for _, pin := range noTwinOrderPins { + twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin) + if twinOrder == nil { + return fmt.Errorf("should not get nil twin order after recovering empty grid, check it") + } + + if !twinOrder.Exist() { + return fmt.Errorf("should not get empty twin order after recovering empty grid, check it") + } + + activeOrderBook.EmitFilled(twinOrder.GetOrder()) + + time.Sleep(100 * time.Millisecond) + } + + s.EmitGridReady() + + time.Sleep(2 * time.Second) + debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders()) + + bbgo.Sync(ctx, s) + + return nil +} + +func (s *Strategy) recoverEmptyGridOnTwinOrderBook( + ctx context.Context, + twinOrderBook *TwinOrderBook, + queryTradesService types.ExchangeTradeHistoryService, + queryOrderService types.ExchangeOrderQueryService, +) error { + if twinOrderBook.EmptyTwinOrderSize() == 0 { + s.logger.Info("no empty grid") + return nil + } + + existedOrders := twinOrderBook.SyncOrderMap() + + until := time.Now() + since := until.Add(-1 * time.Hour) + // hard limit for recover + recoverSinceLimit := time.Date(2023, time.March, 10, 0, 0, 0, 0, time.UTC) + + if s.RecoverGridWithin != 0 && until.Add(-1*s.RecoverGridWithin).After(recoverSinceLimit) { + recoverSinceLimit = until.Add(-1 * s.RecoverGridWithin) + } + + for { + if err := queryTradesToUpdateTwinOrderBook(ctx, s.Symbol, twinOrderBook, queryTradesService, queryOrderService, existedOrders, since, until, s.debugLog); err != nil { + return errors.Wrapf(err, "failed to query trades to update twin orderbook") + } + + until = since + since = until.Add(-6 * time.Hour) + + if twinOrderBook.EmptyTwinOrderSize() == 0 { + s.logger.Infof("stop querying trades because there is no empty twin order on twin orderbook") + break + } + + if s.GridProfitStats != nil && s.GridProfitStats.Since != nil && until.Before(*s.GridProfitStats.Since) { + s.logger.Infof("stop querying trades because the time range is out of the strategy's since (%s)", *s.GridProfitStats.Since) + break + } + + if until.Before(recoverSinceLimit) { + s.logger.Infof("stop querying trades because the time range is out of the limit (%s)", recoverSinceLimit) + break + } + } + + return nil +} + func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error) { book := newTwinOrderBook(pins) @@ -77,7 +270,8 @@ func queryTradesToUpdateTwinOrderBook( queryOrderService types.ExchangeOrderQueryService, existedOrders *types.SyncOrderMap, since, until time.Time, - logger func(format string, args ...interface{})) error { + logger func(format string, args ...interface{}), +) error { if twinOrderBook == nil { return fmt.Errorf("twin orderbook should not be nil, please check it") } From 40ca323b2dc726cfc1f5621063408f65205b857d Mon Sep 17 00:00:00 2001 From: chiahung Date: Thu, 26 Oct 2023 16:29:05 +0800 Subject: [PATCH 120/422] merge recover logic --- pkg/strategy/grid2/recover.go | 41 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index e9be4866de..80a19008a7 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -150,33 +150,36 @@ func (s *Strategy) recover(ctx context.Context) error { s.logger.Infof("twin orderbook after adding open orders\n%s", activeOrdersInTwinOrderBook.String()) - if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil { - s.logger.WithError(err).Error("failed to recover empty grid") - return err - } - - s.logger.Infof("twin orderbook after recovering\n%s", activeOrdersInTwinOrderBook.String()) + if len(noTwinOrderPins) != 0 { + if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil { + s.logger.WithError(err).Error("failed to recover empty grid") + return err + } - if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 { - return fmt.Errorf("there is still empty grid in twin orderbook") - } + s.logger.Infof("twin orderbook after recovering no twin order on grid\n%s", activeOrdersInTwinOrderBook.String()) - for _, pin := range noTwinOrderPins { - twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin) - if twinOrder == nil { - return fmt.Errorf("should not get nil twin order after recovering empty grid, check it") + if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 { + return fmt.Errorf("there is still empty grid in twin orderbook") } - if !twinOrder.Exist() { - return fmt.Errorf("should not get empty twin order after recovering empty grid, check it") - } + for _, pin := range noTwinOrderPins { + twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin) + if twinOrder == nil { + return fmt.Errorf("should not get nil twin order after recovering empty grid, check it") + } - activeOrderBook.EmitFilled(twinOrder.GetOrder()) + if !twinOrder.Exist() { + return fmt.Errorf("should not get empty twin order after recovering empty grid, check it") + } - time.Sleep(100 * time.Millisecond) + activeOrderBook.EmitFilled(twinOrder.GetOrder()) + + time.Sleep(100 * time.Millisecond) + } } - s.EmitGridReady() + // TODO: do not emit ready here, emit ready only once when opening grid or recovering grid after worker stopped + // s.EmitGridReady() time.Sleep(2 * time.Second) debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders()) From 2a85bbebf0e4540a76bf59e0b242087d0f778bf2 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 27 Oct 2023 12:52:36 +0800 Subject: [PATCH 121/422] pkg/exchange: fix precision --- pkg/exchange/bybit/convert.go | 4 ++-- pkg/exchange/bybit/convert_test.go | 5 ++--- pkg/exchange/kucoin/convert.go | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index b13032cd7b..892493e334 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -16,8 +16,8 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market { return types.Market{ Symbol: m.Symbol, LocalSymbol: m.Symbol, - PricePrecision: int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())), - VolumePrecision: int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())), + PricePrecision: -int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())), + VolumePrecision: -int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())), QuoteCurrency: m.QuoteCoin, BaseCurrency: m.BaseCoin, MinNotional: m.LotSizeFilter.MinOrderAmt, diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index 786234d8de..a5ddb08bc5 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -2,7 +2,6 @@ package bybit import ( "fmt" - "math" "strconv" "testing" "time" @@ -67,8 +66,8 @@ func TestToGlobalMarket(t *testing.T) { exp := types.Market{ Symbol: inst.Symbol, LocalSymbol: inst.Symbol, - PricePrecision: int(math.Log10(inst.LotSizeFilter.QuotePrecision.Float64())), - VolumePrecision: int(math.Log10(inst.LotSizeFilter.BasePrecision.Float64())), + PricePrecision: 8, + VolumePrecision: 6, QuoteCurrency: inst.QuoteCoin, BaseCurrency: inst.BaseCoin, MinNotional: inst.LotSizeFilter.MinOrderAmt, diff --git a/pkg/exchange/kucoin/convert.go b/pkg/exchange/kucoin/convert.go index d86f84db95..c335105222 100644 --- a/pkg/exchange/kucoin/convert.go +++ b/pkg/exchange/kucoin/convert.go @@ -39,8 +39,8 @@ func toGlobalMarket(m kucoinapi.Symbol) types.Market { return types.Market{ Symbol: symbol, LocalSymbol: m.Symbol, - PricePrecision: int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4 - VolumePrecision: int(math.Log10(m.BaseIncrement.Float64())), + PricePrecision: -int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4 + VolumePrecision: -int(math.Log10(m.BaseIncrement.Float64())), QuoteCurrency: m.QuoteCurrency, BaseCurrency: m.BaseCurrency, MinNotional: m.QuoteMinSize, From e8c9801535d92004856681506bfad0f849b71123 Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 27 Oct 2023 15:01:41 +0800 Subject: [PATCH 122/422] adjust quantity by max amount --- config/xalign.yaml | 9 +++++++-- pkg/strategy/xalign/strategy.go | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/config/xalign.yaml b/config/xalign.yaml index 51956f74f4..6f07ba4b81 100644 --- a/config/xalign.yaml +++ b/config/xalign.yaml @@ -27,7 +27,6 @@ persistence: db: 0 crossExchangeStrategies: - - xalign: interval: 1m sessions: @@ -41,4 +40,10 @@ crossExchangeStrategies: sell: [USDT] expectedBalances: BTC: 0.0440 - + useTakerOrder: false + dryRun: true + balanceToleranceRange: 10% + maxAmounts: + USDT: 100 + USDC: 100 + TWD: 3000 diff --git a/pkg/strategy/xalign/strategy.go b/pkg/strategy/xalign/strategy.go index 140c1fb500..b6c4f49ad3 100644 --- a/pkg/strategy/xalign/strategy.go +++ b/pkg/strategy/xalign/strategy.go @@ -45,6 +45,7 @@ type Strategy struct { DryRun bool `json:"dryRun"` BalanceToleranceRange fixedpoint.Value `json:"balanceToleranceRange"` Duration types.Duration `json:"for"` + MaxAmounts map[string]fixedpoint.Value `json:"maxAmounts"` faultBalanceRecords map[string][]TimeBalance @@ -156,7 +157,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st switch side { case types.SideTypeBuy: - price := ticker.Sell + var price fixedpoint.Value if taker { price = ticker.Sell } else if spread.Compare(market.TickSize) > 0 { @@ -177,6 +178,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st continue } + maxAmount, ok := s.MaxAmounts[market.QuoteCurrency] + if ok { + requiredQuoteAmount = bbgo.AdjustQuantityByMaxAmount(requiredQuoteAmount, price, maxAmount) + log.Infof("adjusted quantity %f %s by max amount %f %s", requiredQuoteAmount.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency) + } + if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, requiredQuoteAmount); ok { return session, &types.SubmitOrder{ Symbol: symbol, @@ -190,7 +197,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st } case types.SideTypeSell: - price := ticker.Buy + var price fixedpoint.Value if taker { price = ticker.Buy } else if spread.Compare(market.TickSize) > 0 { @@ -209,6 +216,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st continue } + maxAmount, ok := s.MaxAmounts[market.QuoteCurrency] + if ok { + q = bbgo.AdjustQuantityByMaxAmount(q, price, maxAmount) + log.Infof("adjusted quantity %f %s by max amount %f %s", q.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency) + } + if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, q); ok { return session, &types.SubmitOrder{ Symbol: symbol, From ba7e26c800b4fd9a9b20b172b46e316c5a8dbbec Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 27 Oct 2023 15:28:35 +0800 Subject: [PATCH 123/422] pkg/exchange: use NumFractionalDigits instead of math.Log10(Float64) due to precision problem --- pkg/exchange/bybit/convert.go | 5 ++--- pkg/exchange/kucoin/convert.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index 892493e334..d89f194fea 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -2,7 +2,6 @@ package bybit import ( "fmt" - "math" "strconv" "time" @@ -16,8 +15,8 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market { return types.Market{ Symbol: m.Symbol, LocalSymbol: m.Symbol, - PricePrecision: -int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())), - VolumePrecision: -int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())), + PricePrecision: m.LotSizeFilter.QuotePrecision.NumFractionalDigits(), + VolumePrecision: m.LotSizeFilter.BasePrecision.NumFractionalDigits(), QuoteCurrency: m.QuoteCoin, BaseCurrency: m.BaseCoin, MinNotional: m.LotSizeFilter.MinOrderAmt, diff --git a/pkg/exchange/kucoin/convert.go b/pkg/exchange/kucoin/convert.go index c335105222..e83ade5d30 100644 --- a/pkg/exchange/kucoin/convert.go +++ b/pkg/exchange/kucoin/convert.go @@ -3,7 +3,6 @@ package kucoin import ( "fmt" "hash/fnv" - "math" "strings" "time" @@ -39,8 +38,8 @@ func toGlobalMarket(m kucoinapi.Symbol) types.Market { return types.Market{ Symbol: symbol, LocalSymbol: m.Symbol, - PricePrecision: -int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4 - VolumePrecision: -int(math.Log10(m.BaseIncrement.Float64())), + PricePrecision: m.PriceIncrement.NumFractionalDigits(), // convert 0.0001 to 4 + VolumePrecision: m.BaseIncrement.NumFractionalDigits(), QuoteCurrency: m.QuoteCurrency, BaseCurrency: m.BaseCurrency, MinNotional: m.QuoteMinSize, From d07b7669390e4daf9c6e7078257b15b2007936ff Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 27 Oct 2023 16:03:03 +0800 Subject: [PATCH 124/422] pkg/exchange: Use the same conn to avoid concurrent write issues. --- pkg/exchange/bybit/stream.go | 48 +++++++++--------------------------- pkg/types/stream.go | 19 ++++++++------ 2 files changed, 22 insertions(+), 45 deletions(-) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index c6b42cb9b1..eb4137ed37 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -15,10 +15,6 @@ import ( ) const ( - // Bybit: To avoid network or program issues, we recommend that you send the ping heartbeat packet every 20 seconds - // to maintain the WebSocket connection. - pingInterval = 20 * time.Second - // spotArgsLimit can input up to 10 args for each subscription request sent to one connection. spotArgsLimit = 10 ) @@ -244,40 +240,18 @@ func (s *Stream) parseWebSocketEvent(in []byte) (interface{}, error) { } // ping implements the Bybit text message of WebSocket PingPong. -func (s *Stream) ping(ctx context.Context, conn *websocket.Conn, cancelFunc context.CancelFunc) { - defer func() { - log.Debug("[bybit] ping worker stopped") - cancelFunc() - }() - - var pingTicker = time.NewTicker(pingInterval) - defer pingTicker.Stop() - - for { - select { - - case <-ctx.Done(): - return - - case <-s.CloseC: - return - - case <-pingTicker.C: - // it's just for maintaining the liveliness of the connection, so comment out ReqId. - err := conn.WriteJSON(struct { - //ReqId string `json:"req_id"` - Op WsOpType `json:"op"` - }{ - //ReqId: uuid.NewString(), - Op: WsOpTypePing, - }) - if err != nil { - log.WithError(err).Error("ping error", err) - s.Reconnect() - return - } - } +func (s *Stream) ping(conn *websocket.Conn) error { + err := conn.WriteJSON(struct { + Op WsOpType `json:"op"` + }{ + Op: WsOpTypePing, + }) + if err != nil { + log.WithError(err).Error("ping error") + return err } + + return nil } func (s *Stream) handlerConnect() { diff --git a/pkg/types/stream.go b/pkg/types/stream.go index f65927b615..4ce8c161fd 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -57,8 +57,8 @@ type Parser func(message []byte) (interface{}, error) type Dispatcher func(e interface{}) -// HeartBeat keeps connection alive by sending the heartbeat packet. -type HeartBeat func(ctxConn context.Context, conn *websocket.Conn, cancelConn context.CancelFunc) +// HeartBeat keeps connection alive by sending the ping packet. +type HeartBeat func(conn *websocket.Conn) error type BeforeConnect func(ctx context.Context) error @@ -86,7 +86,7 @@ type StandardStream struct { // sg is used to wait until the previous routines are closed. // only handle routines used internally, avoid including external callback func to prevent issues if they have - // bugs and cannot terminate. e.q. heartBeat + // bugs and cannot terminate. sg SyncGroup // ReconnectC is a signal channel for reconnecting @@ -319,6 +319,14 @@ func (s *StandardStream) ping( return case <-pingTicker.C: + if s.heartBeat != nil { + if err := s.heartBeat(conn); err != nil { + // log errors at the concrete class so that we can identify which exchange encountered an error + s.Reconnect() + return + } + } + if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(writeTimeout)); err != nil { log.WithError(err).Error("ping error", err) s.Reconnect() @@ -432,11 +440,6 @@ func (s *StandardStream) DialAndConnect(ctx context.Context) error { s.ping(connCtx, conn, connCancel, pingInterval) }) s.sg.Run() - - if s.heartBeat != nil { - // not included in wg, as it is an external callback func. - go s.heartBeat(connCtx, conn, connCancel) - } return nil } From 39c3d23da323e3222df5d610695a8a776b92f5f0 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 24 Oct 2023 22:17:08 +0800 Subject: [PATCH 125/422] pkg/exchange: support ping/pong --- pkg/exchange/bitget/stream.go | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index eadf25b48a..5a2f6f23d3 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -1,12 +1,26 @@ package bitget import ( + "bytes" "context" "encoding/json" "fmt" + "time" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/types" + "github.com/gorilla/websocket" +) + +const ( + // Client should keep ping the server in every 30 seconds. Server will close the connections which has no ping over + // 120 seconds(even when the client is still receiving data from the server) + pingInterval = 30 * time.Second +) + +var ( + pingBytes = []byte("ping") + pongBytes = []byte("pong") ) //go:generate callbackgen -type Stream @@ -25,6 +39,7 @@ func NewStream() *Stream { stream.SetEndpointCreator(stream.createEndpoint) stream.SetParser(parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) + stream.SetHeartBeat(stream.ping) stream.OnConnect(stream.handlerConnect) stream.OnBookEvent(stream.handleBookEvent) @@ -92,6 +107,12 @@ func (s *Stream) dispatchEvent(event interface{}) { case *MarketTradeEvent: s.EmitMarketTradeEvent(*e) + + case []byte: + // We only handle the 'pong' case. Others are unexpected. + if !bytes.Equal(e, pongBytes) { + log.Errorf("invalid event: %q", e) + } } } @@ -116,6 +137,16 @@ func (s *Stream) handleBookEvent(o BookEvent) { } } +// ping implements the bitget text message of WebSocket PingPong. +func (s *Stream) ping(conn *websocket.Conn) error { + err := conn.WriteMessage(websocket.TextMessage, pingBytes) + if err != nil { + log.WithError(err).Error("ping error", err) + return nil + } + return nil +} + func convertSubscription(sub types.Subscription) (WsArg, error) { arg := WsArg{ // support spot only @@ -146,6 +177,18 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { } func parseWebSocketEvent(in []byte) (interface{}, error) { + switch { + case bytes.Equal(in, pongBytes): + // Return the original raw data may seem redundant because we can validate the string and return nil, + // but we cannot return nil to a lower level handler. This can cause confusion in the next handler, such as + // the dispatch handler. Therefore, I return the original raw data. + return in, nil + default: + return parseEvent(in) + } +} + +func parseEvent(in []byte) (interface{}, error) { var event WsEvent err := json.Unmarshal(in, &event) From 671772a76775f9321e7e098d00031fecbfdcfb09 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 30 Oct 2023 16:28:34 +0800 Subject: [PATCH 126/422] FIX: retry to get open orders only for 5 times and do not sync orders updated in 3 min --- pkg/exchange/retry/order.go | 19 +++++++++++++++++++ pkg/strategy/grid2/active_order_recover.go | 15 +++++++++++---- pkg/strategy/grid2/recover.go | 2 +- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index 15b59c6afa..5a8c8855bf 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -47,6 +47,15 @@ func GeneralBackoff(ctx context.Context, op backoff2.Operation) (err error) { return err } +func GeneralBackoffLite(ctx context.Context, op backoff2.Operation) (err error) { + err = backoff2.Retry(op, backoff2.WithContext( + backoff2.WithMaxRetries( + backoff2.NewExponentialBackOff(), + 5), + ctx)) + return err +} + func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { var op = func() (err2 error) { openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) @@ -57,6 +66,16 @@ func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symb return openOrders, err } +func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { + var op = func() (err2 error) { + openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) + return err2 + } + + err = GeneralBackoffLite(ctx, op) + return openOrders, err +} + func QueryOrderUntilSuccessful(ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery) (order *types.Order, err error) { var op = func() (err2 error) { order, err2 = query.QueryOrder(ctx, opts) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 5042084fc8..82269832e6 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -89,9 +89,10 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders") - notAddNonExistingOpenOrdersAfter := time.Now().Add(-5 * time.Minute) + // do not sync orders which is updated in 3 min, because we may receive from websocket and handle it twice + doNotSyncAfter := time.Now().Add(-3 * time.Minute) - openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, opts.exchange, opts.activeOrderBook.Symbol) + openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, opts.exchange, opts.activeOrderBook.Symbol) if err != nil { opts.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders, skip this time") @@ -116,6 +117,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { delete(openOrdersMap, activeOrder.OrderID) } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + if activeOrder.UpdateTime.After(doNotSyncAfter) { + opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) + continue + } // sleep 100ms to avoid DDOS time.Sleep(100 * time.Millisecond) @@ -130,8 +135,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // update open orders not in active orders for _, openOrder := range openOrdersMap { - // we don't add open orders into active orderbook if updated in 5 min - if openOrder.UpdateTime.After(notAddNonExistingOpenOrdersAfter) { + opts.logger.Infof("found open order #%d is not in active orderbook, updating...", openOrder.OrderID) + // we don't add open orders into active orderbook if updated in 3 min, because we may receive message from websocket and add it twice. + if openOrder.UpdateTime.After(doNotSyncAfter) { + opts.logger.Infof("open order #%d is updated in 3 min, skip updating...", openOrder.OrderID) continue } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 80a19008a7..0634b72d43 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -54,7 +54,7 @@ func (s *Strategy) recover(ctx context.Context) error { activeOrderBook := s.orderExecutor.ActiveMakerOrders() activeOrders := activeOrderBook.Orders() - openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.session.Exchange, s.Symbol) + openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, s.session.Exchange, s.Symbol) if err != nil { return err } From d33240ec83781ed92010941b7fc3104e38e3d3fb Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 30 Oct 2023 17:17:36 +0800 Subject: [PATCH 127/422] rename and simplify import --- pkg/exchange/retry/order.go | 26 +++++++++++----------- pkg/strategy/grid2/active_order_recover.go | 8 +++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index 5a8c8855bf..df6ef6b59c 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -5,10 +5,9 @@ import ( "errors" "strconv" - backoff2 "github.com/cenkalti/backoff/v4" + "github.com/cenkalti/backoff/v4" "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util/backoff" ) type advancedOrderCancelService interface { @@ -18,7 +17,7 @@ type advancedOrderCancelService interface { } func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64) (o *types.Order, err error) { - err = backoff.RetryGeneral(ctx, func() (err2 error) { + var op = func() (err2 error) { o, err2 = queryOrderService.QueryOrder(ctx, types.OrderQuery{ Symbol: symbol, OrderID: strconv.FormatUint(orderId, 10), @@ -33,24 +32,25 @@ func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.Exchange } return err2 - }) + } + err = GeneralBackoff(ctx, op) return o, err } -func GeneralBackoff(ctx context.Context, op backoff2.Operation) (err error) { - err = backoff2.Retry(op, backoff2.WithContext( - backoff2.WithMaxRetries( - backoff2.NewExponentialBackOff(), +func GeneralBackoff(ctx context.Context, op backoff.Operation) (err error) { + err = backoff.Retry(op, backoff.WithContext( + backoff.WithMaxRetries( + backoff.NewExponentialBackOff(), 101), ctx)) return err } -func GeneralBackoffLite(ctx context.Context, op backoff2.Operation) (err error) { - err = backoff2.Retry(op, backoff2.WithContext( - backoff2.WithMaxRetries( - backoff2.NewExponentialBackOff(), +func GeneralLiteBackoff(ctx context.Context, op backoff.Operation) (err error) { + err = backoff.Retry(op, backoff.WithContext( + backoff.WithMaxRetries( + backoff.NewExponentialBackOff(), 5), ctx)) return err @@ -72,7 +72,7 @@ func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, return err2 } - err = GeneralBackoffLite(ctx, op) + err = GeneralLiteBackoff(ctx, op) return openOrders, err } diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 82269832e6..cfdaeac80b 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -89,8 +89,8 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders") - // do not sync orders which is updated in 3 min, because we may receive from websocket and handle it twice - doNotSyncAfter := time.Now().Add(-3 * time.Minute) + // only sync orders which is updated over 3 min, because we may receive from websocket and handle it twice + syncBefore := time.Now().Add(-3 * time.Minute) openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, opts.exchange, opts.activeOrderBook.Symbol) if err != nil { @@ -117,7 +117,7 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { delete(openOrdersMap, activeOrder.OrderID) } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if activeOrder.UpdateTime.After(doNotSyncAfter) { + if activeOrder.UpdateTime.After(syncBefore) { opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) continue } @@ -137,7 +137,7 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { for _, openOrder := range openOrdersMap { opts.logger.Infof("found open order #%d is not in active orderbook, updating...", openOrder.OrderID) // we don't add open orders into active orderbook if updated in 3 min, because we may receive message from websocket and add it twice. - if openOrder.UpdateTime.After(doNotSyncAfter) { + if openOrder.UpdateTime.After(syncBefore) { opts.logger.Infof("open order #%d is updated in 3 min, skip updating...", openOrder.OrderID) continue } From b8401ee177845e70ecf18c17509ce1a55329aa71 Mon Sep 17 00:00:00 2001 From: Himanshu Kumar Mahto <93067059+HimanshuMahto@users.noreply.github.com> Date: Tue, 31 Oct 2023 01:51:04 +0530 Subject: [PATCH 128/422] grammatical error in the code_of_conduct file --- CODE_OF_CONDUCT.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index b70133c9cb..e5e203c084 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,7 +2,7 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our +We as members, contributors, and leaders pledge to participate in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, @@ -15,7 +15,7 @@ diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our -community include: +community includes: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences @@ -33,7 +33,7 @@ Examples of unacceptable behavior include: * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a +* Other conduct that could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities @@ -50,7 +50,7 @@ decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when +This Code of Conduct applies within all community spaces and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed @@ -82,12 +82,11 @@ behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: This violation occurs through a single incident or a series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This +those enforcing the Code of Conduct, for a specified period. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. @@ -98,7 +97,7 @@ permanent ban. sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or +communication with the community for a specified period. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. From 7c19bb9e20fba4eab7f6306088c28a8d7c836061 Mon Sep 17 00:00:00 2001 From: narumi Date: Tue, 31 Oct 2023 13:53:12 +0800 Subject: [PATCH 129/422] submit one order at a time --- config/rebalance.yaml | 10 +- .../rebalance/multi_market_strategy.go | 44 ++++ pkg/strategy/rebalance/position_map.go | 12 +- pkg/strategy/rebalance/profit_stats_map.go | 14 +- pkg/strategy/rebalance/strategy.go | 247 +++++++----------- 5 files changed, 159 insertions(+), 168 deletions(-) create mode 100644 pkg/strategy/rebalance/multi_market_strategy.go diff --git a/config/rebalance.yaml b/config/rebalance.yaml index 14a784c931..bdcd5f6f56 100644 --- a/config/rebalance.yaml +++ b/config/rebalance.yaml @@ -12,9 +12,9 @@ backtest: startTime: "2022-01-01" endTime: "2022-10-01" symbols: - - BTCUSDT - - ETHUSDT - - MAXUSDT + - BTCUSDT + - ETHUSDT + - MAXUSDT account: max: makerFeeRate: 0.075% @@ -28,7 +28,7 @@ backtest: exchangeStrategies: - on: max rebalance: - interval: 1d + cronExpression: "@every 1s" quoteCurrency: USDT targetWeights: BTC: 50% @@ -37,5 +37,5 @@ exchangeStrategies: threshold: 1% maxAmount: 1_000 # max amount to buy or sell per order orderType: LIMIT_MAKER # LIMIT, LIMIT_MAKER or MARKET - dryRun: false + dryRun: true onStart: true diff --git a/pkg/strategy/rebalance/multi_market_strategy.go b/pkg/strategy/rebalance/multi_market_strategy.go new file mode 100644 index 0000000000..e7d17dd0b0 --- /dev/null +++ b/pkg/strategy/rebalance/multi_market_strategy.go @@ -0,0 +1,44 @@ +package rebalance + +import ( + "context" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" +) + +type MultiMarketStrategy struct { + Environ *bbgo.Environment + Session *bbgo.ExchangeSession + + PositionMap PositionMap `persistence:"positionMap"` + ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"` + OrderExecutorMap GeneralOrderExecutorMap + + parent, ctx context.Context + cancel context.CancelFunc +} + +func (s *MultiMarketStrategy) Initialize(ctx context.Context, environ *bbgo.Environment, session *bbgo.ExchangeSession, markets map[string]types.Market, strategyID string) { + s.parent = ctx + s.ctx, s.cancel = context.WithCancel(ctx) + + s.Environ = environ + s.Session = session + + if s.PositionMap == nil { + s.PositionMap = make(PositionMap) + } + s.PositionMap.CreatePositions(markets) + + if s.ProfitStatsMap == nil { + s.ProfitStatsMap = make(ProfitStatsMap) + } + s.ProfitStatsMap.CreateProfitStats(markets) + + s.OrderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap) + s.OrderExecutorMap.BindEnvironment(environ) + s.OrderExecutorMap.BindProfitStats(s.ProfitStatsMap) + s.OrderExecutorMap.Sync(ctx, s) + s.OrderExecutorMap.Bind() +} diff --git a/pkg/strategy/rebalance/position_map.go b/pkg/strategy/rebalance/position_map.go index 772d1726ce..73bdda4996 100644 --- a/pkg/strategy/rebalance/position_map.go +++ b/pkg/strategy/rebalance/position_map.go @@ -6,17 +6,17 @@ import ( type PositionMap map[string]*types.Position -func (m PositionMap) CreatePositions(markets []types.Market) PositionMap { - for _, market := range markets { - if _, ok := m[market.Symbol]; ok { +func (m PositionMap) CreatePositions(markets map[string]types.Market) PositionMap { + for symbol, market := range markets { + if _, ok := m[symbol]; ok { continue } - log.Infof("creating position for symbol %s", market.Symbol) + log.Infof("creating position for symbol %s", symbol) position := types.NewPositionFromMarket(market) position.Strategy = ID - position.StrategyInstanceID = instanceID(market.Symbol) - m[market.Symbol] = position + position.StrategyInstanceID = instanceID(symbol) + m[symbol] = position } return m } diff --git a/pkg/strategy/rebalance/profit_stats_map.go b/pkg/strategy/rebalance/profit_stats_map.go index a84bf5cc90..29e427a6e8 100644 --- a/pkg/strategy/rebalance/profit_stats_map.go +++ b/pkg/strategy/rebalance/profit_stats_map.go @@ -1,17 +1,19 @@ package rebalance -import "github.com/c9s/bbgo/pkg/types" +import ( + "github.com/c9s/bbgo/pkg/types" +) type ProfitStatsMap map[string]*types.ProfitStats -func (m ProfitStatsMap) CreateProfitStats(markets []types.Market) ProfitStatsMap { - for _, market := range markets { - if _, ok := m[market.Symbol]; ok { +func (m ProfitStatsMap) CreateProfitStats(markets map[string]types.Market) ProfitStatsMap { + for symbol, market := range markets { + if _, ok := m[symbol]; ok { continue } - log.Infof("creating profit stats for symbol %s", market.Symbol) - m[market.Symbol] = types.NewProfitStats(market) + log.Infof("creating profit stats for symbol %s", symbol) + m[symbol] = types.NewProfitStats(market) } return m } diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index e15e2507fe..22896d24b3 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/bbgo" @@ -15,6 +16,7 @@ import ( const ID = "rebalance" var log = logrus.WithField("strategy", ID) +var two = fixedpoint.NewFromFloat(2.0) func init() { bbgo.RegisterStrategy(ID, &Strategy{}) @@ -25,23 +27,24 @@ func instanceID(symbol string) string { } type Strategy struct { + *MultiMarketStrategy + Environment *bbgo.Environment - Interval types.Interval `json:"interval"` - QuoteCurrency string `json:"quoteCurrency"` - TargetWeights types.ValueMap `json:"targetWeights"` - Threshold fixedpoint.Value `json:"threshold"` - MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order - OrderType types.OrderType `json:"orderType"` - DryRun bool `json:"dryRun"` - OnStart bool `json:"onStart"` // rebalance on start - - PositionMap PositionMap `persistence:"positionMap"` - ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"` - - session *bbgo.ExchangeSession - orderExecutorMap GeneralOrderExecutorMap - activeOrderBook *bbgo.ActiveOrderBook + CronExpression string `json:"cronExpression"` + QuoteCurrency string `json:"quoteCurrency"` + TargetWeights types.ValueMap `json:"targetWeights"` + Threshold fixedpoint.Value `json:"threshold"` + MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order + OrderType types.OrderType `json:"orderType"` + DryRun bool `json:"dryRun"` + OnStart bool `json:"onStart"` // rebalance on start + + session *bbgo.ExchangeSession + symbols []string + markets map[string]types.Market + activeOrderBook *bbgo.ActiveOrderBook + cron *cron.Cron } func (s *Strategy) Defaults() error { @@ -52,6 +55,13 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { + for currency := range s.TargetWeights { + if currency == s.QuoteCurrency { + continue + } + + s.symbols = append(s.symbols, currency+s.QuoteCurrency) + } return nil } @@ -84,35 +94,22 @@ func (s *Strategy) Validate() error { return nil } -func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - for _, symbol := range s.symbols() { - session.Subscribe(types.KLineChannel, symbol, types.SubscribeOptions{Interval: s.Interval}) - } -} +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { s.session = session - markets, err := s.markets() - if err != nil { - return err - } - - if s.PositionMap == nil { - s.PositionMap = make(PositionMap) - } - s.PositionMap.CreatePositions(markets) - - if s.ProfitStatsMap == nil { - s.ProfitStatsMap = make(ProfitStatsMap) + s.markets = make(map[string]types.Market) + for _, symbol := range s.symbols { + market, ok := s.session.Market(symbol) + if !ok { + return fmt.Errorf("market %s not found", symbol) + } + s.markets[symbol] = market } - s.ProfitStatsMap.CreateProfitStats(markets) - s.orderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap) - s.orderExecutorMap.BindEnvironment(s.Environment) - s.orderExecutorMap.BindProfitStats(s.ProfitStatsMap) - s.orderExecutorMap.Bind() - s.orderExecutorMap.Sync(ctx, s) + s.MultiMarketStrategy = &MultiMarketStrategy{} + s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) s.activeOrderBook = bbgo.NewActiveOrderBook("") s.activeOrderBook.BindStream(s.session.UserDataStream) @@ -123,16 +120,18 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } }) - s.session.MarketDataStream.OnKLineClosed(func(kline types.KLine) { - s.rebalance(ctx) - }) - // the shutdown handler, you can cancel all orders bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() - _ = s.orderExecutorMap.GracefulCancel(ctx) + _ = s.OrderExecutorMap.GracefulCancel(ctx) }) + s.cron = cron.New() + s.cron.AddFunc(s.CronExpression, func() { + s.rebalance(ctx) + }) + s.cron.Start() + return nil } @@ -142,21 +141,24 @@ func (s *Strategy) rebalance(ctx context.Context) { log.WithError(err).Errorf("failed to cancel orders") } - submitOrders, err := s.generateSubmitOrders(ctx) + order, err := s.generateOrder(ctx) if err != nil { - log.WithError(err).Error("failed to generate submit orders") + log.WithError(err).Error("failed to generate order") return } - for _, order := range submitOrders { - log.Infof("generated submit order: %s", order.String()) + + if order == nil { + log.Info("no order generated") + return } + log.Infof("generated order: %s", order.String()) if s.DryRun { log.Infof("dry run, not submitting orders") return } - createdOrders, err := s.orderExecutorMap.SubmitOrders(ctx, submitOrders...) + createdOrders, err := s.OrderExecutorMap.SubmitOrders(ctx, *order) if err != nil { log.WithError(err).Error("failed to submit orders") return @@ -164,7 +166,7 @@ func (s *Strategy) rebalance(ctx context.Context) { s.activeOrderBook.Add(createdOrders...) } -func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) { +func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { m := make(types.ValueMap) for currency := range s.TargetWeights { if currency == s.QuoteCurrency { @@ -177,12 +179,12 @@ func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) { return nil, err } - m[currency] = ticker.Buy.Add(ticker.Sell).Div(fixedpoint.NewFromFloat(2.0)) + m[currency] = ticker.Buy.Add(ticker.Sell).Div(two) } return m, nil } -func (s *Strategy) balances() (types.BalanceMap, error) { +func (s *Strategy) selectBalances() (types.BalanceMap, error) { m := make(types.BalanceMap) balances := s.session.GetAccount().Balances() for currency := range s.TargetWeights { @@ -195,47 +197,37 @@ func (s *Strategy) balances() (types.BalanceMap, error) { return m, nil } -func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []types.SubmitOrder, err error) { - prices, err := s.prices(ctx) +func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error) { + prices, err := s.queryMidPrices(ctx) if err != nil { return nil, err } - balances, err := s.balances() + + balances, err := s.selectBalances() if err != nil { return nil, err } - marketValues := prices.Mul(balanceToTotal(balances)) - currentWeights := marketValues.Normalize() - for currency, targetWeight := range s.TargetWeights { - if currency == s.QuoteCurrency { - continue - } + values := prices.Mul(toValueMap(balances)) + weights := values.Normalize() - symbol := currency + s.QuoteCurrency - currentWeight := currentWeights[currency] - currentPrice := prices[currency] + for symbol, market := range s.markets { + target := s.TargetWeights[market.BaseCurrency] + weight := weights[market.BaseCurrency] + midPrice := prices[market.BaseCurrency] - log.Infof("%s price: %v, current weight: %v, target weight: %v", - symbol, - currentPrice, - currentWeight, - targetWeight) + log.Infof("%s mid price: %s", symbol, midPrice.String()) + log.Infof("%s weight: %.2f%%, target: %.2f%%", market.BaseCurrency, weight.Float64()*100, target.Float64()*100) // calculate the difference between current weight and target weight // if the difference is less than threshold, then we will not create the order - weightDifference := targetWeight.Sub(currentWeight) - if weightDifference.Abs().Compare(s.Threshold) < 0 { - log.Infof("%s weight distance |%v - %v| = |%v| less than the threshold: %v", - symbol, - currentWeight, - targetWeight, - weightDifference, - s.Threshold) + diff := target.Sub(weight) + if diff.Abs().Compare(s.Threshold) < 0 { + log.Infof("%s weight is close to target, skip", market.BaseCurrency) continue } - quantity := weightDifference.Mul(marketValues.Sum()).Div(currentPrice) + quantity := diff.Mul(values.Sum()).Div(midPrice) side := types.SideTypeBuy if quantity.Sign() < 0 { @@ -243,94 +235,47 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []typ quantity = quantity.Abs() } - maxAmount := s.adjustMaxAmountByBalance(side, currency, currentPrice, balances) - if maxAmount.Sign() > 0 { - quantity = bbgo.AdjustQuantityByMaxAmount(quantity, currentPrice, maxAmount) - log.Infof("adjust the quantity %v (%s %s @ %v) by max amount %v", - quantity, + if s.MaxAmount.Float64() > 0 { + quantity = bbgo.AdjustQuantityByMaxAmount(quantity, midPrice, s.MaxAmount) + log.Infof("adjust quantity %s (%s %s @ %s) by max amount %s", + quantity.String(), symbol, side.String(), - currentPrice, - s.MaxAmount) - } - - log.Debugf("symbol: %v, quantity: %v", symbol, quantity) - - order := types.SubmitOrder{ - Symbol: symbol, - Side: side, - Type: s.OrderType, - Quantity: quantity, - Price: currentPrice, + midPrice.String(), + s.MaxAmount.String()) } - if ok := s.checkMinimalOrderQuantity(order); ok { - submitOrders = append(submitOrders, order) + if side == types.SideTypeBuy { + quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(midPrice)) + } else if side == types.SideTypeSell { + quantity = fixedpoint.Min(quantity, balances[market.BaseCurrency].Available) } - } - - return submitOrders, err -} -func (s *Strategy) symbols() (symbols []string) { - for currency := range s.TargetWeights { - if currency == s.QuoteCurrency { + if market.IsDustQuantity(quantity, midPrice) { + log.Infof("quantity %s (%s %s @ %s) is dust quantity, skip", + quantity.String(), + symbol, + side.String(), + midPrice.String()) continue } - symbols = append(symbols, currency+s.QuoteCurrency) - } - return symbols -} - -func (s *Strategy) markets() ([]types.Market, error) { - markets := []types.Market{} - for _, symbol := range s.symbols() { - market, ok := s.session.Market(symbol) - if !ok { - return nil, fmt.Errorf("market %s not found", symbol) - } - markets = append(markets, market) - } - return markets, nil -} - -func (s *Strategy) adjustMaxAmountByBalance(side types.SideType, currency string, currentPrice fixedpoint.Value, balances types.BalanceMap) fixedpoint.Value { - var maxAmount fixedpoint.Value - - switch side { - case types.SideTypeBuy: - maxAmount = balances[s.QuoteCurrency].Available - case types.SideTypeSell: - maxAmount = balances[currency].Available.Mul(currentPrice) - default: - log.Errorf("unknown side type: %s", side) - return fixedpoint.Zero - } - - if s.MaxAmount.Sign() > 0 { - maxAmount = fixedpoint.Min(s.MaxAmount, maxAmount) - } - return maxAmount -} - -func (s *Strategy) checkMinimalOrderQuantity(order types.SubmitOrder) bool { - if order.Quantity.Compare(order.Market.MinQuantity) < 0 { - log.Infof("order quantity is too small: %f < %f", order.Quantity.Float64(), order.Market.MinQuantity.Float64()) - return false - } - - if order.Quantity.Mul(order.Price).Compare(order.Market.MinNotional) < 0 { - log.Infof("order min notional is too small: %f < %f", order.Quantity.Mul(order.Price).Float64(), order.Market.MinNotional.Float64()) - return false + return &types.SubmitOrder{ + Symbol: symbol, + Side: side, + Type: s.OrderType, + Quantity: quantity, + Price: midPrice, + }, nil } - return true + return nil, nil } -func balanceToTotal(balances types.BalanceMap) types.ValueMap { +func toValueMap(balances types.BalanceMap) types.ValueMap { m := make(types.ValueMap) for _, b := range balances { - m[b.Currency] = b.Total() + // m[b.Currency] = b.Net() + m[b.Currency] = b.Available } return m } From 1d2e46eca8c9eebd89c5d4b8b596d05d17136a7b Mon Sep 17 00:00:00 2001 From: Yu-Cheng Date: Tue, 31 Oct 2023 12:36:59 +0800 Subject: [PATCH 130/422] trade: query trades from db paginately --- pkg/service/trade.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/service/trade.go b/pkg/service/trade.go index ae8379fa0d..9020290a77 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -23,7 +23,12 @@ type QueryTradesOptions struct { Sessions []string Symbol string LastGID int64 - Since *time.Time + + // inclusive + Since *time.Time + + // exclusive + Until *time.Time // ASC or DESC Ordering string @@ -272,11 +277,19 @@ func (s *TradeService) Query(options QueryTradesOptions) ([]types.Trade, error) sel := sq.Select("*"). From("trades") + if options.LastGID != 0 { + sel = sel.Where(sq.Gt{"gid": options.LastGID}) + } if options.Since != nil { sel = sel.Where(sq.GtOrEq{"traded_at": options.Since}) } + if options.Until != nil { + sel = sel.Where(sq.Lt{"traded_at": options.Until}) + } - sel = sel.Where(sq.Eq{"symbol": options.Symbol}) + if options.Symbol != "" { + sel = sel.Where(sq.Eq{"symbol": options.Symbol}) + } if options.Exchange != "" { sel = sel.Where(sq.Eq{"exchange": options.Exchange}) @@ -412,4 +425,3 @@ func SelectLastTrades(ex types.ExchangeName, symbol string, isMargin, isFutures, OrderBy("traded_at DESC"). Limit(limit) } - From 4bc177f21bd0be289fd3d75753683447e26c1f55 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 31 Oct 2023 11:56:15 +0800 Subject: [PATCH 131/422] pkg/exchange: refactor get symbol api --- .../bitget/bitgetapi/get_symbols_request.go | 17 ++++- pkg/exchange/bitget/convert.go | 23 +++++++ pkg/exchange/bitget/convert_test.go | 66 +++++++++++++++++++ pkg/exchange/bitget/exchange.go | 32 ++++----- 4 files changed, 116 insertions(+), 22 deletions(-) create mode 100644 pkg/exchange/bitget/convert_test.go diff --git a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go index c3c4e64a96..48a202a2ef 100644 --- a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go +++ b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go @@ -9,6 +9,17 @@ import ( "github.com/c9s/bbgo/pkg/fixedpoint" ) +type SymbolStatus string + +const ( + // SymbolOffline represent market is suspended, users cannot trade. + SymbolOffline SymbolStatus = "offline" + // SymbolGray represents market is online, but user trading is not available. + SymbolGray SymbolStatus = "gray" + // SymbolOnline trading begins, users can trade. + SymbolOnline SymbolStatus = "online" +) + type Symbol struct { Symbol string `json:"symbol"` SymbolName string `json:"symbolName"` @@ -18,10 +29,10 @@ type Symbol struct { MaxTradeAmount fixedpoint.Value `json:"maxTradeAmount"` TakerFeeRate fixedpoint.Value `json:"takerFeeRate"` MakerFeeRate fixedpoint.Value `json:"makerFeeRate"` - PriceScale int `json:"priceScale"` - QuantityScale int `json:"quantityScale"` + PriceScale fixedpoint.Value `json:"priceScale"` + QuantityScale fixedpoint.Value `json:"quantityScale"` MinTradeUSDT fixedpoint.Value `json:"minTradeUSDT"` - Status string `json:"status"` + Status SymbolStatus `json:"status"` BuyLimitPriceRatio fixedpoint.Value `json:"buyLimitPriceRatio"` SellLimitPriceRatio fixedpoint.Value `json:"sellLimitPriceRatio"` } diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 0836b4dd07..f40d1d58a5 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,6 +1,7 @@ package bitget import ( + "math" "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" @@ -23,3 +24,25 @@ func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { MaxWithdrawAmount: fixedpoint.Zero, } } + +func toGlobalMarket(s bitgetapi.Symbol) types.Market { + if s.Status != bitgetapi.SymbolOnline { + log.Warnf("The symbol %s is not online", s.Symbol) + } + return types.Market{ + Symbol: s.SymbolName, + LocalSymbol: s.Symbol, + PricePrecision: s.PriceScale.Int(), + VolumePrecision: s.QuantityScale.Int(), + QuoteCurrency: s.QuoteCoin, + BaseCurrency: s.BaseCoin, + MinNotional: s.MinTradeUSDT, + MinAmount: s.MinTradeUSDT, + MinQuantity: s.MinTradeAmount, + MaxQuantity: s.MaxTradeAmount, + StepSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.QuantityScale.Int())), + TickSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.PriceScale.Int())), + MinPrice: fixedpoint.Zero, + MaxPrice: fixedpoint.Zero, + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go new file mode 100644 index 0000000000..6ab315eaa3 --- /dev/null +++ b/pkg/exchange/bitget/convert_test.go @@ -0,0 +1,66 @@ +package bitget + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestToGlobalMarket(t *testing.T) { + // sample: + //{ + // "symbol":"BTCUSDT_SPBL", + // "symbolName":"BTCUSDT", + // "baseCoin":"BTC", + // "quoteCoin":"USDT", + // "minTradeAmount":"0.0001", + // "maxTradeAmount":"10000", + // "takerFeeRate":"0.001", + // "makerFeeRate":"0.001", + // "priceScale":"4", + // "quantityScale":"8", + // "minTradeUSDT":"5", + // "status":"online", + // "buyLimitPriceRatio": "0.05", + // "sellLimitPriceRatio": "0.05" + // } + inst := bitgetapi.Symbol{ + Symbol: "BTCUSDT_SPBL", + SymbolName: "BTCUSDT", + BaseCoin: "BTC", + QuoteCoin: "USDT", + MinTradeAmount: fixedpoint.NewFromFloat(0.0001), + MaxTradeAmount: fixedpoint.NewFromFloat(10000), + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + PriceScale: fixedpoint.NewFromFloat(4), + QuantityScale: fixedpoint.NewFromFloat(8), + MinTradeUSDT: fixedpoint.NewFromFloat(5), + Status: bitgetapi.SymbolOnline, + BuyLimitPriceRatio: fixedpoint.NewFromFloat(0.05), + SellLimitPriceRatio: fixedpoint.NewFromFloat(0.05), + } + + exp := types.Market{ + Symbol: inst.SymbolName, + LocalSymbol: inst.Symbol, + PricePrecision: 4, + VolumePrecision: 8, + QuoteCurrency: inst.QuoteCoin, + BaseCurrency: inst.BaseCoin, + MinNotional: inst.MinTradeUSDT, + MinAmount: inst.MinTradeUSDT, + MinQuantity: inst.MinTradeAmount, + MaxQuantity: inst.MaxTradeAmount, + StepSize: fixedpoint.NewFromFloat(0.00000001), + MinPrice: fixedpoint.Zero, + MaxPrice: fixedpoint.Zero, + TickSize: fixedpoint.NewFromFloat(0.0001), + } + + assert.Equal(t, toGlobalMarket(inst), exp) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 0d17c1b006..b7af3828bd 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -2,12 +2,13 @@ package bitget import ( "context" - "math" + "fmt" + "time" "github.com/sirupsen/logrus" + "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -19,6 +20,11 @@ var log = logrus.WithFields(logrus.Fields{ "exchange": ID, }) +var ( + // queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols + queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) +) + type Exchange struct { key, secret, passphrase string @@ -54,7 +60,10 @@ func (e *Exchange) NewStream() types.Stream { } func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { - // TODO implement me + if err := queryMarketRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("markets rate limiter wait error: %w", err) + } + req := e.client.NewGetSymbolsRequest() symbols, err := req.Do(ctx) if err != nil { @@ -64,22 +73,7 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { markets := types.MarketMap{} for _, s := range symbols { symbol := toGlobalSymbol(s.SymbolName) - markets[symbol] = types.Market{ - Symbol: s.SymbolName, - LocalSymbol: s.Symbol, - PricePrecision: s.PriceScale, - VolumePrecision: s.QuantityScale, - QuoteCurrency: s.QuoteCoin, - BaseCurrency: s.BaseCoin, - MinNotional: s.MinTradeUSDT, - MinAmount: s.MinTradeUSDT, - MinQuantity: s.MinTradeAmount, - MaxQuantity: s.MaxTradeAmount, - StepSize: fixedpoint.NewFromFloat(math.Pow10(-s.QuantityScale)), - TickSize: fixedpoint.NewFromFloat(math.Pow10(-s.PriceScale)), - MinPrice: fixedpoint.Zero, - MaxPrice: fixedpoint.Zero, - } + markets[symbol] = toGlobalMarket(s) } return markets, nil From 64cad567274d8c0637dd5e50eef779490ff32478 Mon Sep 17 00:00:00 2001 From: Himanshu Kumar Mahto <93067059+HimanshuMahto@users.noreply.github.com> Date: Wed, 1 Nov 2023 02:20:58 +0530 Subject: [PATCH 132/422] grammatical errors in the README.md --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 12af0d06f1..762f0b0820 100644 --- a/README.md +++ b/README.md @@ -101,25 +101,25 @@ the implementation. | xnav | this strategy helps you record the current net asset value | tool | no | | xalign | this strategy aligns your balance position automatically | tool | no | | xfunding | a funding rate fee strategy | funding | no | -| autoborrow | this strategy uses margin to borrow assets, to help you keep the minimal balance | tool | no | -| pivotshort | this strategy finds the pivot low and entry the trade when the price breaks the previous low | long/short | | +| autoborrow | this strategy uses margin to borrow assets, to help you keep a minimal balance | tool | no | +| pivotshort | this strategy finds the pivot low and enters the trade when the price breaks the previous low | long/short | | | schedule | this strategy buy/sell with a fixed quantity periodically, you can use this as a single DCA, or to refill the fee asset like BNB. | tool | | irr | this strategy opens the position based on the predicated return rate | long/short | | -| bollmaker | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker | | -| wall | this strategy creates wall (large amount order) on the order book | maker | no | +| bollmaker | this strategy holds a long-term long/short position, places maker orders on both sides, and uses a bollinger band to control the position size | maker | | +| wall | this strategy creates a wall (large amount of order) on the order book | maker | no | | scmaker | this market making strategy is designed for stable coin markets, like USDC/USDT | maker | | | drift | | long/short | | -| rsicross | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator | long/short | | +| rsicross | this strategy opens a long position when the fast rsi crosses over the slow rsi, this is a demo strategy for using the v2 indicator | long/short | | | marketcap | this strategy implements a strategy that rebalances the portfolio based on the market capitalization | rebalance | no | | supertrend | this strategy uses DEMA and Supertrend indicator to open the long/short position | long/short | | -| trendtrader | this strategy opens long/short position based on the trendline breakout | long/short | | +| trendtrader | this strategy opens a long/short position based on the trendline breakout | long/short | | | elliottwave | | long/short | | | ewoDgtrd | | long/short | | | fixedmaker | | maker | | | factoryzoo | | long/short | | | fmaker | | maker | | | linregmaker | a linear regression based market maker | maker | | -| convert | convert strategy is a tool that helps you convert specific asset to a target asset | tool | no | +| convert | convert strategy is a tool that helps you convert a specific asset to a target asset | tool | no | @@ -250,7 +250,7 @@ To start bbgo with the frontend dashboard: bbgo run --enable-webserver ``` -If you want to switch to other dotenv file, you can add an `--dotenv` option or `--config`: +If you want to switch to another dotenv file, you can add an `--dotenv` option or `--config`: ```sh bbgo sync --dotenv .env.dev --config config/grid.yaml --session binance @@ -292,7 +292,7 @@ You could also add the script to crontab so that the system time could get synch ### Testnet (Paper Trading) -Currently only supports binance testnet. To run bbgo in testnet, apply new API keys +Currently only supports Binance testnet. To run bbgo in testnet, apply new API keys from [Binance Test Network](https://testnet.binance.vision), and set the following env before you start bbgo: ```bash @@ -319,7 +319,7 @@ You can only use one database driver MySQL or SQLite to store your trading data. #### Configure MySQL Database -To use MySQL database for data syncing, first, you need to install your mysql server: +To use MySQL database for data syncing, first, you need to install your MySQL server: ```sh # For Ubuntu Linux @@ -406,7 +406,7 @@ Check out the strategy directory [strategy](pkg/strategy) for all built-in strat - `drift` - drift strategy. - `grid2` - the second-generation grid strategy. -To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example if +To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example, if you want to run `buyandhold` strategy: @@ -524,7 +524,7 @@ bbgo userdatastream --session binance In order to minimize the strategy code, bbgo supports dynamic dependency injection. -Before executing your strategy, bbgo injects the components into your strategy object if it found the embedded field +Before executing your strategy, bbgo injects the components into your strategy object if it finds the embedded field that is using bbgo component. for example: ```go @@ -591,7 +591,7 @@ streambook.BindStream(stream) 1. Click the "Fork" button from the GitHub repository. 2. Clone your forked repository into `$GOPATH/github.com/c9s/bbgo`. -3. Change the directory into `$GOPATH/github.com/c9s/bbgo`. +3. Change the directory to `$GOPATH/github.com/c9s/bbgo`. 4. Create a branch and start your development. 5. Test your changes. 6. Push your changes to your fork. @@ -621,7 +621,7 @@ make embed && go run -tags web ./cmd/bbgo-lorca ### Looking For A New Strategy? You can write an article about BBGO on any topic, in 750-1500 words for exchange, and I can implement the strategy for -you (depending on the complexity and efforts). If you're interested in, DM me in telegram or +you (depending on the complexity and effort). If you're interested in, DM me in telegram or twitter , and we can discuss. ### Adding New Crypto Exchange support? From 102b662f7c6ebb763a1bc5f44428f9550f87735c Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 31 Oct 2023 21:53:13 +0800 Subject: [PATCH 133/422] pkg/exchange: support kline subscription on stream --- pkg/exchange/bitget/stream.go | 76 +++++++-- pkg/exchange/bitget/stream_callbacks.go | 10 ++ pkg/exchange/bitget/stream_test.go | 201 ++++++++++++++++++++++++ pkg/exchange/bitget/types.go | 132 ++++++++++++++++ pkg/exchange/bitget/types_test.go | 43 +++++ 5 files changed, 451 insertions(+), 11 deletions(-) create mode 100644 pkg/exchange/bitget/types_test.go diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 5a2f6f23d3..039c65127b 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -5,17 +5,11 @@ import ( "context" "encoding/json" "fmt" - "time" + "github.com/gorilla/websocket" + "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/types" - "github.com/gorilla/websocket" -) - -const ( - // Client should keep ping the server in every 30 seconds. Server will close the connections which has no ping over - // 120 seconds(even when the client is still receiving data from the server) - pingInterval = 30 * time.Second ) var ( @@ -29,11 +23,15 @@ type Stream struct { bookEventCallbacks []func(o BookEvent) marketTradeEventCallbacks []func(o MarketTradeEvent) + KLineEventCallbacks []func(o KLineEvent) + + lastCandle map[string]types.KLine } func NewStream() *Stream { stream := &Stream{ StandardStream: types.NewStandardStream(), + lastCandle: map[string]types.KLine{}, } stream.SetEndpointCreator(stream.createEndpoint) @@ -44,6 +42,7 @@ func NewStream() *Stream { stream.OnBookEvent(stream.handleBookEvent) stream.OnMarketTradeEvent(stream.handleMaretTradeEvent) + stream.OnKLineEvent(stream.handleKLineEvent) return stream } @@ -108,6 +107,9 @@ func (s *Stream) dispatchEvent(event interface{}) { case *MarketTradeEvent: s.EmitMarketTradeEvent(*e) + case *KLineEvent: + s.EmitKLineEvent(*e) + case []byte: // We only handle the 'pong' case. Others are unexpected. if !bytes.Equal(e, pongBytes) { @@ -171,6 +173,15 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { case types.MarketTradeChannel: arg.Channel = ChannelTrade return arg, nil + + case types.KLineChannel: + interval, found := toLocalInterval[sub.Options.Interval] + if !found { + return WsArg{}, fmt.Errorf("interval %s not supported on KLine subscription", sub.Options.Interval) + } + + arg.Channel = ChannelType(interval) + return arg, nil } return arg, fmt.Errorf("unsupported stream channel: %s", sub.Channel) @@ -200,7 +211,8 @@ func parseEvent(in []byte) (interface{}, error) { return &event, nil } - switch event.Arg.Channel { + ch := event.Arg.Channel + switch ch { case ChannelOrderBook, ChannelOrderBook5, ChannelOrderBook15: var book BookEvent err = json.Unmarshal(event.Data, &book.Events) @@ -222,9 +234,26 @@ func parseEvent(in []byte) (interface{}, error) { trade.actionType = event.Action trade.instId = event.Arg.InstId return &trade, nil - } - return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) + default: + + // handle the `KLine` case here to avoid complicating the code structure. + if strings.HasPrefix(string(ch), "candle") { + var kline KLineEvent + err = json.Unmarshal(event.Data, &kline.Events) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into KLineEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + kline.actionType = event.Action + kline.channel = ch + kline.instId = event.Arg.InstId + return &kline, nil + } + // return an error for any other case + + return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) + } } func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { @@ -242,3 +271,28 @@ func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { s.EmitMarketTrade(globalTrade) } } + +func (s *Stream) handleKLineEvent(k KLineEvent) { + if k.actionType == ActionTypeSnapshot { + // we don't support snapshot event + return + } + + interval, found := toGlobalInterval[string(k.channel)] + if !found { + log.Errorf("unexpected interval %s on KLine subscription", k.channel) + return + } + + for _, kline := range k.Events { + last, ok := s.lastCandle[k.CacheKey()] + if ok && kline.StartTime.Time().After(last.StartTime.Time()) { + last.Closed = true + s.EmitKLineClosed(last) + } + + kLine := kline.ToGlobal(interval, k.instId) + s.EmitKLine(kLine) + s.lastCandle[k.CacheKey()] = kLine + } +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go index 01da4388f8..82ef7beae3 100644 --- a/pkg/exchange/bitget/stream_callbacks.go +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -23,3 +23,13 @@ func (s *Stream) EmitMarketTradeEvent(o MarketTradeEvent) { cb(o) } } + +func (s *Stream) OnKLineEvent(cb func(o KLineEvent)) { + s.KLineEventCallbacks = append(s.KLineEventCallbacks, cb) +} + +func (s *Stream) EmitKLineEvent(o KLineEvent) { + for _, cb := range s.KLineEventCallbacks { + cb(o) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index c25fcd9426..b33e6afa20 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -106,6 +106,22 @@ func TestStream(t *testing.T) { <-c }) + t.Run("kline test", func(t *testing.T) { + s.Subscribe(types.KLineChannel, "BTCUSDT", types.SubscribeOptions{Interval: types.Interval1w}) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnKLine(func(kline types.KLine) { + t.Log("got update", kline) + }) + s.OnKLineClosed(func(kline types.KLine) { + t.Log("got closed update", kline) + }) + c := make(chan struct{}) + <-c + }) + } func TestStream_parseWebSocketEvent(t *testing.T) { @@ -453,6 +469,174 @@ func Test_parseWebSocketEvent_MarketTrade(t *testing.T) { }) } +func Test_parseWebSocketEvent_KLine(t *testing.T) { + t.Run("KLine event", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.49","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + + eventFn := func(in string, actionType ActionType) { + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + kline, ok := res.(*KLineEvent) + assert.True(t, ok) + assert.Equal(t, KLineEvent{ + channel: "candle5m", + Events: KLineSlice{ + { + StartTime: types.NewMillisecondTimestampFromInt(1698744600000), + OpenPrice: fixedpoint.NewFromFloat(34361.49), + HighestPrice: fixedpoint.NewFromFloat(34458.98), + LowestPrice: fixedpoint.NewFromFloat(34355.53), + ClosePrice: fixedpoint.NewFromFloat(34416.41), + Volume: fixedpoint.NewFromFloat(99.6631), + }, + }, + actionType: actionType, + instId: "BTCUSDT", + }, *kline) + } + + t.Run("snapshot type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeSnapshot) + eventFn(snapshotInput, ActionTypeSnapshot) + }) + + t.Run("update type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeUpdate) + eventFn(snapshotInput, ActionTypeUpdate) + }) + }) + + t.Run("Unexpected length of kline", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","34416.41","99.6631", "123456"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "unexpected kline length") + }) + + t.Run("Unexpected timestamp", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["timestamp","34361.49","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "timestamp") + }) + + t.Run("Unexpected open price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","1p","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "open price") + }) + + t.Run("Unexpected highest price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","3p","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "highest price") + }) + + t.Run("Unexpected lowest price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","1p","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "lowest price") + }) + + t.Run("Unexpected close price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","1c","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "close price") + }) + + t.Run("Unexpected volume", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","34416.41", "1v"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "volume") + }) +} + func Test_convertSubscription(t *testing.T) { t.Run("BookChannel.ChannelOrderBook5", func(t *testing.T) { res, err := convertSubscription(types.Subscription{ @@ -512,4 +696,21 @@ func Test_convertSubscription(t *testing.T) { InstId: "BTCUSDT", }, res) }) + t.Run("CandleChannel", func(t *testing.T) { + for gInterval, localInterval := range toLocalInterval { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.KLineChannel, + Options: types.SubscribeOptions{ + Interval: gInterval, + }, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelType(localInterval), + InstId: "BTCUSDT", + }, res) + } + }) } diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index df75f5bf25..a1107cad66 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -260,3 +261,134 @@ type MarketTradeEvent struct { actionType ActionType instId string } + +var ( + toLocalInterval = map[types.Interval]string{ + types.Interval1m: "candle1m", + types.Interval5m: "candle5m", + types.Interval15m: "candle15m", + types.Interval30m: "candle30m", + types.Interval1h: "candle1H", + types.Interval4h: "candle4H", + types.Interval12h: "candle12H", + types.Interval1d: "candle1D", + types.Interval1w: "candle1W", + } + + toGlobalInterval = map[string]types.Interval{ + "candle1m": types.Interval1m, + "candle5m": types.Interval5m, + "candle15m": types.Interval15m, + "candle30m": types.Interval30m, + "candle1H": types.Interval1h, + "candle4H": types.Interval4h, + "candle12H": types.Interval12h, + "candle1D": types.Interval1d, + "candle1W": types.Interval1w, + } +) + +type KLine struct { + StartTime types.MillisecondTimestamp + OpenPrice fixedpoint.Value + HighestPrice fixedpoint.Value + LowestPrice fixedpoint.Value + ClosePrice fixedpoint.Value + Volume fixedpoint.Value +} + +func (k KLine) ToGlobal(interval types.Interval, symbol string) types.KLine { + startTime := k.StartTime.Time() + + return types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(startTime), + EndTime: types.Time(startTime.Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: k.OpenPrice, + Close: k.ClosePrice, + High: k.HighestPrice, + Low: k.LowestPrice, + Volume: k.Volume, + QuoteVolume: fixedpoint.Zero, // not supported + TakerBuyBaseAssetVolume: fixedpoint.Zero, // not supported + TakerBuyQuoteAssetVolume: fixedpoint.Zero, // not supported + LastTradeID: 0, // not supported + NumberOfTrades: 0, // not supported + Closed: false, + } +} + +type KLineSlice []KLine + +func (m *KLineSlice) UnmarshalJSON(b []byte) error { + if m == nil { + return errors.New("nil pointer of kline slice") + } + s, err := parseKLineSliceJSON(b) + if err != nil { + return err + } + + *m = s + return nil +} + +// parseKLineSliceJSON tries to parse a 2 dimensional string array into a KLineSlice +// +// [ +// +// ["1597026383085", "8533.02", "8553.74", "8527.17", "8548.26", "45247"] +// ] +func parseKLineSliceJSON(in []byte) (slice KLineSlice, err error) { + var rawKLines [][]json.RawMessage + + err = json.Unmarshal(in, &rawKLines) + if err != nil { + return slice, err + } + + for _, raw := range rawKLines { + if len(raw) != 6 { + return nil, fmt.Errorf("unexpected kline length: %d, data: %q", len(raw), raw) + } + var kline KLine + if err = json.Unmarshal(raw[0], &kline.StartTime); err != nil { + return nil, fmt.Errorf("failed to unmarshal into timestamp: %q", raw[0]) + } + if err = json.Unmarshal(raw[1], &kline.OpenPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into open price: %q", raw[1]) + } + if err = json.Unmarshal(raw[2], &kline.HighestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into highest price: %q", raw[2]) + } + if err = json.Unmarshal(raw[3], &kline.LowestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into lowest price: %q", raw[3]) + } + if err = json.Unmarshal(raw[4], &kline.ClosePrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into close price: %q", raw[4]) + } + if err = json.Unmarshal(raw[5], &kline.Volume); err != nil { + return nil, fmt.Errorf("failed to unmarshal into volume: %q", raw[5]) + } + + slice = append(slice, kline) + } + + return slice, nil +} + +type KLineEvent struct { + Events KLineSlice + + // internal use + actionType ActionType + channel ChannelType + instId string +} + +func (k KLineEvent) CacheKey() string { + // e.q: candle5m.BTCUSDT + return fmt.Sprintf("%s.%s", k.channel, k.instId) +} diff --git a/pkg/exchange/bitget/types_test.go b/pkg/exchange/bitget/types_test.go new file mode 100644 index 0000000000..1850566201 --- /dev/null +++ b/pkg/exchange/bitget/types_test.go @@ -0,0 +1,43 @@ +package bitget + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestKLine_ToGlobal(t *testing.T) { + startTime := int64(1698744600000) + interval := types.Interval1m + k := KLine{ + StartTime: types.NewMillisecondTimestampFromInt(startTime), + OpenPrice: fixedpoint.NewFromFloat(34361.49), + HighestPrice: fixedpoint.NewFromFloat(34458.98), + LowestPrice: fixedpoint.NewFromFloat(34355.53), + ClosePrice: fixedpoint.NewFromFloat(34416.41), + Volume: fixedpoint.NewFromFloat(99.6631), + } + + assert.Equal(t, types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: "BTCUSDT", + StartTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time()), + EndTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(34361.49), + Close: fixedpoint.NewFromFloat(34416.41), + High: fixedpoint.NewFromFloat(34458.98), + Low: fixedpoint.NewFromFloat(34355.53), + Volume: fixedpoint.NewFromFloat(99.6631), + QuoteVolume: fixedpoint.Zero, + TakerBuyBaseAssetVolume: fixedpoint.Zero, + TakerBuyQuoteAssetVolume: fixedpoint.Zero, + LastTradeID: 0, + NumberOfTrades: 0, + Closed: false, + }, k.ToGlobal(interval, "BTCUSDT")) +} From 2cea0894043d2b00f3d898a26ac0c0b8eb3f5aee Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 1 Nov 2023 11:46:43 +0800 Subject: [PATCH 134/422] pkg/exchange: add rate limiter for query ticker, account --- pkg/exchange/bitget/convert_test.go | 32 ++++++++++++++++++++++++- pkg/exchange/bitget/exchange.go | 36 +++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 6ab315eaa3..5a80a045ac 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -10,7 +10,37 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func TestToGlobalMarket(t *testing.T) { +func Test_toGlobalBalance(t *testing.T) { + // sample: + // { + // "coinId":"10012", + // "coinName":"usdt", + // "available":"0", + // "frozen":"0", + // "lock":"0", + // "uTime":"1622697148" + // } + asset := bitgetapi.AccountAsset{ + CoinId: 2, + CoinName: "USDT", + Available: fixedpoint.NewFromFloat(1.2), + Frozen: fixedpoint.NewFromFloat(0.5), + Lock: fixedpoint.NewFromFloat(0.5), + UTime: types.NewMillisecondTimestampFromInt(1622697148), + } + + assert.Equal(t, types.Balance{ + Currency: "USDT", + Available: fixedpoint.NewFromFloat(1.2), + Locked: fixedpoint.NewFromFloat(1), // frozen + lock + Borrowed: fixedpoint.Zero, + Interest: fixedpoint.Zero, + NetAsset: fixedpoint.Zero, + MaxWithdrawAmount: fixedpoint.Zero, + }, toGlobalBalance(asset)) +} + +func Test_toGlobalMarket(t *testing.T) { // sample: //{ // "symbol":"BTCUSDT_SPBL", diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index b7af3828bd..a3a5700165 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -23,6 +23,10 @@ var log = logrus.WithFields(logrus.Fields{ var ( // queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryAccountRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-account-assets + queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker + queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { @@ -80,11 +84,15 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { } func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { + if err := queryTickerRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("ticker rate limiter wait error: %w", err) + } + req := e.client.NewGetTickerRequest() req.Symbol(symbol) ticker, err := req.Do(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to query ticker: %w", err) } return &types.Ticker{ @@ -110,10 +118,25 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { + bals, err := e.QueryAccountBalances(ctx) + if err != nil { + return nil, err + } + + account := types.NewAccount() + account.UpdateBalances(bals) + return account, nil +} + +func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { + if err := queryAccountRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("account rate limiter wait error: %w", err) + } + req := e.client.NewGetAccountAssetsRequest() resp, err := req.Do(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to query account assets: %w", err) } bals := types.BalanceMap{} @@ -122,14 +145,7 @@ func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { bals[asset.CoinName] = b } - account := types.NewAccount() - account.UpdateBalances(bals) - return account, nil -} - -func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { - // TODO implement me - panic("implement me") + return bals, nil } func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { From 470eb7dc0946a64c58eae9deaa5ad8860e5a0177 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 15:22:53 +0800 Subject: [PATCH 135/422] cmd: skip reports for session has no trade --- pkg/cmd/backtest.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 7ed612ee31..979412e9bc 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -528,6 +528,11 @@ var BacktestCmd = &cobra.Command{ for _, session := range environ.Sessions() { for symbol, trades := range session.Trades { + if len(trades.Trades) == 0 { + log.Warnf("session has no %s trades", symbol) + continue + } + tradeState := sessionTradeStats[session.Name][symbol] profitFactor := tradeState.ProfitFactor winningRatio := tradeState.WinningRatio @@ -598,8 +603,11 @@ var BacktestCmd = &cobra.Command{ }, } -func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, intervalProfit *types.IntervalProfitCollector, - profitFactor, winningRatio fixedpoint.Value) ( +func createSymbolReport( + userConfig *bbgo.Config, session *bbgo.ExchangeSession, symbol string, trades []types.Trade, + intervalProfit *types.IntervalProfitCollector, + profitFactor, winningRatio fixedpoint.Value, +) ( *backtest.SessionSymbolReport, error, ) { @@ -669,7 +677,10 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, return &symbolReport, nil } -func verify(userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time) error { +func verify( + userConfig *bbgo.Config, backtestService *service.BacktestService, + sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time, +) error { for _, sourceExchange := range sourceExchanges { err := backtestService.Verify(sourceExchange, userConfig.Backtest.Symbols, startTime, endTime) if err != nil { @@ -709,7 +720,10 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap { return types.SupportedIntervals } -func sync(ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time) error { +func sync( + ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, + sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time, +) error { for _, symbol := range userConfig.Backtest.Symbols { for _, sourceExchange := range sourceExchanges { var supportIntervals = getExchangeIntervals(sourceExchange) From 7a48d001a26aa64a3c122aafa575ef955fb3925e Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 15:23:27 +0800 Subject: [PATCH 136/422] backtest: return closed kline channel when empty symbol is given --- pkg/backtest/exchange.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index 851c4df14e..28a6b49a4e 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -382,6 +382,14 @@ func (e *Exchange) SubscribeMarketData( } log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) + if len(symbols) == 0 { + log.Warnf("empty symbols, will not query kline data from the database") + + c := make(chan types.KLine) + close(c) + return c, nil + } + klineC, errC := e.srv.QueryKLinesCh(startTime, endTime, e, symbols, intervals) go func() { if err := <-errC; err != nil { From 00d4805321064c055a16bce34ec5716317ff1ec7 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 1 Nov 2023 16:14:21 +0800 Subject: [PATCH 137/422] pkg/exchange: add query tickers api --- pkg/exchange/bitget/convert.go | 13 ++++++++ pkg/exchange/bitget/convert_test.go | 49 +++++++++++++++++++++++++++++ pkg/exchange/bitget/exchange.go | 48 +++++++++++++++++++--------- 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index f40d1d58a5..1339089fd1 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -46,3 +46,16 @@ func toGlobalMarket(s bitgetapi.Symbol) types.Market { MaxPrice: fixedpoint.Zero, } } + +func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker { + return types.Ticker{ + Time: ticker.Ts.Time(), + Volume: ticker.BaseVol, + Last: ticker.Close, + Open: ticker.OpenUtc0, + High: ticker.High24H, + Low: ticker.Low24H, + Buy: ticker.BuyOne, + Sell: ticker.SellOne, + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 5a80a045ac..770e4e5b8c 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -94,3 +94,52 @@ func Test_toGlobalMarket(t *testing.T) { assert.Equal(t, toGlobalMarket(inst), exp) } + +func Test_toGlobalTicker(t *testing.T) { + // sample: + // { + // "symbol": "BTCUSDT", + // "high24h": "24175.65", + // "low24h": "23677.75", + // "close": "24014.11", + // "quoteVol": "177689342.3025", + // "baseVol": "7421.5009", + // "usdtVol": "177689342.302407", + // "ts": "1660704288118", + // "buyOne": "24013.94", + // "sellOne": "24014.06", + // "bidSz": "0.0663", + // "askSz": "0.0119", + // "openUtc0": "23856.72", + // "changeUtc":"0.00301", + // "change":"0.00069" + // } + ticker := bitgetapi.Ticker{ + Symbol: "BTCUSDT", + High24H: fixedpoint.NewFromFloat(24175.65), + Low24H: fixedpoint.NewFromFloat(23677.75), + Close: fixedpoint.NewFromFloat(24014.11), + QuoteVol: fixedpoint.NewFromFloat(177689342.3025), + BaseVol: fixedpoint.NewFromFloat(7421.5009), + UsdtVol: fixedpoint.NewFromFloat(177689342.302407), + Ts: types.NewMillisecondTimestampFromInt(1660704288118), + BuyOne: fixedpoint.NewFromFloat(24013.94), + SellOne: fixedpoint.NewFromFloat(24014.06), + BidSz: fixedpoint.NewFromFloat(0.0663), + AskSz: fixedpoint.NewFromFloat(0.0119), + OpenUtc0: fixedpoint.NewFromFloat(23856.72), + ChangeUtc: fixedpoint.NewFromFloat(0.00301), + Change: fixedpoint.NewFromFloat(0.00069), + } + + assert.Equal(t, types.Ticker{ + Time: types.NewMillisecondTimestampFromInt(1660704288118).Time(), + Volume: fixedpoint.NewFromFloat(7421.5009), + Last: fixedpoint.NewFromFloat(24014.11), + Open: fixedpoint.NewFromFloat(23856.72), + High: fixedpoint.NewFromFloat(24175.65), + Low: fixedpoint.NewFromFloat(23677.75), + Buy: fixedpoint.NewFromFloat(24013.94), + Sell: fixedpoint.NewFromFloat(24014.06), + }, toGlobalTicker(ticker)) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index a3a5700165..700bd2ea7d 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -27,6 +27,8 @@ var ( queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) // queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers + queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { @@ -90,26 +92,44 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke req := e.client.NewGetTickerRequest() req.Symbol(symbol) - ticker, err := req.Do(ctx) + resp, err := req.Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query ticker: %w", err) } - return &types.Ticker{ - Time: ticker.Ts.Time(), - Volume: ticker.BaseVol, - Last: ticker.Close, - Open: ticker.OpenUtc0, - High: ticker.High24H, - Low: ticker.Low24H, - Buy: ticker.BuyOne, - Sell: ticker.SellOne, - }, nil + ticker := toGlobalTicker(*resp) + return &ticker, nil } -func (e *Exchange) QueryTickers(ctx context.Context, symbol ...string) (map[string]types.Ticker, error) { - // TODO implement me - panic("implement me") +func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[string]types.Ticker, error) { + tickers := map[string]types.Ticker{} + if len(symbols) > 0 { + for _, s := range symbols { + t, err := e.QueryTicker(ctx, s) + if err != nil { + return nil, err + } + + tickers[s] = *t + } + + return tickers, nil + } + + if err := queryTickersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("tickers rate limiter wait error: %w", err) + } + + resp, err := e.client.NewGetAllTickersRequest().Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query tickers: %w", err) + } + + for _, s := range resp { + tickers[s.Symbol] = toGlobalTicker(s) + } + + return tickers, nil } func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { From 9dc57f01cd54875bf1575f08fe81dc5b58e8eb8f Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 16:57:07 +0800 Subject: [PATCH 138/422] wall: refactor wall strategy with common.Strategy --- config/wall.yaml | 10 +++- pkg/strategy/wall/strategy.go | 109 ++++++++-------------------------- 2 files changed, 34 insertions(+), 85 deletions(-) diff --git a/config/wall.yaml b/config/wall.yaml index 6280882812..2f6104d3c1 100644 --- a/config/wall.yaml +++ b/config/wall.yaml @@ -10,6 +10,14 @@ sessions: exchange: max envVarPrefix: MAX + +logging: + trade: true + order: true + # fields: + # env: prod + + exchangeStrategies: - on: max @@ -33,6 +41,6 @@ exchangeStrategies: byLayer: linear: domain: [ 1, 3 ] - range: [ 10.0, 30.0 ] + range: [ 10000.0, 30000.0 ] diff --git a/pkg/strategy/wall/strategy.go b/pkg/strategy/wall/strategy.go index 5cbb4294ff..0967b241a8 100644 --- a/pkg/strategy/wall/strategy.go +++ b/pkg/strategy/wall/strategy.go @@ -6,12 +6,11 @@ import ( "sync" "time" - "github.com/c9s/bbgo/pkg/core" - "github.com/c9s/bbgo/pkg/util" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -31,9 +30,10 @@ func init() { } type Strategy struct { - Environment *bbgo.Environment - StandardIndicatorSet *bbgo.StandardIndicatorSet - Market types.Market + *common.Strategy + + Environment *bbgo.Environment + Market types.Market // Symbol is the market symbol you want to trade Symbol string `json:"symbol"` @@ -60,18 +60,8 @@ type Strategy struct { session *bbgo.ExchangeSession - // persistence fields - Position *types.Position `json:"position,omitempty" persistence:"position"` - ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` - activeAdjustmentOrders *bbgo.ActiveOrderBook activeWallOrders *bbgo.ActiveOrderBook - orderStore *core.OrderStore - tradeCollector *core.TradeCollector - - groupID uint32 - - stopC chan struct{} } func (s *Strategy) ID() string { @@ -149,7 +139,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo Price: askPrice, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, }) case types.SideTypeSell: @@ -175,7 +164,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo Price: bidPrice, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, }) } @@ -189,12 +177,13 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo return err } - s.orderStore.Add(createdOrders...) s.activeAdjustmentOrders.Add(createdOrders...) return nil } func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.OrderExecutor) error { + log.Infof("placing wall orders...") + var submitOrders []types.SubmitOrder var startPrice = s.FixedPrice for i := 0; i < s.NumLayers; i++ { @@ -217,7 +206,6 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order Price: price, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, } submitOrders = append(submitOrders, order) switch s.Side { @@ -240,74 +228,27 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order return err } - s.orderStore.Add(createdOrders...) + log.Infof("wall orders placed: %+v", createdOrders) + s.activeWallOrders.Add(createdOrders...) return err } -func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + // initial required information s.session = session - // calculate group id for orders - instanceID := s.InstanceID() - s.groupID = util.FNV32(instanceID) - - // If position is nil, we need to allocate a new position for calculation - if s.Position == nil { - s.Position = types.NewPositionFromMarket(s.Market) - } - - if s.ProfitStats == nil { - s.ProfitStats = types.NewProfitStats(s.Market) - } - - // Always update the position fields - s.Position.Strategy = ID - s.Position.StrategyInstanceID = instanceID - - s.stopC = make(chan struct{}) - s.activeWallOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeWallOrders.BindStream(session.UserDataStream) s.activeAdjustmentOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeAdjustmentOrders.BindStream(session.UserDataStream) - s.orderStore = core.NewOrderStore(s.Symbol) - s.orderStore.BindStream(session.UserDataStream) - - s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore) - - s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { - bbgo.Notify(trade) - s.ProfitStats.AddTrade(trade) - - if profit.Compare(fixedpoint.Zero) == 0 { - s.Environment.RecordPosition(s.Position, trade, nil) - } else { - log.Infof("%s generated profit: %v", s.Symbol, profit) - p := s.Position.NewProfit(trade, profit, netProfit) - p.Strategy = ID - p.StrategyInstanceID = instanceID - bbgo.Notify(&p) - - s.ProfitStats.AddProfit(p) - bbgo.Notify(&s.ProfitStats) - - s.Environment.RecordPosition(s.Position, trade, &p) - } - }) - - s.tradeCollector.OnPositionUpdate(func(position *types.Position) { - log.Infof("position changed: %s", s.Position) - bbgo.Notify(s.Position) - }) - - s.tradeCollector.BindStream(session.UserDataStream) - session.UserDataStream.OnStart(func() { - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -318,9 +259,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil { + if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -331,9 +272,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } @@ -342,9 +283,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil { + if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -365,9 +306,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } } @@ -377,7 +318,6 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() - close(s.stopC) if err := s.activeWallOrders.GracefulCancel(ctx, s.session.Exchange); err != nil { log.WithError(err).Errorf("graceful cancel order error") @@ -387,7 +327,8 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se log.WithError(err).Errorf("graceful cancel order error") } - s.tradeCollector.Process() + // check if there is a canceled order had partially filled. + s.OrderExecutor.TradeCollector().Process() }) return nil From ffea4901edd05c6174e232aaf6ad1a7c089046bc Mon Sep 17 00:00:00 2001 From: narumi Date: Fri, 3 Nov 2023 15:07:24 +0800 Subject: [PATCH 139/422] fix buy quantity --- pkg/strategy/rebalance/strategy.go | 45 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index 22896d24b3..40c0fa8211 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -40,7 +40,6 @@ type Strategy struct { DryRun bool `json:"dryRun"` OnStart bool `json:"onStart"` // rebalance on start - session *bbgo.ExchangeSession symbols []string markets map[string]types.Market activeOrderBook *bbgo.ActiveOrderBook @@ -97,11 +96,9 @@ func (s *Strategy) Validate() error { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.session = session - s.markets = make(map[string]types.Market) for _, symbol := range s.symbols { - market, ok := s.session.Market(symbol) + market, ok := session.Market(symbol) if !ok { return fmt.Errorf("market %s not found", symbol) } @@ -112,7 +109,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) s.activeOrderBook = bbgo.NewActiveOrderBook("") - s.activeOrderBook.BindStream(s.session.UserDataStream) + s.activeOrderBook.BindStream(session.UserDataStream) session.UserDataStream.OnStart(func() { if s.OnStart { @@ -137,7 +134,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. func (s *Strategy) rebalance(ctx context.Context) { // cancel active orders before rebalance - if err := s.session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { log.WithError(err).Errorf("failed to cancel orders") } @@ -174,7 +171,7 @@ func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { continue } - ticker, err := s.session.Exchange.QueryTicker(ctx, currency+s.QuoteCurrency) + ticker, err := s.Session.Exchange.QueryTicker(ctx, currency+s.QuoteCurrency) if err != nil { return nil, err } @@ -186,7 +183,7 @@ func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { func (s *Strategy) selectBalances() (types.BalanceMap, error) { m := make(types.BalanceMap) - balances := s.session.GetAccount().Balances() + balances := s.Session.GetAccount().Balances() for currency := range s.TargetWeights { balance, ok := balances[currency] if !ok { @@ -235,28 +232,36 @@ func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error quantity = quantity.Abs() } - if s.MaxAmount.Float64() > 0 { - quantity = bbgo.AdjustQuantityByMaxAmount(quantity, midPrice, s.MaxAmount) - log.Infof("adjust quantity %s (%s %s @ %s) by max amount %s", - quantity.String(), - symbol, - side.String(), - midPrice.String(), - s.MaxAmount.String()) + ticker, err := s.Session.Exchange.QueryTicker(ctx, symbol) + if err != nil { + return nil, err } + var price fixedpoint.Value if side == types.SideTypeBuy { - quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(midPrice)) + price = ticker.Buy + quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(ticker.Sell)) } else if side == types.SideTypeSell { + price = ticker.Sell quantity = fixedpoint.Min(quantity, balances[market.BaseCurrency].Available) } - if market.IsDustQuantity(quantity, midPrice) { + if s.MaxAmount.Float64() > 0 { + quantity = bbgo.AdjustQuantityByMaxAmount(quantity, price, s.MaxAmount) + log.Infof("adjusted quantity %s (%s %s @ %s) by max amount %s", + quantity.String(), + symbol, + side.String(), + price.String(), + s.MaxAmount.String()) + } + + if market.IsDustQuantity(quantity, price) { log.Infof("quantity %s (%s %s @ %s) is dust quantity, skip", quantity.String(), symbol, side.String(), - midPrice.String()) + price.String()) continue } @@ -265,7 +270,7 @@ func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error Side: side, Type: s.OrderType, Quantity: quantity, - Price: midPrice, + Price: price, }, nil } return nil, nil From cdebcc9a587eaf7bae49a1ff5aa8b22ec2278181 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 4 Nov 2023 12:55:22 +0800 Subject: [PATCH 140/422] add .env.local.example --- .env.local.example | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .env.local.example diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 0000000000..b6552d4d76 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,13 @@ +SLACK_TOKEN=YOUR_TOKEN +SLACK_CHANNEL=CHANNEL_NAME + +# DB_DRIVER="sqlite3" +# DB_DSN="bbgo.sqlite3" +DB_DRIVER=mysql +DB_DSN=root@tcp(127.0.0.1:3306)/bbgo + +MAX_API_KEY=YOUR_API_KEY +MAX_API_SECRET=YOUR_API_SECRET + +BINANCE_API_KEY=YOUR_API_KEY +BINANCE_API_SECRET=YOUR_API_SECRET From 6cce5a2268b80d18e1db4262b80c4fc80e046060 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 3 Nov 2023 18:12:42 +0800 Subject: [PATCH 141/422] grid2: respect s.BaseGridNum and add a failing test case --- pkg/strategy/grid2/strategy.go | 22 +++++- pkg/strategy/grid2/strategy_test.go | 113 +++++++++++++++++++++++----- 2 files changed, 114 insertions(+), 21 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index ce3f77dc2f..0c8248d93d 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -796,6 +796,8 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( if numberOfSellOrders > 0 { numberOfSellOrders-- } + + s.logger.Infof("calculated number of sell orders: %d", numberOfSellOrders) } // if the maxBaseQuantity is less than minQuantity, then we need to reduce the number of the sell orders @@ -810,8 +812,12 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( s.Market.MinQuantity) if baseQuantity.Compare(minBaseQuantity) <= 0 { + s.logger.Infof("base quantity %s is less than min base quantity: %s, adjusting...", baseQuantity.String(), minBaseQuantity.String()) + baseQuantity = s.Market.RoundUpQuantityByPrecision(minBaseQuantity) numberOfSellOrders = int(math.Floor(baseInvestment.Div(baseQuantity).Float64())) + + s.logger.Infof("adjusted base quantity to %s", baseQuantity.String()) } s.logger.Infof("grid base investment sell orders: %d", numberOfSellOrders) @@ -824,7 +830,8 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( // quoteInvestment = (p1 + p2 + p3) * q // maxBuyQuantity = quoteInvestment / (p1 + p2 + p3) si := -1 - for i := len(pins) - 1 - numberOfSellOrders; i >= 0; i-- { + end := len(pins) - 1 + for i := end - numberOfSellOrders - 1; i >= 0; i-- { pin := pins[i] price := fixedpoint.Value(pin) @@ -844,6 +851,7 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( // requiredQuote = requiredQuote.Add(quantity.Mul(nextLowerPrice)) totalQuotePrice = totalQuotePrice.Add(nextLowerPrice) } + } else { // for orders that buy if s.ProfitSpread.IsZero() && i+1 == si { @@ -851,7 +859,7 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( } // should never place a buy order at the upper price - if i == len(pins)-1 { + if i == end { continue } @@ -859,8 +867,11 @@ func (s *Strategy) calculateBaseQuoteInvestmentQuantity( } } + s.logger.Infof("total quote price: %f", totalQuotePrice.Float64()) if totalQuotePrice.Sign() > 0 && quoteInvestment.Sign() > 0 { quoteSideQuantity := quoteInvestment.Div(totalQuotePrice) + + s.logger.Infof("quote side quantity: %f = %f / %f", quoteSideQuantity.Float64(), quoteInvestment.Float64(), totalQuotePrice.Float64()) if numberOfSellOrders > 0 { return fixedpoint.Min(quoteSideQuantity, baseQuantity), nil } @@ -1058,6 +1069,11 @@ func (s *Strategy) openGrid(ctx context.Context, session *bbgo.ExchangeSession) return err2 } + if s.BaseGridNum > 0 { + sell1 := fixedpoint.Value(s.grid.Pins[len(s.grid.Pins)-1-s.BaseGridNum]) + lastPrice = sell1.Sub(s.Market.TickSize) + } + // check if base and quote are enough var totalBase = fixedpoint.Zero var totalQuote = fixedpoint.Zero @@ -1432,6 +1448,8 @@ func calculateMinimalQuoteInvestment(market types.Market, grid *Grid) fixedpoint for i := len(pins) - 2; i >= 0; i-- { pin := pins[i] price := fixedpoint.Value(pin) + + // TODO: should we round the quote here before adding? totalQuote = totalQuote.Add(price.Mul(minQuantity)) } diff --git a/pkg/strategy/grid2/strategy_test.go b/pkg/strategy/grid2/strategy_test.go index 9f592959f5..0110782dd6 100644 --- a/pkg/strategy/grid2/strategy_test.go +++ b/pkg/strategy/grid2/strategy_test.go @@ -204,6 +204,65 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) + t.Run("base and quote #2", func(t *testing.T) { + gridNum := int64(22) + upperPrice := number(35500.000000) + lowerPrice := number(34450.000000) + quoteInvestment := number(18.47) + baseInvestment := number(0.010700) + lastPrice := number(34522.930000) + baseGridNum := int(20) + + s := newTestStrategy() + s.GridNum = gridNum + s.BaseGridNum = baseGridNum + s.LowerPrice = lowerPrice + s.UpperPrice = upperPrice + s.grid = NewGrid(lowerPrice, upperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) + s.grid.CalculateArithmeticPins() + assert.Equal(t, 22, len(s.grid.Pins)) + + quantity, err := s.calculateBaseQuoteInvestmentQuantity(quoteInvestment, baseInvestment, lastPrice, s.grid.Pins) + assert.NoError(t, err) + assert.Equal(t, "0.000535", quantity.String()) + + s.QuantityOrAmount.Quantity = quantity + + orders, err := s.generateGridOrders(quoteInvestment, baseInvestment, lastPrice) + assert.NoError(t, err) + if !assert.Equal(t, 21, len(orders)) { + for _, o := range orders { + t.Logf("- %s %s", o.Price.String(), o.Side) + } + } + + assertPriceSide(t, []PriceSideAssert{ + {number(35500.0), types.SideTypeSell}, + {number(35450.0), types.SideTypeSell}, + {number(35400.0), types.SideTypeSell}, + {number(35350.0), types.SideTypeSell}, + {number(35300.0), types.SideTypeSell}, + {number(35250.0), types.SideTypeSell}, + {number(35200.0), types.SideTypeSell}, + {number(35150.0), types.SideTypeSell}, + {number(35100.0), types.SideTypeSell}, + {number(35050.0), types.SideTypeSell}, + {number(35000.0), types.SideTypeSell}, + {number(34950.0), types.SideTypeSell}, + {number(34900.0), types.SideTypeSell}, + {number(34850.0), types.SideTypeSell}, + {number(34800.0), types.SideTypeSell}, + {number(34750.0), types.SideTypeSell}, + {number(34700.0), types.SideTypeSell}, + {number(34650.0), types.SideTypeSell}, + {number(34600.0), types.SideTypeSell}, + {number(34550.0), types.SideTypeSell}, + // -- fake trade price at 34549.9 + // -- 34500 should be empty + {number(34450.0), types.SideTypeBuy}, + }, orders) + }) + t.Run("base and quote with pre-calculated baseGridNumber", func(t *testing.T) { s := newTestStrategy() s.grid = NewGrid(s.LowerPrice, s.UpperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) @@ -519,11 +578,11 @@ func newTestMarket(symbol string) types.Market { BaseCurrency: "BTC", QuoteCurrency: "USDT", TickSize: number(0.01), - StepSize: number(0.00001), + StepSize: number(0.000001), PricePrecision: 2, VolumePrecision: 8, - MinNotional: number(10.0), - MinQuantity: number(0.001), + MinNotional: number(8.0), + MinQuantity: number(0.0003), } case "ETHUSDT": return types.Market{ @@ -534,7 +593,7 @@ func newTestMarket(symbol string) types.Market { PricePrecision: 2, VolumePrecision: 6, MinNotional: number(8.000), - MinQuantity: number(0.00030), + MinQuantity: number(0.0046), } } @@ -577,12 +636,17 @@ func newTestOrder(price, quantity fixedpoint.Value, side types.SideType) types.O } } -func newTestStrategy() *Strategy { - market := newTestMarket("BTCUSDT") +func newTestStrategy(va ...string) *Strategy { + symbol := "BTCUSDT" + if len(va) > 0 { + symbol = va[0] + } + + market := newTestMarket(symbol) s := &Strategy{ logger: logrus.NewEntry(logrus.New()), - Symbol: "BTCUSDT", + Symbol: symbol, Market: market, GridProfitStats: newGridProfitStats(market), UpperPrice: number(20_000), @@ -790,7 +854,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -858,7 +924,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -946,7 +1014,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Market: s.Market, Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -963,7 +1033,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Market: s.Market, Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder2, order), "%+v is not equal to %+v", order, expectedSubmitOrder2) return []types.Order{ {SubmitOrder: expectedSubmitOrder2}, @@ -1060,7 +1132,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { } orderExecutor := gridmocks.NewMockOrderExecutor(mockCtrl) - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder, order), "%+v is not equal to %+v", order, expectedSubmitOrder) return []types.Order{ {SubmitOrder: expectedSubmitOrder}, @@ -1078,7 +1152,9 @@ func TestStrategy_handleOrderFilled(t *testing.T) { Tag: orderTag, } - orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func(ctx context.Context, order types.SubmitOrder) (types.OrderSlice, error) { + orderExecutor.EXPECT().SubmitOrders(ctx, gomock.Any()).DoAndReturn(func( + ctx context.Context, order types.SubmitOrder, + ) (types.OrderSlice, error) { assert.True(t, equalOrdersIgnoreClientOrderID(expectedSubmitOrder2, order), "%+v is not equal to %+v", order, expectedSubmitOrder2) return []types.Order{ {SubmitOrder: expectedSubmitOrder2}, @@ -1190,14 +1266,14 @@ func TestStrategy_aggregateOrderQuoteAmountAndFeeRetry(t *testing.T) { func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { t.Run("7 grids", func(t *testing.T) { - s := newTestStrategy() + s := newTestStrategy("ETHUSDT") s.UpperPrice = number(1660) s.LowerPrice = number(1630) s.QuoteInvestment = number(61) s.GridNum = 7 grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 60.46, minQuoteInvestment.Float64(), 0.01) + assert.InDelta(t, 48.36, minQuoteInvestment.Float64(), 0.01) err := s.checkMinimalQuoteInvestment(grid) assert.NoError(t, err) @@ -1207,12 +1283,11 @@ func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { s := newTestStrategy() // 10_000 * 0.001 = 10USDT // 20_000 * 0.001 = 20USDT - // hence we should have at least: 20USDT * 10 grids s.QuoteInvestment = number(10_000) s.GridNum = 10 grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 129.9999, minQuoteInvestment.Float64(), 0.01) + assert.InDelta(t, 103.999, minQuoteInvestment.Float64(), 0.01) err := s.checkMinimalQuoteInvestment(grid) assert.NoError(t, err) @@ -1225,11 +1300,11 @@ func TestStrategy_checkMinimalQuoteInvestment(t *testing.T) { grid := s.newGrid() minQuoteInvestment := calculateMinimalQuoteInvestment(s.Market, grid) - assert.InDelta(t, 14979.995499, minQuoteInvestment.Float64(), 0.001) + assert.InDelta(t, 11983.996400, minQuoteInvestment.Float64(), 0.001) err := s.checkMinimalQuoteInvestment(grid) assert.Error(t, err) - assert.EqualError(t, err, "need at least 14979.995500 USDT for quote investment, 10000.000000 USDT given") + assert.EqualError(t, err, "need at least 11983.996400 USDT for quote investment, 10000.000000 USDT given") }) } From e614741a48c56b89c415e34feab69fd2ed19a9e7 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 3 Nov 2023 18:22:33 +0800 Subject: [PATCH 142/422] grid2: add another test case for 0 baseGridNum --- pkg/strategy/grid2/strategy_test.go | 60 ++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/strategy_test.go b/pkg/strategy/grid2/strategy_test.go index 0110782dd6..70d6f97efc 100644 --- a/pkg/strategy/grid2/strategy_test.go +++ b/pkg/strategy/grid2/strategy_test.go @@ -204,7 +204,7 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) - t.Run("base and quote #2", func(t *testing.T) { + t.Run("base and quote with predefined base grid num", func(t *testing.T) { gridNum := int64(22) upperPrice := number(35500.000000) lowerPrice := number(34450.000000) @@ -263,6 +263,64 @@ func TestStrategy_generateGridOrders(t *testing.T) { }, orders) }) + t.Run("base and quote", func(t *testing.T) { + gridNum := int64(22) + upperPrice := number(35500.000000) + lowerPrice := number(34450.000000) + quoteInvestment := number(20.0) + baseInvestment := number(0.010700) + lastPrice := number(34522.930000) + baseGridNum := int(0) + + s := newTestStrategy() + s.GridNum = gridNum + s.BaseGridNum = baseGridNum + s.LowerPrice = lowerPrice + s.UpperPrice = upperPrice + s.grid = NewGrid(lowerPrice, upperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) + s.grid.CalculateArithmeticPins() + assert.Equal(t, 22, len(s.grid.Pins)) + + quantity, err := s.calculateBaseQuoteInvestmentQuantity(quoteInvestment, baseInvestment, lastPrice, s.grid.Pins) + assert.NoError(t, err) + assert.Equal(t, "0.00029006", quantity.String()) + + s.QuantityOrAmount.Quantity = quantity + + orders, err := s.generateGridOrders(quoteInvestment, baseInvestment, lastPrice) + assert.NoError(t, err) + if !assert.Equal(t, 21, len(orders)) { + for _, o := range orders { + t.Logf("- %s %s", o.Price.String(), o.Side) + } + } + + assertPriceSide(t, []PriceSideAssert{ + {number(35500.0), types.SideTypeSell}, + {number(35450.0), types.SideTypeSell}, + {number(35400.0), types.SideTypeSell}, + {number(35350.0), types.SideTypeSell}, + {number(35300.0), types.SideTypeSell}, + {number(35250.0), types.SideTypeSell}, + {number(35200.0), types.SideTypeSell}, + {number(35150.0), types.SideTypeSell}, + {number(35100.0), types.SideTypeSell}, + {number(35050.0), types.SideTypeSell}, + {number(35000.0), types.SideTypeSell}, + {number(34950.0), types.SideTypeSell}, + {number(34900.0), types.SideTypeSell}, + {number(34850.0), types.SideTypeSell}, + {number(34800.0), types.SideTypeSell}, + {number(34750.0), types.SideTypeSell}, + {number(34700.0), types.SideTypeSell}, + {number(34650.0), types.SideTypeSell}, + {number(34600.0), types.SideTypeSell}, + {number(34550.0), types.SideTypeSell}, + // -- 34500 should be empty + {number(34450.0), types.SideTypeBuy}, + }, orders) + }) + t.Run("base and quote with pre-calculated baseGridNumber", func(t *testing.T) { s := newTestStrategy() s.grid = NewGrid(s.LowerPrice, s.UpperPrice, fixedpoint.NewFromInt(s.GridNum), s.Market.TickSize) From 358aef770fb71346933f6a75aa4bc31230b97376 Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 6 Nov 2023 17:13:16 +0800 Subject: [PATCH 143/422] FIX: fix skip syncing active order --- pkg/strategy/grid2/active_order_recover.go | 11 ++++++++--- pkg/strategy/grid2/recover.go | 20 ++++++++++++++++++-- pkg/strategy/grid2/recover_test.go | 4 ++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index cfdaeac80b..91b52ed68b 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,6 +2,7 @@ package grid2 import ( "context" + "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -125,9 +126,13 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // sleep 100ms to avoid DDOS time.Sleep(100 * time.Millisecond) - if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID); err != nil { - opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) - errs = multierr.Append(errs, err) + if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + if strings.Contains(err.Error(), "skip syncing active order") { + opts.logger.Infof("[ActiveOrderRecover] skip syncing active order #%d, because the updated_at is in 3 min", activeOrder.OrderID) + } else { + opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + errs = multierr.Append(errs, err) + } continue } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 0634b72d43..4a6456884b 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -13,6 +14,8 @@ import ( "github.com/pkg/errors" ) +var syncWindow = -3 * time.Minute + /* Background knowledge 1. active orderbook add orders only when receive new order event or call Add/Update method manually @@ -91,6 +94,8 @@ func (s *Strategy) recover(ctx context.Context) error { pins := s.getGrid().Pins + syncBefore := time.Now().Add(syncWindow) + activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) @@ -135,7 +140,14 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { - syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, activeOrder.GetOrder().OrderID) + order := activeOrder.GetOrder() + if err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + if strings.Contains(err.Error(), "skip syncing active order") { + s.logger.Infof("[Recover] skip handle active order #%d, because the updated_at is in 3 min", order.OrderID) + } else { + s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) + } + } continue } @@ -250,7 +262,7 @@ func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error return book, nil } -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) error { updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), @@ -260,6 +272,10 @@ func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, return err } + if updatedOrder.UpdateTime.After(syncBefore) { + return fmt.Errorf("skip syncing active order, because its updated_at is after %s", syncBefore) + } + activeOrderBook.Update(*updatedOrder) return nil diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go index bdfd191eed..332c84d624 100644 --- a/pkg/strategy/grid2/recover_test.go +++ b/pkg/strategy/grid2/recover_test.go @@ -114,7 +114,7 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { return } @@ -144,7 +144,7 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { return } From dcff850c64e8b4ca7b160b1997b691b3b70da41c Mon Sep 17 00:00:00 2001 From: chiahung Date: Mon, 6 Nov 2023 18:52:01 +0800 Subject: [PATCH 144/422] FEATURE: add ttl for position/grid2.profit_stats persistence --- pkg/strategy/grid2/profit_stats.go | 11 +++++++++++ pkg/strategy/grid2/strategy.go | 5 +++++ pkg/types/position.go | 11 +++++++++++ 3 files changed, 27 insertions(+) diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index cd8367c23c..c396029370 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -24,6 +24,9 @@ type GridProfitStats struct { Market types.Market `json:"market,omitempty"` Since *time.Time `json:"since,omitempty"` InitialOrderID uint64 `json:"initialOrderID"` + + // ttl is the ttl to keep in persistence + ttl time.Duration } func newGridProfitStats(market types.Market) *GridProfitStats { @@ -40,6 +43,14 @@ func newGridProfitStats(market types.Market) *GridProfitStats { } } +func (s *GridProfitStats) SetTTL(ttl time.Duration) { + s.ttl = ttl +} + +func (s *GridProfitStats) Expiration() time.Duration { + return s.ttl +} + func (s *GridProfitStats) AddTrade(trade types.Trade) { if s.TotalFee == nil { s.TotalFee = make(map[string]fixedpoint.Value) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 0c8248d93d..bc361a72b2 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -177,6 +177,7 @@ type Strategy struct { GridProfitStats *GridProfitStats `persistence:"grid_profit_stats"` Position *types.Position `persistence:"position"` + PersistenceTTL types.Duration `json:"persistenceTTL"` // ExchangeSession is an injection field ExchangeSession *bbgo.ExchangeSession @@ -1835,13 +1836,17 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.ProfitSpread = s.Market.TruncatePrice(s.ProfitSpread) } + s.logger.Infof("ttl: %s", s.PersistenceTTL.Duration()) + if s.GridProfitStats == nil { s.GridProfitStats = newGridProfitStats(s.Market) } + s.GridProfitStats.SetTTL(s.PersistenceTTL.Duration()) if s.Position == nil { s.Position = types.NewPositionFromMarket(s.Market) } + s.Position.SetTTL(s.PersistenceTTL.Duration()) // initialize and register prometheus metrics if s.PrometheusLabels != nil { diff --git a/pkg/types/position.go b/pkg/types/position.go index 983c1c5511..4b649a48fa 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -65,6 +65,17 @@ type Position struct { // Modify position callbacks modifyCallbacks []func(baseQty fixedpoint.Value, quoteQty fixedpoint.Value, price fixedpoint.Value) + + // ttl is the ttl to keep in persistence + ttl time.Duration +} + +func (s *Position) SetTTL(ttl time.Duration) { + s.ttl = ttl +} + +func (s *Position) Expiration() time.Duration { + return s.ttl } func (p *Position) CsvHeader() []string { From 82ac8f184f9bdbe63a8cd868253ef99eb794f2d0 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 22:17:29 +0800 Subject: [PATCH 145/422] pkg/exchange: to periodically fetch the fee rate --- pkg/exchange/bybit/market_info_poller.go | 137 ++++++++++++++ pkg/exchange/bybit/market_info_poller_test.go | 173 ++++++++++++++++++ pkg/exchange/bybit/stream.go | 67 +------ pkg/exchange/bybit/stream_test.go | 135 +------------- 4 files changed, 328 insertions(+), 184 deletions(-) create mode 100644 pkg/exchange/bybit/market_info_poller.go create mode 100644 pkg/exchange/bybit/market_info_poller_test.go diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go new file mode 100644 index 0000000000..dac0172f9f --- /dev/null +++ b/pkg/exchange/bybit/market_info_poller.go @@ -0,0 +1,137 @@ +package bybit + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/util" +) + +const ( + // To maintain aligned fee rates, it's important to update fees frequently. + feeRatePollingPeriod = time.Minute +) + +type symbolFeeDetail struct { + bybitapi.FeeRate + + BaseCoin string + QuoteCoin string +} + +// feeRatePoller pulls the specified market data from bbgo QueryMarkets. +type feeRatePoller struct { + mu sync.Mutex + once sync.Once + client MarketInfoProvider + + symbolFeeDetail map[string]symbolFeeDetail +} + +func newFeeRatePoller(marketInfoProvider MarketInfoProvider) *feeRatePoller { + return &feeRatePoller{ + client: marketInfoProvider, + symbolFeeDetail: map[string]symbolFeeDetail{}, + } +} + +func (p *feeRatePoller) Start(ctx context.Context) { + p.once.Do(func() { + p.startLoop(ctx) + }) +} + +func (p *feeRatePoller) startLoop(ctx context.Context) { + ticker := time.NewTicker(feeRatePollingPeriod) + defer ticker.Stop() + + // Make sure the first poll should succeed by retrying with a shorter period. + _ = util.Retry(ctx, util.InfiniteRetry, 30*time.Second, + func() error { return p.poll(ctx) }, + func(e error) { log.WithError(e).Warn("failed to update fee rate") }) + + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); !errors.Is(err, context.Canceled) { + log.WithError(err).Error("context done with error") + } + + return + case <-ticker.C: + if err := p.poll(ctx); err != nil { + log.WithError(err).Warn("failed to update fee rate") + } + } + } +} + +func (p *feeRatePoller) poll(ctx context.Context) error { + symbolFeeRate, err := p.getAllFeeRates(ctx) + if err != nil { + return err + } + + p.mu.Lock() + p.symbolFeeDetail = symbolFeeRate + p.mu.Unlock() + + return nil +} + +func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, error) { + p.mu.Lock() + defer p.mu.Unlock() + + fee, ok := p.symbolFeeDetail[symbol] + if !ok { + return symbolFeeDetail{}, fmt.Errorf("%s fee rate not found", symbol) + } + return fee, nil +} + +func (e *feeRatePoller) getAllFeeRates(ctx context.Context) (map[string]symbolFeeDetail, error) { + feeRates, err := e.client.GetAllFeeRates(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get fee rates: %w", err) + } + + symbolMap := map[string]symbolFeeDetail{} + for _, f := range feeRates.List { + if _, found := symbolMap[f.Symbol]; !found { + symbolMap[f.Symbol] = symbolFeeDetail{FeeRate: f} + } + } + + mkts, err := e.client.QueryMarkets(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get markets: %w", err) + } + + // update base coin, quote coin into symbolFeeDetail + for _, mkt := range mkts { + feeRate, found := symbolMap[mkt.Symbol] + if !found { + continue + } + + feeRate.BaseCoin = mkt.BaseCurrency + feeRate.QuoteCoin = mkt.QuoteCurrency + + symbolMap[mkt.Symbol] = feeRate + } + + // remove trading pairs that are not present in spot market. + for k, v := range symbolMap { + if len(v.BaseCoin) == 0 || len(v.QuoteCoin) == 0 { + log.Debugf("related market not found: %s, skipping the associated trade", k) + delete(symbolMap, k) + } + } + + return symbolMap, nil +} diff --git a/pkg/exchange/bybit/market_info_poller_test.go b/pkg/exchange/bybit/market_info_poller_test.go new file mode 100644 index 0000000000..f2b58c466b --- /dev/null +++ b/pkg/exchange/bybit/market_info_poller_test.go @@ -0,0 +1,173 @@ +package bybit + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/exchange/bybit/mocks" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestFeeRatePoller_getAllFeeRates(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + unknownErr := errors.New("unknown err") + + t.Run("succeeds", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + feeRates := bybitapi.FeeRates{ + List: []bybitapi.FeeRate{ + { + Symbol: "BTCUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "ETHUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "OPTIONCOIN", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + }, + } + + mkts := types.MarketMap{ + "BTCUSDT": types.Market{ + Symbol: "BTCUSDT", + QuoteCurrency: "USDT", + BaseCurrency: "BTC", + }, + "ETHUSDT": types.Market{ + Symbol: "ETHUSDT", + QuoteCurrency: "USDT", + BaseCurrency: "ETH", + }, + } + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) + mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(mkts, nil).Times(1) + + expFeeRates := map[string]symbolFeeDetail{ + "BTCUSDT": { + FeeRate: feeRates.List[0], + BaseCoin: "BTC", + QuoteCoin: "USDT", + }, + "ETHUSDT": { + FeeRate: feeRates.List[1], + BaseCoin: "ETH", + QuoteCoin: "USDT", + }, + } + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.NoError(t, err) + assert.Equal(t, expFeeRates, symbolFeeDetails) + }) + + t.Run("failed to query markets", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + feeRates := bybitapi.FeeRates{ + List: []bybitapi.FeeRate{ + { + Symbol: "BTCUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "ETHUSDT", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + { + Symbol: "OPTIONCOIN", + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + }, + }, + } + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) + mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(nil, unknownErr).Times(1) + + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.Equal(t, fmt.Errorf("failed to get markets: %w", unknownErr), err) + assert.Equal(t, map[string]symbolFeeDetail(nil), symbolFeeDetails) + }) + + t.Run("failed to get fee rates", func(t *testing.T) { + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + s := &feeRatePoller{ + client: mockMarketProvider, + } + + ctx := context.Background() + + mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(bybitapi.FeeRates{}, unknownErr).Times(1) + + symbolFeeDetails, err := s.getAllFeeRates(ctx) + assert.Equal(t, fmt.Errorf("failed to call get fee rates: %w", unknownErr), err) + assert.Equal(t, map[string]symbolFeeDetail(nil), symbolFeeDetails) + }) +} + +func Test_feeRatePoller_Get(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) + t.Run("succeeds", func(t *testing.T) { + symbol := "BTCUSDT" + expFeeDetail := symbolFeeDetail{ + FeeRate: bybitapi.FeeRate{ + Symbol: symbol, + TakerFeeRate: fixedpoint.NewFromFloat(0.1), + MakerFeeRate: fixedpoint.NewFromFloat(0.2), + }, + BaseCoin: "BTC", + QuoteCoin: "USDT", + } + + s := &feeRatePoller{ + client: mockMarketProvider, + symbolFeeDetail: map[string]symbolFeeDetail{ + symbol: expFeeDetail, + }, + } + + res, err := s.Get(symbol) + assert.NoError(t, err) + assert.Equal(t, expFeeDetail, res) + }) + t.Run("succeeds", func(t *testing.T) { + symbol := "BTCUSDT" + s := &feeRatePoller{ + client: mockMarketProvider, + symbolFeeDetail: map[string]symbolFeeDetail{}, + } + + _, err := s.Get(symbol) + assert.ErrorContains(t, err, symbol) + }) +} diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index eb4137ed37..2bb2626575 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -47,8 +47,7 @@ type Stream struct { key, secret string streamDataProvider StreamDataProvider - // TODO: update the fee rate at 7:00 am UTC; rotation required. - symbolFeeDetails map[string]*symbolFeeDetail + feeRateProvider *feeRatePoller bookEventCallbacks []func(e BookEvent) marketTradeEventCallbacks []func(e []MarketTradeEvent) @@ -65,13 +64,17 @@ func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream key: key, secret: secret, streamDataProvider: userDataProvider, + feeRateProvider: newFeeRatePoller(userDataProvider), } stream.SetEndpointCreator(stream.createEndpoint) stream.SetParser(stream.parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) stream.SetHeartBeat(stream.ping) - stream.SetBeforeConnect(stream.getAllFeeRates) + stream.SetBeforeConnect(func(ctx context.Context) error { + go stream.feeRateProvider.Start(ctx) + return nil + }) stream.OnConnect(stream.handlerConnect) stream.OnAuth(stream.handleAuthEvent) @@ -403,13 +406,13 @@ func (s *Stream) handleKLineEvent(klineEvent KLineEvent) { func (s *Stream) handleTradeEvent(events []TradeEvent) { for _, event := range events { - feeRate, found := s.symbolFeeDetails[event.Symbol] - if !found { - log.Warnf("unexpected symbol found, fee rate not supported, symbol: %s", event.Symbol) + feeRate, err := s.feeRateProvider.Get(event.Symbol) + if err != nil { + log.Warnf("failed to get fee rate by symbol: %s", event.Symbol) continue } - gTrade, err := event.toGlobalTrade(*feeRate) + gTrade, err := event.toGlobalTrade(feeRate) if err != nil { log.WithError(err).Errorf("unable to convert: %+v", event) continue @@ -417,53 +420,3 @@ func (s *Stream) handleTradeEvent(events []TradeEvent) { s.StandardStream.EmitTradeUpdate(*gTrade) } } - -type symbolFeeDetail struct { - bybitapi.FeeRate - - BaseCoin string - QuoteCoin string -} - -// getAllFeeRates retrieves all fee rates from the Bybit API and then fetches markets to ensure the base coin and quote coin -// are correct. -func (e *Stream) getAllFeeRates(ctx context.Context) error { - feeRates, err := e.streamDataProvider.GetAllFeeRates(ctx) - if err != nil { - return fmt.Errorf("failed to call get fee rates: %w", err) - } - - symbolMap := map[string]*symbolFeeDetail{} - for _, f := range feeRates.List { - if _, found := symbolMap[f.Symbol]; !found { - symbolMap[f.Symbol] = &symbolFeeDetail{FeeRate: f} - } - } - - mkts, err := e.streamDataProvider.QueryMarkets(ctx) - if err != nil { - return fmt.Errorf("failed to get markets: %w", err) - } - - // update base coin, quote coin into symbolFeeDetail - for _, mkt := range mkts { - feeRate, found := symbolMap[mkt.Symbol] - if !found { - continue - } - - feeRate.BaseCoin = mkt.BaseCurrency - feeRate.QuoteCoin = mkt.QuoteCurrency - } - - // remove trading pairs that are not present in spot market. - for k, v := range symbolMap { - if len(v.BaseCoin) == 0 || len(v.QuoteCoin) == 0 { - log.Debugf("related market not found: %s, skipping the associated trade", k) - delete(symbolMap, k) - } - } - - e.symbolFeeDetails = symbolMap - return nil -} diff --git a/pkg/exchange/bybit/stream_test.go b/pkg/exchange/bybit/stream_test.go index e1fec2af80..8ed651022b 100644 --- a/pkg/exchange/bybit/stream_test.go +++ b/pkg/exchange/bybit/stream_test.go @@ -9,11 +9,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" - "github.com/c9s/bbgo/pkg/exchange/bybit/mocks" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/testutil" "github.com/c9s/bbgo/pkg/types" @@ -36,7 +34,7 @@ func getTestClientOrSkip(t *testing.T) *Stream { } func TestStream(t *testing.T) { - t.Skip() + //t.Skip() s := getTestClientOrSkip(t) symbols := []string{ @@ -70,12 +68,12 @@ func TestStream(t *testing.T) { err := s.Connect(context.Background()) assert.NoError(t, err) - s.OnBookSnapshot(func(book types.SliceOrderBook) { - t.Log("got snapshot", book) - }) - s.OnBookUpdate(func(book types.SliceOrderBook) { - t.Log("got update", book) - }) + //s.OnBookSnapshot(func(book types.SliceOrderBook) { + // t.Log("got snapshot", book) + //}) + //s.OnBookUpdate(func(book types.SliceOrderBook) { + // t.Log("got update", book) + //}) c := make(chan struct{}) <-c }) @@ -175,7 +173,7 @@ func TestStream(t *testing.T) { assert.NoError(t, err) s.OnTradeUpdate(func(trade types.Trade) { - t.Log("got update", trade) + t.Log("got update", trade.Fee, trade.FeeCurrency, trade) }) c := make(chan struct{}) <-c @@ -467,120 +465,3 @@ func Test_convertSubscription(t *testing.T) { assert.Equal(t, genTopic(TopicTypeMarketTrade, "BTCUSDT"), res) }) } - -func TestStream_getFeeRate(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - unknownErr := errors.New("unknown err") - - t.Run("succeeds", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - feeRates := bybitapi.FeeRates{ - List: []bybitapi.FeeRate{ - { - Symbol: "BTCUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "ETHUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "OPTIONCOIN", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - }, - } - - mkts := types.MarketMap{ - "BTCUSDT": types.Market{ - Symbol: "BTCUSDT", - QuoteCurrency: "USDT", - BaseCurrency: "BTC", - }, - "ETHUSDT": types.Market{ - Symbol: "ETHUSDT", - QuoteCurrency: "USDT", - BaseCurrency: "ETH", - }, - } - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) - mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(mkts, nil).Times(1) - - expFeeRates := map[string]*symbolFeeDetail{ - "BTCUSDT": { - FeeRate: feeRates.List[0], - BaseCoin: "BTC", - QuoteCoin: "USDT", - }, - "ETHUSDT": { - FeeRate: feeRates.List[1], - BaseCoin: "ETH", - QuoteCoin: "USDT", - }, - } - err := s.getAllFeeRates(ctx) - assert.NoError(t, err) - assert.Equal(t, expFeeRates, s.symbolFeeDetails) - }) - - t.Run("failed to query markets", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - feeRates := bybitapi.FeeRates{ - List: []bybitapi.FeeRate{ - { - Symbol: "BTCUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "ETHUSDT", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - { - Symbol: "OPTIONCOIN", - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - }, - }, - } - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(feeRates, nil).Times(1) - mockMarketProvider.EXPECT().QueryMarkets(ctx).Return(nil, unknownErr).Times(1) - - err := s.getAllFeeRates(ctx) - assert.Equal(t, fmt.Errorf("failed to get markets: %w", unknownErr), err) - assert.Equal(t, map[string]*symbolFeeDetail(nil), s.symbolFeeDetails) - }) - - t.Run("failed to get fee rates", func(t *testing.T) { - mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - s := &Stream{ - streamDataProvider: mockMarketProvider, - } - - ctx := context.Background() - - mockMarketProvider.EXPECT().GetAllFeeRates(ctx).Return(bybitapi.FeeRates{}, unknownErr).Times(1) - - err := s.getAllFeeRates(ctx) - assert.Equal(t, fmt.Errorf("failed to call get fee rates: %w", unknownErr), err) - assert.Equal(t, map[string]*symbolFeeDetail(nil), s.symbolFeeDetails) - }) -} From f595cc9cc05bac4501377af6617a9b5dd61175cd Mon Sep 17 00:00:00 2001 From: Edwin Date: Sun, 5 Nov 2023 23:41:57 +0800 Subject: [PATCH 146/422] pkg/exchange: add query open orders --- pkg/exchange/bitget/bitgetapi/v2/client.go | 17 ++ .../bitget/bitgetapi/v2/client_test.go | 42 ++++ .../v2/get_unfilled_orders_request.go | 50 ++++ .../get_unfilled_orders_request_requestgen.go | 221 ++++++++++++++++++ pkg/exchange/bitget/bitgetapi/v2/types.go | 42 ++++ pkg/exchange/bitget/convert.go | 99 ++++++++ pkg/exchange/bitget/convert_test.go | 129 ++++++++++ pkg/exchange/bitget/exchange.go | 52 ++++- 8 files changed, 647 insertions(+), 5 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/client.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/client_test.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/types.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client.go b/pkg/exchange/bitget/bitgetapi/v2/client.go new file mode 100644 index 0000000000..3a2b2204d5 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/client.go @@ -0,0 +1,17 @@ +package bitgetapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" +) + +type APIResponse = bitgetapi.APIResponse + +type Client struct { + Client requestgen.AuthenticatedAPIClient +} + +func NewClient(client *bitgetapi.RestClient) *Client { + return &Client{Client: client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go new file mode 100644 index 0000000000..21c86dfc6b --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -0,0 +1,42 @@ +package bitgetapi + +import ( + "context" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/testutil" +) + +func getTestClientOrSkip(t *testing.T) *Client { + if b, _ := strconv.ParseBool(os.Getenv("CI")); b { + t.Skip("skip test for CI") + } + + key, secret, ok := testutil.IntegrationTestConfigured(t, "BITGET") + if !ok { + t.Skip("BITGET_* env vars are not configured") + return nil + } + + client := bitgetapi.NewClient() + client.Auth(key, secret, os.Getenv("BITGET_API_PASSPHRASE")) + + return NewClient(client) +} + +func TestClient(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + + t.Run("GetUnfilledOrdersRequest", func(t *testing.T) { + req := client.NewGetUnfilledOrdersRequest().StartTime(1) + resp, err := req.Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go new file mode 100644 index 0000000000..178b31bbac --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go @@ -0,0 +1,50 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type UnfilledOrder struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + // Size is base coin when orderType=limit; quote coin when orderType=market + Size fixedpoint.Value `json:"size"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + Status OrderStatus `json:"status"` + BasePrice fixedpoint.Value `json:"basePrice"` + BaseVolume fixedpoint.Value `json:"baseVolume"` + QuoteVolume fixedpoint.Value `json:"quoteVolume"` + EnterPointSource string `json:"enterPointSource"` + OrderSource string `json:"orderSource"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` +} + +//go:generate GetRequest -url "/api/v2/spot/trade/unfilled-orders" -type GetUnfilledOrdersRequest -responseDataType []UnfilledOrder +type GetUnfilledOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol *string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (c *Client) NewGetUnfilledOrdersRequest() *GetUnfilledOrdersRequest { + return &GetUnfilledOrdersRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go new file mode 100644 index 0000000000..a3bb598193 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go @@ -0,0 +1,221 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/unfilled-orders -type GetUnfilledOrdersRequest -responseDataType []UnfilledOrder"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (g *GetUnfilledOrdersRequest) Symbol(symbol string) *GetUnfilledOrdersRequest { + g.symbol = &symbol + return g +} + +func (g *GetUnfilledOrdersRequest) Limit(limit string) *GetUnfilledOrdersRequest { + g.limit = &limit + return g +} + +func (g *GetUnfilledOrdersRequest) IdLessThan(idLessThan string) *GetUnfilledOrdersRequest { + g.idLessThan = &idLessThan + return g +} + +func (g *GetUnfilledOrdersRequest) StartTime(startTime int64) *GetUnfilledOrdersRequest { + g.startTime = &startTime + return g +} + +func (g *GetUnfilledOrdersRequest) EndTime(endTime int64) *GetUnfilledOrdersRequest { + g.endTime = &endTime + return g +} + +func (g *GetUnfilledOrdersRequest) OrderId(orderId string) *GetUnfilledOrdersRequest { + g.orderId = &orderId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetUnfilledOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if g.idLessThan != nil { + idLessThan := *g.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if g.orderId != nil { + orderId := *g.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetUnfilledOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetUnfilledOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetUnfilledOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetUnfilledOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetUnfilledOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetUnfilledOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetUnfilledOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetUnfilledOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetUnfilledOrdersRequest) Do(ctx context.Context) ([]UnfilledOrder, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/unfilled-orders" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data []UnfilledOrder + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/types.go b/pkg/exchange/bitget/bitgetapi/v2/types.go new file mode 100644 index 0000000000..82a91859c5 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/types.go @@ -0,0 +1,42 @@ +package bitgetapi + +type SideType string + +const ( + SideTypeBuy SideType = "buy" + SideTypeSell SideType = "sell" +) + +type OrderType string + +const ( + OrderTypeLimit OrderType = "limit" + OrderTypeMarket OrderType = "market" +) + +type OrderForce string + +const ( + OrderForceGTC OrderForce = "gtc" + OrderForcePostOnly OrderForce = "post_only" + OrderForceFOK OrderForce = "fok" + OrderForceIOC OrderForce = "ioc" +) + +type OrderStatus string + +const ( + OrderStatusInit OrderStatus = "init" + OrderStatusNew OrderStatus = "new" + OrderStatusLive OrderStatus = "live" + OrderStatusPartialFilled OrderStatus = "partially_filled" + OrderStatusFilled OrderStatus = "filled" + OrderStatusCancelled OrderStatus = "cancelled" +) + +func (o OrderStatus) IsWorking() bool { + return o == OrderStatusInit || + o == OrderStatusNew || + o == OrderStatusLive || + o == OrderStatusPartialFilled +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 1339089fd1..a4a9756d05 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,10 +1,13 @@ package bitget import ( + "fmt" "math" + "strconv" "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -59,3 +62,99 @@ func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker { Sell: ticker.SellOne, } } + +func toGlobalSideType(side v2.SideType) (types.SideType, error) { + switch side { + case v2.SideTypeBuy: + return types.SideTypeBuy, nil + + case v2.SideTypeSell: + return types.SideTypeSell, nil + + default: + return types.SideType(side), fmt.Errorf("unexpected side: %s", side) + } +} + +func toGlobalOrderType(s v2.OrderType) (types.OrderType, error) { + switch s { + case v2.OrderTypeMarket: + return types.OrderTypeMarket, nil + + case v2.OrderTypeLimit: + return types.OrderTypeLimit, nil + + default: + return types.OrderType(s), fmt.Errorf("unexpected order type: %s", s) + } +} + +func toGlobalOrderStatus(status v2.OrderStatus) (types.OrderStatus, error) { + switch status { + case v2.OrderStatusInit, v2.OrderStatusNew, v2.OrderStatusLive: + return types.OrderStatusNew, nil + + case v2.OrderStatusPartialFilled: + return types.OrderStatusPartiallyFilled, nil + + case v2.OrderStatusFilled: + return types.OrderStatusFilled, nil + + case v2.OrderStatusCancelled: + return types.OrderStatusCanceled, nil + + default: + return types.OrderStatus(status), fmt.Errorf("unexpected order status: %s", status) + } +} + +// unfilledOrderToGlobalOrder convert the local order to global. +// +// Note that the quantity unit, according official document: Base coin when orderType=limit; Quote coin when orderType=market +// https://bitgetlimited.github.io/apidoc/zh/spot/#19671a1099 +func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { + side, err := toGlobalSideType(order.Side) + if err != nil { + return nil, err + } + + orderType, err := toGlobalOrderType(order.OrderType) + if err != nil { + return nil, err + } + + status, err := toGlobalOrderStatus(order.Status) + if err != nil { + return nil, err + } + + qty := order.Size + price := order.PriceAvg + + // The market order will be executed immediately, so this check is used to handle corner cases. + if orderType == types.OrderTypeMarket { + qty = order.BaseVolume + log.Warnf("!!! The price(%f) and quantity(%f) are not verified for market orders, because we only receive limit orders in the test environment !!!", price.Float64(), qty.Float64()) + } + + return &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: order.ClientOrderId, + Symbol: order.Symbol, + Side: side, + Type: orderType, + Quantity: qty, + Price: price, + // Bitget does not include the "time-in-force" field in its API response for spot trading, so we set GTC. + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(order.OrderId), + UUID: strconv.FormatInt(int64(order.OrderId), 10), + Status: status, + ExecutedQuantity: order.BaseVolume, + IsWorking: order.Status.IsWorking(), + CreationTime: types.Time(order.CTime.Time()), + UpdateTime: types.Time(order.UTime.Time()), + }, nil +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 770e4e5b8c..8b8bc61c7e 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -1,11 +1,13 @@ package bitget import ( + "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -143,3 +145,130 @@ func Test_toGlobalTicker(t *testing.T) { Sell: fixedpoint.NewFromFloat(24014.06), }, toGlobalTicker(ticker)) } + +func Test_toGlobalSideType(t *testing.T) { + side, err := toGlobalSideType(v2.SideTypeBuy) + assert.NoError(t, err) + assert.Equal(t, types.SideTypeBuy, side) + + side, err = toGlobalSideType(v2.SideTypeSell) + assert.NoError(t, err) + assert.Equal(t, types.SideTypeSell, side) + + _, err = toGlobalSideType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalOrderType(t *testing.T) { + orderType, err := toGlobalOrderType(v2.OrderTypeMarket) + assert.NoError(t, err) + assert.Equal(t, types.OrderTypeMarket, orderType) + + orderType, err = toGlobalOrderType(v2.OrderTypeLimit) + assert.NoError(t, err) + assert.Equal(t, types.OrderTypeLimit, orderType) + + _, err = toGlobalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalOrderStatus(t *testing.T) { + status, err := toGlobalOrderStatus(v2.OrderStatusInit) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusNew) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusLive) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusNew, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusFilled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusFilled, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusPartialFilled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusPartiallyFilled, status) + + status, err = toGlobalOrderStatus(v2.OrderStatusCancelled) + assert.NoError(t, err) + assert.Equal(t, types.OrderStatusCanceled, status) + + _, err = toGlobalOrderStatus("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_unfilledOrderToGlobalOrder(t *testing.T) { + var ( + assert = assert.New(t) + orderId = 1105087175647989764 + unfilledOrder = v2.UnfilledOrder{ + Symbol: "BTCUSDT", + OrderId: types.StrInt64(orderId), + ClientOrderId: "74b86af3-6098-479c-acac-bfb074c067f3", + PriceAvg: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeBuy, + Status: v2.OrderStatusLive, + BasePrice: fixedpoint.NewFromFloat(0), + BaseVolume: fixedpoint.NewFromFloat(0), + QuoteVolume: fixedpoint.NewFromFloat(0), + EnterPointSource: "API", + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1660704288118), + UTime: types.NewMillisecondTimestampFromInt(1660704288118), + } + ) + + t.Run("succeeds", func(t *testing.T) { + order, err := unfilledOrderToGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(&types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusNew, + ExecutedQuantity: fixedpoint.NewFromFloat(0), + IsWorking: true, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + }, order) + }) + + t.Run("failed to convert side", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Side = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder type", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.OrderType = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder status", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Status = "xxx" + + _, err := unfilledOrderToGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 700bd2ea7d..e4eef08a7f 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -3,18 +3,24 @@ package bitget import ( "context" "fmt" + "strconv" "time" "github.com/sirupsen/logrus" "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/types" ) -const ID = "bitget" +const ( + ID = "bitget" -const PlatformToken = "BGB" + PlatformToken = "BGB" + + queryOpenOrdersLimit = 100 +) var log = logrus.WithFields(logrus.Fields{ "exchange": ID, @@ -29,12 +35,15 @@ var ( queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryOpenOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Unfilled-Orders + queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { key, secret, passphrase string - client *bitgetapi.RestClient + client *bitgetapi.RestClient + v2Client *v2.Client } func New(key, secret, passphrase string) *Exchange { @@ -49,6 +58,7 @@ func New(key, secret, passphrase string) *Exchange { secret: secret, passphrase: passphrase, client: client, + v2Client: v2.NewClient(client), } } @@ -174,8 +184,40 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr } func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { - // TODO implement me - panic("implement me") + var nextCursor types.StrInt64 + for { + if err := queryOpenOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("open order rate limiter wait error: %w", err) + } + + req := e.v2Client.NewGetUnfilledOrdersRequest(). + Symbol(symbol). + Limit(strconv.FormatInt(queryOpenOrdersLimit, 10)) + if nextCursor != 0 { + req.IdLessThan(strconv.FormatInt(int64(nextCursor), 10)) + } + + openOrders, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query open orders: %w", err) + } + + for _, o := range openOrders { + order, err := unfilledOrderToGlobalOrder(o) + if err != nil { + return nil, fmt.Errorf("failed to convert order, err: %v", err) + } + + orders = append(orders, *order) + } + + if len(openOrders) != queryOpenOrdersLimit { + break + } + nextCursor = openOrders[len(openOrders)-1].OrderId + } + + return orders, nil } func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { From c8becbe4f5f558afbcdbaa9e7896bdcf9e051ab7 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 10:56:19 +0800 Subject: [PATCH 147/422] bbgo.sync when syncActiveOrders --- pkg/strategy/grid2/active_order_recover.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index cfdaeac80b..55b48fcdfd 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -30,7 +30,7 @@ func (s *Strategy) initializeRecoverC() bool { if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.recoverC = make(chan struct{}, 1) + s.recoverC = make(chan struct{}, 10) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true @@ -66,22 +66,26 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { exchange: s.session.Exchange, } + var lastRecoverTime time.Time + for { select { - case <-ctx.Done(): return case <-ticker.C: - if err := syncActiveOrders(ctx, opts); err != nil { - log.WithError(err).Errorf("unable to sync active orders") + s.recoverC <- struct{}{} + bbgo.Sync(ctx, s) + case <-s.recoverC: + if !time.Now().After(lastRecoverTime.Add(10 * time.Minute)) { + continue } - case <-s.recoverC: if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") + } else { + lastRecoverTime = time.Now() } - } } } From 7de49155eb515e5c0a56e1896a88f78ce0a39d30 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 13:30:58 +0800 Subject: [PATCH 148/422] fix --- pkg/strategy/grid2/profit_stats.go | 3 +++ pkg/strategy/grid2/strategy.go | 2 +- pkg/types/position.go | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index c396029370..a770b67fe4 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -44,6 +44,9 @@ func newGridProfitStats(market types.Market) *GridProfitStats { } func (s *GridProfitStats) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } s.ttl = ttl } diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index bc361a72b2..620d91ce1f 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1836,7 +1836,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.ProfitSpread = s.Market.TruncatePrice(s.ProfitSpread) } - s.logger.Infof("ttl: %s", s.PersistenceTTL.Duration()) + s.logger.Infof("persistence ttl: %s", s.PersistenceTTL.Duration()) if s.GridProfitStats == nil { s.GridProfitStats = newGridProfitStats(s.Market) diff --git a/pkg/types/position.go b/pkg/types/position.go index 4b649a48fa..589fa2a2be 100644 --- a/pkg/types/position.go +++ b/pkg/types/position.go @@ -71,6 +71,9 @@ type Position struct { } func (s *Position) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } s.ttl = ttl } From df2fd170db8a5a1298ef719ad5f8a98db3a1906c Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 14:39:29 +0800 Subject: [PATCH 149/422] return bool to let syncActiveOrderBook really sync or skip --- pkg/strategy/grid2/active_order_recover.go | 21 +++++------------ pkg/strategy/grid2/recover.go | 26 ++++++++++------------ pkg/strategy/grid2/recover_test.go | 6 +++-- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 91b52ed68b..16b93e9fda 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,7 +2,6 @@ package grid2 import ( "context" - "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -118,22 +117,12 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { delete(openOrdersMap, activeOrder.OrderID) } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if activeOrder.UpdateTime.After(syncBefore) { - opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) - continue - } - // sleep 100ms to avoid DDOS - time.Sleep(100 * time.Millisecond) - - if err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { - if strings.Contains(err.Error(), "skip syncing active order") { - opts.logger.Infof("[ActiveOrderRecover] skip syncing active order #%d, because the updated_at is in 3 min", activeOrder.OrderID) - } else { - opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) - errs = multierr.Append(errs, err) - } - continue + if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) + errs = multierr.Append(errs, err) + } else if !isActiveOrderBookUpdated { + opts.logger.Infof("[ActiveOrderRecover] active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) } } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 4a6456884b..7859a84af7 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strconv" - "strings" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -141,12 +140,10 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { order := activeOrder.GetOrder() - if err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { - if strings.Contains(err.Error(), "skip syncing active order") { - s.logger.Infof("[Recover] skip handle active order #%d, because the updated_at is in 3 min", order.OrderID) - } else { - s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) - } + if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) + } else if !isActiveOrderBookUpdated { + s.logger.Infof("[Recover] active order #%d is updated in 3 min, skip updating...", order.OrderID) } continue } @@ -262,23 +259,24 @@ func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error return book, nil } -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) error { +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) (bool, error) { updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), }) + isActiveOrderBookUpdated := false + if err != nil { - return err + return isActiveOrderBookUpdated, err } - if updatedOrder.UpdateTime.After(syncBefore) { - return fmt.Errorf("skip syncing active order, because its updated_at is after %s", syncBefore) + isActiveOrderBookUpdated = updatedOrder.UpdateTime.Before(syncBefore) + if isActiveOrderBookUpdated { + activeOrderBook.Update(*updatedOrder) } - activeOrderBook.Update(*updatedOrder) - - return nil + return isActiveOrderBookUpdated, nil } func queryTradesToUpdateTwinOrderBook( diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go index 332c84d624..fbd6ecb7c0 100644 --- a/pkg/strategy/grid2/recover_test.go +++ b/pkg/strategy/grid2/recover_test.go @@ -114,7 +114,8 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { + _, err := syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now()) + if !assert.NoError(err) { return } @@ -144,7 +145,8 @@ func TestSyncActiveOrder(t *testing.T) { OrderID: strconv.FormatUint(order.OrderID, 10), }).Return(&updatedOrder, nil) - if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now())) { + _, err := syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID, time.Now()) + if !assert.NoError(err) { return } From 2049e71cf6bfd382c7dc3118a91bd568d6463e09 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 7 Nov 2023 14:53:00 +0800 Subject: [PATCH 150/422] pkg/exchange: rm the retry --- pkg/exchange/bybit/market_info_poller.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go index dac0172f9f..6266c8d292 100644 --- a/pkg/exchange/bybit/market_info_poller.go +++ b/pkg/exchange/bybit/market_info_poller.go @@ -46,14 +46,13 @@ func (p *feeRatePoller) Start(ctx context.Context) { } func (p *feeRatePoller) startLoop(ctx context.Context) { + err := p.poll(ctx) + if err != nil { + log.WithError(err).Warn("failed to initialize the fee rate, the ticker is scheduled to update it subsequently") + } + ticker := time.NewTicker(feeRatePollingPeriod) defer ticker.Stop() - - // Make sure the first poll should succeed by retrying with a shorter period. - _ = util.Retry(ctx, util.InfiniteRetry, 30*time.Second, - func() error { return p.poll(ctx) }, - func(e error) { log.WithError(e).Warn("failed to update fee rate") }) - for { select { case <-ctx.Done(): From e6fc0067477d61a56da611e684547f2a2242e013 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 15:21:48 +0800 Subject: [PATCH 151/422] recoverC back to size 1 --- pkg/strategy/grid2/active_order_recover.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 55b48fcdfd..5c64488b4d 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -30,7 +30,7 @@ func (s *Strategy) initializeRecoverC() bool { if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.recoverC = make(chan struct{}, 10) + s.recoverC = make(chan struct{}, 1) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true From 4a40c8bea28cc63a1c70cdddf9a68dc495e2c1bd Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 7 Nov 2023 17:00:29 +0800 Subject: [PATCH 152/422] refactor --- pkg/strategy/grid2/active_order_recover.go | 10 +++++++--- pkg/strategy/grid2/recover.go | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 16b93e9fda..87db11bf9c 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -116,12 +116,16 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // no need to sync active order already in active orderbook, because we only need to know if it filled or not. delete(openOrdersMap, activeOrder.OrderID) } else { - opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + opts.logger.Infof("[ActiveOrderRecover] found active order #%d is not in the open orders, updating...", activeOrder.OrderID) - if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore); err != nil { + isActiveOrderBookUpdated, err := syncActiveOrder(ctx, opts.activeOrderBook, opts.orderQueryService, activeOrder.OrderID, syncBefore) + if err != nil { opts.logger.WithError(err).Errorf("[ActiveOrderRecover] unable to query order #%d", activeOrder.OrderID) errs = multierr.Append(errs, err) - } else if !isActiveOrderBookUpdated { + continue + } + + if !isActiveOrderBookUpdated { opts.logger.Infof("[ActiveOrderRecover] active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) } } diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 7859a84af7..9418dd5a8a 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -98,8 +98,8 @@ func (s *Strategy) recover(ctx context.Context) error { activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) - s.logger.Infof("active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) - s.logger.Infof("open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) + s.logger.Infof("[Recover] active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) + s.logger.Infof("[Recover] open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) // remove index 0, because twin orderbook's price is from the second one pins = pins[1:] @@ -131,7 +131,9 @@ func (s *Strategy) recover(ctx context.Context) error { // case 1 if activeOrderID == 0 { - activeOrderBook.Add(openOrder.GetOrder()) + order := openOrder.GetOrder() + s.logger.Infof("[Recover] found open order #%d is not in the active orderbook, adding...", order.OrderID) + activeOrderBook.Add(order) // also add open orders into active order's twin orderbook, we will use this active orderbook to recover empty price grid activeOrdersInTwinOrderBook.AddTwinOrder(v, openOrder) continue @@ -140,11 +142,17 @@ func (s *Strategy) recover(ctx context.Context) error { // case 2 if openOrderID == 0 { order := activeOrder.GetOrder() - if isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore); err != nil { + s.logger.Infof("[Recover] found active order #%d is not in the open orders, updating...", order.OrderID) + isActiveOrderBookUpdated, err := syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, order.OrderID, syncBefore) + if err != nil { s.logger.WithError(err).Errorf("[Recover] unable to query order #%d", order.OrderID) - } else if !isActiveOrderBookUpdated { + continue + } + + if !isActiveOrderBookUpdated { s.logger.Infof("[Recover] active order #%d is updated in 3 min, skip updating...", order.OrderID) } + continue } From b41f4712d79c93ad9f1808092427a73943a79dff Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 7 Nov 2023 17:17:38 +0800 Subject: [PATCH 153/422] pkg/exchange: add fee recover --- pkg/exchange/bybit/market_info_poller.go | 10 ++--- pkg/exchange/bybit/market_info_poller_test.go | 12 ++--- pkg/exchange/bybit/stream.go | 44 ++++++++++++++++--- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go index 6266c8d292..95664718f1 100644 --- a/pkg/exchange/bybit/market_info_poller.go +++ b/pkg/exchange/bybit/market_info_poller.go @@ -8,7 +8,6 @@ import ( "time" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" - "github.com/c9s/bbgo/pkg/util" ) const ( @@ -82,15 +81,12 @@ func (p *feeRatePoller) poll(ctx context.Context) error { return nil } -func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, error) { +func (p *feeRatePoller) Get(symbol string) (symbolFeeDetail, bool) { p.mu.Lock() defer p.mu.Unlock() - fee, ok := p.symbolFeeDetail[symbol] - if !ok { - return symbolFeeDetail{}, fmt.Errorf("%s fee rate not found", symbol) - } - return fee, nil + fee, found := p.symbolFeeDetail[symbol] + return fee, found } func (e *feeRatePoller) getAllFeeRates(ctx context.Context) (map[string]symbolFeeDetail, error) { diff --git a/pkg/exchange/bybit/market_info_poller_test.go b/pkg/exchange/bybit/market_info_poller_test.go index f2b58c466b..cf8dd6fdb9 100644 --- a/pkg/exchange/bybit/market_info_poller_test.go +++ b/pkg/exchange/bybit/market_info_poller_test.go @@ -137,7 +137,7 @@ func Test_feeRatePoller_Get(t *testing.T) { defer mockCtrl.Finish() mockMarketProvider := mocks.NewMockStreamDataProvider(mockCtrl) - t.Run("succeeds", func(t *testing.T) { + t.Run("found", func(t *testing.T) { symbol := "BTCUSDT" expFeeDetail := symbolFeeDetail{ FeeRate: bybitapi.FeeRate{ @@ -156,18 +156,18 @@ func Test_feeRatePoller_Get(t *testing.T) { }, } - res, err := s.Get(symbol) - assert.NoError(t, err) + res, found := s.Get(symbol) + assert.True(t, found) assert.Equal(t, expFeeDetail, res) }) - t.Run("succeeds", func(t *testing.T) { + t.Run("not found", func(t *testing.T) { symbol := "BTCUSDT" s := &feeRatePoller{ client: mockMarketProvider, symbolFeeDetail: map[string]symbolFeeDetail{}, } - _, err := s.Get(symbol) - assert.ErrorContains(t, err, symbol) + _, found := s.Get(symbol) + assert.False(t, found) }) } diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 2bb2626575..8911125673 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -10,6 +10,7 @@ import ( "github.com/gorilla/websocket" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -22,6 +23,11 @@ const ( var ( // wsAuthRequest specifies the duration for which a websocket request's authentication is valid. wsAuthRequest = 10 * time.Second + // The default taker/maker fees can help us in estimating trading fees in the SPOT market, because trade fees are not + // provided for traditional accounts on Bybit. + // https://www.bybit.com/en-US/help-center/article/Trading-Fee-Structure + defaultTakerFee = fixedpoint.NewFromFloat(0.001) + defaultMakerFee = fixedpoint.NewFromFloat(0.001) ) // MarketInfoProvider calculates trade fees since trading fees are not supported by streaming. @@ -48,6 +54,7 @@ type Stream struct { key, secret string streamDataProvider StreamDataProvider feeRateProvider *feeRatePoller + marketsInfo types.MarketMap bookEventCallbacks []func(e BookEvent) marketTradeEventCallbacks []func(e []MarketTradeEvent) @@ -71,8 +78,14 @@ func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream stream.SetParser(stream.parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) stream.SetHeartBeat(stream.ping) - stream.SetBeforeConnect(func(ctx context.Context) error { + stream.SetBeforeConnect(func(ctx context.Context) (err error) { go stream.feeRateProvider.Start(ctx) + + stream.marketsInfo, err = stream.streamDataProvider.QueryMarkets(ctx) + if err != nil { + log.WithError(err).Error("failed to query market info before to connect stream") + return err + } return nil }) stream.OnConnect(stream.handlerConnect) @@ -406,10 +419,31 @@ func (s *Stream) handleKLineEvent(klineEvent KLineEvent) { func (s *Stream) handleTradeEvent(events []TradeEvent) { for _, event := range events { - feeRate, err := s.feeRateProvider.Get(event.Symbol) - if err != nil { - log.Warnf("failed to get fee rate by symbol: %s", event.Symbol) - continue + feeRate, found := s.feeRateProvider.Get(event.Symbol) + if !found { + feeRate = symbolFeeDetail{ + FeeRate: bybitapi.FeeRate{ + Symbol: event.Symbol, + TakerFeeRate: defaultTakerFee, + MakerFeeRate: defaultMakerFee, + }, + BaseCoin: "", + QuoteCoin: "", + } + + if market, ok := s.marketsInfo[event.Symbol]; ok { + feeRate.BaseCoin = market.BaseCurrency + feeRate.QuoteCoin = market.QuoteCurrency + } + + // The error log level was utilized due to a detected discrepancy in the fee calculations. + log.Errorf("failed to get %s fee rate, use default taker fee %f, maker fee %f, base coin: %s, quote coin: %s", + event.Symbol, + feeRate.TakerFeeRate.Float64(), + feeRate.MakerFeeRate.Float64(), + feeRate.BaseCoin, + feeRate.QuoteCoin, + ) } gTrade, err := event.toGlobalTrade(feeRate) From 52d4f50c883fb945a1968a9d374895ce371f28c8 Mon Sep 17 00:00:00 2001 From: chiahung Date: Wed, 8 Nov 2023 11:15:06 +0800 Subject: [PATCH 154/422] remove sync every ticker --- pkg/strategy/grid2/active_order_recover.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 5c64488b4d..c25d45bc40 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -75,7 +75,6 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { case <-ticker.C: s.recoverC <- struct{}{} - bbgo.Sync(ctx, s) case <-s.recoverC: if !time.Now().After(lastRecoverTime.Add(10 * time.Minute)) { continue From 2d650cd1d9766f271ec130e4872c8a5e2e091359 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:08:21 +0800 Subject: [PATCH 155/422] pkg/exchange: add defensive program to ensure the order length is expected --- pkg/exchange/bitget/exchange.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index e4eef08a7f..542a3bfc26 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -211,10 +211,16 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ orders = append(orders, *order) } - if len(openOrders) != queryOpenOrdersLimit { + orderLen := len(openOrders) + // a defensive programming to ensure the length of order response is expected. + if orderLen > queryOpenOrdersLimit { + return nil, fmt.Errorf("unexpected open orders length %d", orderLen) + } + + if orderLen < queryOpenOrdersLimit { break } - nextCursor = openOrders[len(openOrders)-1].OrderId + nextCursor = openOrders[orderLen-1].OrderId } return orders, nil From 6531dbcf1ae7e95f47a2d6291eb4b2ad0a1e17b3 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:18:28 +0800 Subject: [PATCH 156/422] go: update requestgen to v1.3.6 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 91eca5f8be..57f0bf0994 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/squirrel v1.5.3 github.com/adshao/go-binance/v2 v2.4.2 github.com/c-bata/goptuna v0.8.1 - github.com/c9s/requestgen v1.3.5 + github.com/c9s/requestgen v1.3.6 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b github.com/cenkalti/backoff/v4 v4.2.0 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index 710eba662d..56327767ef 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY= github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4= github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs= github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= +github.com/c9s/requestgen v1.3.6 h1:ul7dZ2uwGYjNBjreooRfSY10WTXvQmQSjZsHebz6QfE= +github.com/c9s/requestgen v1.3.6/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b/go.mod h1:EKObf66Cp7erWxym2de+07qNN5T1N9PXxHdh97N44EQ= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= From 2c072281d781a1b7819ee279eb746cf7f22ed08e Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 8 Nov 2023 22:43:01 +0800 Subject: [PATCH 157/422] pkg/exchange: add assertion for api response --- .../cancel_order_request_requestgen.go | 20 +++++++++++++++++- pkg/exchange/bybit/bybitapi/client.go | 21 ++++++++++++++++--- .../get_account_info_request_requestgen.go | 20 +++++++++++++++++- .../get_fee_rates_request_requestgen.go | 20 +++++++++++++++++- ...get_instruments_info_request_requestgen.go | 20 +++++++++++++++++- .../get_k_lines_request_requestgen.go | 20 +++++++++++++++++- .../get_open_orders_request_requestgen.go | 20 +++++++++++++++++- ...uest.go => get_order_histories_request.go} | 0 .../get_order_histories_request_requestgen.go | 20 +++++++++++++++++- .../get_tickers_request_requestgen.go | 20 +++++++++++++++++- .../get_wallet_balances_request_requestgen.go | 20 +++++++++++++++++- .../place_order_request_requestgen.go | 20 +++++++++++++++++- .../v3/get_trades_request_requestgen.go | 20 +++++++++++++++++- 13 files changed, 227 insertions(+), 14 deletions(-) rename pkg/exchange/bybit/bybitapi/{get_order_history_request.go => get_order_histories_request.go} (100%) diff --git a/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go b/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go index ad168ded99..715bf42148 100644 --- a/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/cancel_order_request_requestgen.go @@ -187,6 +187,12 @@ func (p *CancelOrderRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (p *CancelOrderRequest) GetPath() string { + return "/v5/order/cancel" +} + +// Do generates the request object and send the request object to the API endpoint func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, error) { params, err := p.GetParameters() @@ -195,7 +201,9 @@ func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, erro } query := url.Values{} - apiURL := "/v5/order/cancel" + var apiURL string + + apiURL = p.GetPath() req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) if err != nil { @@ -211,6 +219,16 @@ func (p *CancelOrderRequest) Do(ctx context.Context) (*CancelOrderResponse, erro if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data CancelOrderResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/client.go b/pkg/exchange/bybit/bybitapi/client.go index 034d297500..4d9f5818b1 100644 --- a/pkg/exchange/bybit/bybitapi/client.go +++ b/pkg/exchange/bybit/bybitapi/client.go @@ -162,10 +162,25 @@ sample: */ type APIResponse struct { - RetCode uint `json:"retCode"` - RetMsg string `json:"retMsg"` - Result json.RawMessage `json:"result"` + // Success/Error code + RetCode uint `json:"retCode"` + // Success/Error msg. OK, success, SUCCESS indicate a successful response + RetMsg string `json:"retMsg"` + // Business data result + Result json.RawMessage `json:"result"` + // Extend info. Most of the time, it is {} RetExtInfo json.RawMessage `json:"retExtInfo"` // Time is current timestamp (ms) Time types.MillisecondTimestamp `json:"time"` } + +func (a APIResponse) Validate() error { + if a.RetCode != 0 { + return a.Error() + } + return nil +} + +func (a APIResponse) Error() error { + return fmt.Errorf("retCode: %d, retMsg: %s, retExtInfo: %q, time: %s", a.RetCode, a.RetMsg, a.RetExtInfo, a.Time) +} diff --git a/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go index 8d9be7df7d..c7907f404b 100644 --- a/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_account_info_request_requestgen.go @@ -109,13 +109,21 @@ func (g *GetAccountInfoRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetAccountInfoRequest) GetPath() string { + return "/v5/account/info" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetAccountInfoRequest) Do(ctx context.Context) (*AccountInfo, error) { // no body params var params interface{} query := url.Values{} - apiURL := "/v5/account/info" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -131,6 +139,16 @@ func (g *GetAccountInfoRequest) Do(ctx context.Context) (*AccountInfo, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data AccountInfo if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go index ac182c22ac..e9db73677c 100644 --- a/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_fee_rates_request_requestgen.go @@ -156,6 +156,12 @@ func (g *GetFeeRatesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetFeeRatesRequest) GetPath() string { + return "/v5/account/fee-rate" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { // no body params @@ -165,7 +171,9 @@ func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { return nil, err } - apiURL := "/v5/account/fee-rate" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -181,6 +189,16 @@ func (g *GetFeeRatesRequest) Do(ctx context.Context) (*FeeRates, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data FeeRates if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go index 7e9a6536ac..ed2fc884f5 100644 --- a/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_instruments_info_request_requestgen.go @@ -169,6 +169,12 @@ func (g *GetInstrumentsInfoRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetInstrumentsInfoRequest) GetPath() string { + return "/v5/market/instruments-info" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, error) { // no body params @@ -178,7 +184,9 @@ func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, e return nil, err } - apiURL := "/v5/market/instruments-info" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -194,6 +202,16 @@ func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) (*InstrumentsInfo, e if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data InstrumentsInfo if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go index d6b3d06c3f..e08072a7ba 100644 --- a/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_k_lines_request_requestgen.go @@ -204,6 +204,12 @@ func (g *GetKLinesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetKLinesRequest) GetPath() string { + return "/v5/market/kline" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { // no body params @@ -213,7 +219,9 @@ func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { return nil, err } - apiURL := "/v5/market/kline" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -229,6 +237,16 @@ func (g *GetKLinesRequest) Do(ctx context.Context) (*KLinesResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data KLinesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go index aa9d13cc63..3b7c29ba47 100644 --- a/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_open_orders_request_requestgen.go @@ -258,6 +258,12 @@ func (g *GetOpenOrdersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetOpenOrdersRequest) GetPath() string { + return "/v5/order/realtime" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) { // no body params @@ -267,7 +273,9 @@ func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) return nil, err } - apiURL := "/v5/order/realtime" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -283,6 +291,16 @@ func (g *GetOpenOrdersRequest) Do(ctx context.Context) (*OrdersResponse, error) if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data OrdersResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_order_history_request.go b/pkg/exchange/bybit/bybitapi/get_order_histories_request.go similarity index 100% rename from pkg/exchange/bybit/bybitapi/get_order_history_request.go rename to pkg/exchange/bybit/bybitapi/get_order_histories_request.go diff --git a/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go index 479d9eff40..62b3624106 100644 --- a/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_order_histories_request_requestgen.go @@ -262,6 +262,12 @@ func (g *GetOrderHistoriesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetOrderHistoriesRequest) GetPath() string { + return "/v5/order/history" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, error) { // no body params @@ -271,7 +277,9 @@ func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, err return nil, err } - apiURL := "/v5/order/history" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -287,6 +295,16 @@ func (g *GetOrderHistoriesRequest) Do(ctx context.Context) (*OrdersResponse, err if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data OrdersResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go index 0f85973d8d..59a81a4282 100644 --- a/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_tickers_request_requestgen.go @@ -143,6 +143,12 @@ func (g *GetTickersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetTickersRequest) GetPath() string { + return "/v5/market/tickers" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { // no body params @@ -152,7 +158,9 @@ func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { return nil, err } - apiURL := "/v5/market/tickers" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -168,5 +176,15 @@ func (g *GetTickersRequest) Do(ctx context.Context) (*APIResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } return &apiResponse, nil } diff --git a/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go b/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go index 14ae4029cb..601d26ce9d 100644 --- a/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/get_wallet_balances_request_requestgen.go @@ -143,6 +143,12 @@ func (g *GetWalletBalancesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetWalletBalancesRequest) GetPath() string { + return "/v5/account/wallet-balance" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesResponse, error) { // no body params @@ -152,7 +158,9 @@ func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesRespo return nil, err } - apiURL := "/v5/account/wallet-balance" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -168,6 +176,16 @@ func (g *GetWalletBalancesRequest) Do(ctx context.Context) (*WalletBalancesRespo if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data WalletBalancesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go b/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go index bb85937cec..95e3aaa0df 100644 --- a/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/place_order_request_requestgen.go @@ -496,6 +496,12 @@ func (p *PlaceOrderRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (p *PlaceOrderRequest) GetPath() string { + return "/v5/order/create" +} + +// Do generates the request object and send the request object to the API endpoint func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) { params, err := p.GetParameters() @@ -504,7 +510,9 @@ func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) } query := url.Values{} - apiURL := "/v5/order/create" + var apiURL string + + apiURL = p.GetPath() req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) if err != nil { @@ -520,6 +528,16 @@ func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data PlaceOrderResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err diff --git a/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go index 00d7dd80a9..3336db113a 100644 --- a/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go +++ b/pkg/exchange/bybit/bybitapi/v3/get_trades_request_requestgen.go @@ -205,6 +205,12 @@ func (g *GetTradesRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetTradesRequest) GetPath() string { + return "/spot/v3/private/my-trades" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { // no body params @@ -214,7 +220,9 @@ func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { return nil, err } - apiURL := "/spot/v3/private/my-trades" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -230,6 +238,16 @@ func (g *GetTradesRequest) Do(ctx context.Context) (*TradesResponse, error) { if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data TradesResponse if err := json.Unmarshal(apiResponse.Result, &data); err != nil { return nil, err From 3978fca27df6b9e524092e93568c8c65a22f8b87 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 11:51:16 +0800 Subject: [PATCH 158/422] pkg/exchange: support query closed orders --- .../bitget/bitgetapi/v2/client_test.go | 12 +- .../v2/get_history_orders_request.go | 102 ++++++++ .../get_history_orders_request_requestgen.go | 222 ++++++++++++++++++ .../v2/get_history_orders_request_test.go | 121 ++++++++++ pkg/exchange/bitget/convert.go | 85 +++++++ pkg/exchange/bitget/convert_test.go | 193 +++++++++++++++ pkg/exchange/bitget/exchange.go | 64 ++++- 7 files changed, 792 insertions(+), 7 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 21c86dfc6b..3d56735e9e 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -2,12 +2,11 @@ package bitgetapi import ( "context" + "github.com/stretchr/testify/assert" "os" "strconv" "testing" - "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/testutil" ) @@ -25,7 +24,6 @@ func getTestClientOrSkip(t *testing.T) *Client { client := bitgetapi.NewClient() client.Auth(key, secret, os.Getenv("BITGET_API_PASSPHRASE")) - return NewClient(client) } @@ -39,4 +37,12 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("resp: %+v", resp) }) + + t.Run("GetHistoryOrdersRequest", func(t *testing.T) { + // market buy + req, err := client.NewGetHistoryOrdersRequest().Symbol("APEUSDT").Do(ctx) + assert.NoError(t, err) + + t.Logf("place order resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go new file mode 100644 index 0000000000..51807eb5ed --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go @@ -0,0 +1,102 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "encoding/json" + "fmt" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +type FeeDetail struct { + // NewFees should have a value because when I was integrating, it already prompted, + // "If there is no 'newFees' field, this data represents earlier historical data." + NewFees struct { + // Amount deducted by coupons, unit：currency obtained from the transaction. + DeductedByCoupon fixedpoint.Value `json:"c"` + // Amount deducted in BGB (Bitget Coin), unit：BGB + DeductedInBGB fixedpoint.Value `json:"d"` + // If the BGB balance is insufficient to cover the fees, the remaining amount is deducted from the + //currency obtained from the transaction. + DeductedFromCurrency fixedpoint.Value `json:"r"` + // The total fee amount to be paid, unit ：currency obtained from the transaction. + ToBePaid fixedpoint.Value `json:"t"` + // ignored + Deduction bool `json:"deduction"` + // ignored + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + } `json:"newFees"` +} + +type OrderDetail struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` + Price fixedpoint.Value `json:"price"` + // Size is base coin when orderType=limit; quote coin when orderType=market + Size fixedpoint.Value `json:"size"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + Status OrderStatus `json:"status"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + BaseVolume fixedpoint.Value `json:"baseVolume"` + QuoteVolume fixedpoint.Value `json:"quoteVolume"` + EnterPointSource string `json:"enterPointSource"` + // The value is json string, so we unmarshal it after unmarshal OrderDetail + FeeDetailRaw string `json:"feeDetail"` + OrderSource string `json:"orderSource"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` + + FeeDetail FeeDetail +} + +func (o *OrderDetail) UnmarshalJSON(data []byte) error { + if o == nil { + return fmt.Errorf("failed to unmarshal json from nil pointer order detail") + } + // define new type to avoid loop reference + type AuxOrderDetail OrderDetail + + var aux AuxOrderDetail + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + *o = OrderDetail(aux) + + if len(aux.FeeDetailRaw) == 0 { + return nil + } + + var feeDetail FeeDetail + if err := json.Unmarshal([]byte(aux.FeeDetailRaw), &feeDetail); err != nil { + return fmt.Errorf("unexpected fee detail raw: %s, err: %w", aux.FeeDetailRaw, err) + } + o.FeeDetail = feeDetail + + return nil +} + +//go:generate GetRequest -url "/api/v2/spot/trade/history-orders" -type GetHistoryOrdersRequest -responseDataType []OrderDetail +type GetHistoryOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol *string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (c *Client) NewGetHistoryOrdersRequest() *GetHistoryOrdersRequest { + return &GetHistoryOrdersRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go new file mode 100644 index 0000000000..0a681f3228 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go @@ -0,0 +1,222 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/history-orders -type GetHistoryOrdersRequest -responseDataType []OrderDetail"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (g *GetHistoryOrdersRequest) Symbol(symbol string) *GetHistoryOrdersRequest { + g.symbol = &symbol + return g +} + +func (g *GetHistoryOrdersRequest) Limit(limit string) *GetHistoryOrdersRequest { + g.limit = &limit + return g +} + +func (g *GetHistoryOrdersRequest) IdLessThan(idLessThan string) *GetHistoryOrdersRequest { + g.idLessThan = &idLessThan + return g +} + +func (g *GetHistoryOrdersRequest) StartTime(startTime int64) *GetHistoryOrdersRequest { + g.startTime = &startTime + return g +} + +func (g *GetHistoryOrdersRequest) EndTime(endTime int64) *GetHistoryOrdersRequest { + g.endTime = &endTime + return g +} + +func (g *GetHistoryOrdersRequest) OrderId(orderId string) *GetHistoryOrdersRequest { + g.orderId = &orderId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetHistoryOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if g.idLessThan != nil { + idLessThan := *g.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if g.orderId != nil { + orderId := *g.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetHistoryOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetHistoryOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetHistoryOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetHistoryOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetHistoryOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetHistoryOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetHistoryOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetHistoryOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/history-orders" + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + var data []OrderDetail + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go new file mode 100644 index 0000000000..07e319faba --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go @@ -0,0 +1,121 @@ +package bitgetapi + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestOrderDetail_UnmarshalJSON(t *testing.T) { + var ( + assert = assert.New(t) + ) + t.Run("empty fee", func(t *testing.T) { + input := `{ + "userId":"8672173294", + "symbol":"APEUSDT", + "orderId":"1104342023170068480", + "clientOid":"f3d6a1ee-4e94-48b5-a6e0-25f3e93d92e1", + "price":"1.2000000000000000", + "size":"5.0000000000000000", + "orderType":"limit", + "side":"buy", + "status":"cancelled", + "priceAvg":"0", + "baseVolume":"0.0000000000000000", + "quoteVolume":"0.0000000000000000", + "enterPointSource":"API", + "feeDetail":"", + "orderSource":"normal", + "cTime":"1699021576683", + "uTime":"1699021649099" + }` + var od OrderDetail + err := json.Unmarshal([]byte(input), &od) + assert.NoError(err) + assert.Equal(OrderDetail{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104342023170068480), + ClientOrderId: "f3d6a1ee-4e94-48b5-a6e0-25f3e93d92e1", + Price: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: OrderTypeLimit, + Side: SideTypeBuy, + Status: OrderStatusCancelled, + PriceAvg: fixedpoint.Zero, + BaseVolume: fixedpoint.Zero, + QuoteVolume: fixedpoint.Zero, + EnterPointSource: "API", + FeeDetailRaw: "", + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1699021576683), + UTime: types.NewMillisecondTimestampFromInt(1699021649099), + FeeDetail: FeeDetail{}, + }, od) + }) + + t.Run("fee", func(t *testing.T) { + input := `{ + "userId":"8672173294", + "symbol":"APEUSDT", + "orderId":"1104337778433757184", + "clientOid":"8afea7bd-d873-44fe-aff8-6a1fae3cc765", + "price":"1.4000000000000000", + "size":"5.0000000000000000", + "orderType":"limit", + "side":"sell", + "status":"filled", + "priceAvg":"1.4001000000000000", + "baseVolume":"5.0000000000000000", + "quoteVolume":"7.0005000000000000", + "enterPointSource":"API", + "feeDetail":"{\"newFees\":{\"c\":0,\"d\":0,\"deduction\":false,\"r\":-0.0070005,\"t\":-0.0070005,\"totalDeductionFee\":0},\"USDT\":{\"deduction\":false,\"feeCoinCode\":\"USDT\",\"totalDeductionFee\":0,\"totalFee\":-0.007000500000}}", + "orderSource":"normal", + "cTime":"1699020564659", + "uTime":"1699020564688" + }` + var od OrderDetail + err := json.Unmarshal([]byte(input), &od) + assert.NoError(err) + assert.Equal(OrderDetail{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104337778433757184), + ClientOrderId: "8afea7bd-d873-44fe-aff8-6a1fae3cc765", + Price: fixedpoint.NewFromFloat(1.4), + Size: fixedpoint.NewFromFloat(5), + OrderType: OrderTypeLimit, + Side: SideTypeSell, + Status: OrderStatusFilled, + PriceAvg: fixedpoint.NewFromFloat(1.4001), + BaseVolume: fixedpoint.NewFromFloat(5), + QuoteVolume: fixedpoint.NewFromFloat(7.0005), + EnterPointSource: "API", + FeeDetailRaw: `{"newFees":{"c":0,"d":0,"deduction":false,"r":-0.0070005,"t":-0.0070005,"totalDeductionFee":0},"USDT":{"deduction":false,"feeCoinCode":"USDT","totalDeductionFee":0,"totalFee":-0.007000500000}}`, + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1699020564659), + UTime: types.NewMillisecondTimestampFromInt(1699020564688), + FeeDetail: FeeDetail{ + NewFees: struct { + DeductedByCoupon fixedpoint.Value `json:"c"` + DeductedInBGB fixedpoint.Value `json:"d"` + DeductedFromCurrency fixedpoint.Value `json:"r"` + ToBePaid fixedpoint.Value `json:"t"` + Deduction bool `json:"deduction"` + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + }{DeductedByCoupon: fixedpoint.NewFromFloat(0), + DeductedInBGB: fixedpoint.NewFromFloat(0), + DeductedFromCurrency: fixedpoint.NewFromFloat(-0.0070005), + ToBePaid: fixedpoint.NewFromFloat(-0.0070005), + Deduction: false, + TotalDeductionFee: fixedpoint.Zero, + }, + }, + }, od) + }) +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a4a9756d05..24785a81f1 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,6 +1,7 @@ package bitget import ( + "errors" "fmt" "math" "strconv" @@ -158,3 +159,87 @@ func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { UpdateTime: types.Time(order.UTime.Time()), }, nil } + +func toGlobalOrder(order v2.OrderDetail) (*types.Order, error) { + side, err := toGlobalSideType(order.Side) + if err != nil { + return nil, err + } + + orderType, err := toGlobalOrderType(order.OrderType) + if err != nil { + return nil, err + } + + status, err := toGlobalOrderStatus(order.Status) + if err != nil { + return nil, err + } + + qty := order.Size + price := order.Price + + if orderType == types.OrderTypeMarket { + price = order.PriceAvg + if side == types.SideTypeBuy { + qty, err = processMarketBuyQuantity(order.BaseVolume, order.QuoteVolume, order.PriceAvg, order.Size, order.Status) + if err != nil { + return nil, err + } + } + } + + return &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: order.ClientOrderId, + Symbol: order.Symbol, + Side: side, + Type: orderType, + Quantity: qty, + Price: price, + // Bitget does not include the "time-in-force" field in its API response for spot trading, so we set GTC. + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(order.OrderId), + UUID: strconv.FormatInt(int64(order.OrderId), 10), + Status: status, + ExecutedQuantity: order.BaseVolume, + IsWorking: order.Status.IsWorking(), + CreationTime: types.Time(order.CTime.Time()), + UpdateTime: types.Time(order.UTime.Time()), + }, nil +} + +// processMarketBuyQuantity returns the estimated base quantity or real. The order size will be 'quote quantity' when side is buy and +// type is market, so we need to convert that. This is because the unit of types.Order.Quantity is base coin. +// +// If the order status is PartialFilled, return estimated base coin quantity. +// If the order status is Filled, return the filled base quantity instead of the buy quantity, because a market order on the buy side +// cannot execute all. +// Otherwise, return zero. +func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoint.Value, orderStatus v2.OrderStatus) (fixedpoint.Value, error) { + switch orderStatus { + case v2.OrderStatusInit, v2.OrderStatusNew, v2.OrderStatusLive, v2.OrderStatusCancelled: + return fixedpoint.Zero, nil + + case v2.OrderStatusPartialFilled: + // sanity check for avoid divide 0 + if priceAvg.IsZero() { + return fixedpoint.Zero, errors.New("priceAvg for a partialFilled should not be zero") + } + // calculate the remaining quote coin quantity. + remainPrice := buyQty.Sub(filledPrice) + // calculate the remaining base coin quantity. + remainBaseCoinQty := remainPrice.Div(priceAvg) + // Estimated quantity that may be purchased. + return filledQty.Add(remainBaseCoinQty), nil + + case v2.OrderStatusFilled: + // Market buy orders may not purchase the entire quantity, hence the use of filledQty here. + return filledQty, nil + + default: + return fixedpoint.Zero, fmt.Errorf("failed to execute market buy quantity due to unexpected order status %s ", orderStatus) + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 8b8bc61c7e..1747980748 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -272,3 +272,196 @@ func Test_unfilledOrderToGlobalOrder(t *testing.T) { assert.ErrorContains(err, "xxx") }) } + +func Test_toGlobalOrder(t *testing.T) { + var ( + assert = assert.New(t) + orderId = 1105087175647989764 + unfilledOrder = v2.OrderDetail{ + UserId: 123456, + Symbol: "BTCUSDT", + OrderId: types.StrInt64(orderId), + ClientOrderId: "74b86af3-6098-479c-acac-bfb074c067f3", + Price: fixedpoint.NewFromFloat(1.2), + Size: fixedpoint.NewFromFloat(5), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeBuy, + Status: v2.OrderStatusFilled, + PriceAvg: fixedpoint.NewFromFloat(1.4), + BaseVolume: fixedpoint.NewFromFloat(5), + QuoteVolume: fixedpoint.NewFromFloat(7.0005), + EnterPointSource: "API", + FeeDetailRaw: `{\"newFees\":{\"c\":0,\"d\":0,\"deduction\":false,\"r\":-0.0070005,\"t\":-0.0070005,\"totalDeductionFee\":0},\"USDT\":{\"deduction\":false,\"feeCoinCode\":\"USDT\",\"totalDeductionFee\":0,\"totalFee\":-0.007000500000}}`, + OrderSource: "normal", + CTime: types.NewMillisecondTimestampFromInt(1660704288118), + UTime: types.NewMillisecondTimestampFromInt(1660704288118), + } + + expOrder = &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusFilled, + ExecutedQuantity: fixedpoint.NewFromFloat(5), + IsWorking: false, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + } + ) + + t.Run("succeeds with limit buy", func(t *testing.T) { + order, err := toGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(expOrder, order) + }) + + t.Run("succeeds with limit sell", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeSell + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeSell + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with market sell", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeSell + newUnfilledOrder.OrderType = v2.OrderTypeMarket + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeSell + newExpOrder.Type = types.OrderTypeMarket + newExpOrder.Price = newUnfilledOrder.PriceAvg + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with market buy", func(t *testing.T) { + newUnfilledOrder := unfilledOrder + newUnfilledOrder.Side = v2.SideTypeBuy + newUnfilledOrder.OrderType = v2.OrderTypeMarket + + newExpOrder := *expOrder + newExpOrder.Side = types.SideTypeBuy + newExpOrder.Type = types.OrderTypeMarket + newExpOrder.Price = newUnfilledOrder.PriceAvg + newExpOrder.Quantity = newUnfilledOrder.BaseVolume + + order, err := toGlobalOrder(newUnfilledOrder) + assert.NoError(err) + assert.Equal(&newExpOrder, order) + }) + + t.Run("succeeds with limit buy", func(t *testing.T) { + order, err := toGlobalOrder(unfilledOrder) + assert.NoError(err) + assert.Equal(&types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "74b86af3-6098-479c-acac-bfb074c067f3", + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(5), + Price: fixedpoint.NewFromFloat(1.2), + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(orderId), + UUID: strconv.FormatInt(int64(orderId), 10), + Status: types.OrderStatusFilled, + ExecutedQuantity: fixedpoint.NewFromFloat(5), + IsWorking: false, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1660704288118).Time()), + }, order) + }) + + t.Run("failed to convert side", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Side = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder type", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.OrderType = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("failed to convert oder status", func(t *testing.T) { + newOrder := unfilledOrder + newOrder.Status = "xxx" + + _, err := toGlobalOrder(newOrder) + assert.ErrorContains(err, "xxx") + }) +} + +func Test_processMarketBuyQuantity(t *testing.T) { + var ( + assert = assert.New(t) + filledBaseCoinQty = fixedpoint.NewFromFloat(3.5648) + filledPrice = fixedpoint.NewFromFloat(4.99998848) + priceAvg = fixedpoint.NewFromFloat(1.4026) + buyQty = fixedpoint.NewFromFloat(5) + ) + + t.Run("zero quantity on Init/New/Live/Cancelled", func(t *testing.T) { + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusInit) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusNew) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusLive) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + + qty, err = processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusCancelled) + assert.NoError(err) + assert.Equal(fixedpoint.Zero, qty) + }) + + t.Run("5 on PartialFilled", func(t *testing.T) { + priceAvg := fixedpoint.NewFromFloat(2) + buyQty := fixedpoint.NewFromFloat(10) + filledPrice := fixedpoint.NewFromFloat(4) + filledBaseCoinQty := fixedpoint.NewFromFloat(2) + + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusPartialFilled) + assert.NoError(err) + assert.Equal(fixedpoint.NewFromFloat(5), qty) + }) + + t.Run("3.5648 on Filled", func(t *testing.T) { + qty, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, v2.OrderStatusFilled) + assert.NoError(err) + assert.Equal(fixedpoint.NewFromFloat(3.5648), qty) + }) + + t.Run("unexpected order status", func(t *testing.T) { + _, err := processMarketBuyQuantity(filledBaseCoinQty, filledPrice, priceAvg, buyQty, "xxx") + assert.ErrorContains(err, "xxx") + }) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 542a3bfc26..298054e0ab 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -7,6 +7,7 @@ import ( "time" "github.com/sirupsen/logrus" + "go.uber.org/multierr" "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" @@ -19,7 +20,9 @@ const ( PlatformToken = "BGB" - queryOpenOrdersLimit = 100 + queryLimit = 100 + maxOrderIdLen = 36 + queryMaxDuration = 90 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -37,6 +40,8 @@ var ( queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // queryOpenOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Unfilled-Orders queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // closedQueryOrdersRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Get-History-Orders + closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) ) type Exchange struct { @@ -192,7 +197,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ req := e.v2Client.NewGetUnfilledOrdersRequest(). Symbol(symbol). - Limit(strconv.FormatInt(queryOpenOrdersLimit, 10)) + Limit(strconv.FormatInt(queryLimit, 10)) if nextCursor != 0 { req.IdLessThan(strconv.FormatInt(int64(nextCursor), 10)) } @@ -213,11 +218,11 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ orderLen := len(openOrders) // a defensive programming to ensure the length of order response is expected. - if orderLen > queryOpenOrdersLimit { + if orderLen > queryLimit { return nil, fmt.Errorf("unexpected open orders length %d", orderLen) } - if orderLen < queryOpenOrdersLimit { + if orderLen < queryLimit { break } nextCursor = openOrders[orderLen-1].OrderId @@ -226,6 +231,57 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return orders, nil } +// QueryClosedOrders queries closed order by time range(`CTime`) and id. The order of the response is in descending order. +// If you need to retrieve all data, please utilize the function pkg/exchange/batch.ClosedOrderBatchQuery. +// +// ** Since is inclusive, Until is exclusive. If you use a time range to query, you must provide both a start time and an end time. ** +// ** Since and Until cannot exceed 90 days. ** +// ** Since from the last 90 days can be queried. ** +func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { + if since.Sub(time.Now()) > queryMaxDuration { + return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", since) + } + if until.Before(since) { + return nil, fmt.Errorf("end time %s before start %s", until, since) + } + if until.Sub(since) > queryMaxDuration { + return nil, fmt.Errorf("the start time %s and end time %s cannot exceed 90 days", since, until) + } + if lastOrderID != 0 { + log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The order of response is in descending order, so the last order id not supported.") + } + + if err := closedQueryOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) + } + res, err := e.v2Client.NewGetHistoryOrdersRequest(). + Symbol(symbol). + Limit(strconv.Itoa(queryLimit)). + StartTime(since.UnixMilli()). + EndTime(until.UnixMilli()). + Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call get order histories error: %w", err) + } + + for _, order := range res { + o, err2 := toGlobalOrder(order) + if err2 != nil { + err = multierr.Append(err, err2) + continue + } + + if o.Status.Closed() { + orders = append(orders, *o) + } + } + if err != nil { + return nil, err + } + + return types.SortOrdersAscending(orders), nil +} + func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { // TODO implement me panic("implement me") From 2c842e54e8f5513ed98aee2eb2f966fee171349a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 17:01:04 +0800 Subject: [PATCH 159/422] scmaker: fix scmaker stream book binding --- pkg/strategy/scmaker/strategy.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index ecccefc8b0..d9eb8d7aa4 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -19,8 +19,6 @@ import ( const ID = "scmaker" -var ten = fixedpoint.NewFromInt(10) - type advancedOrderCancelApi interface { CancelAllOrders(ctx context.Context) ([]types.Order, error) CancelOrdersBySymbol(ctx context.Context, symbol string) ([]types.Order, error) @@ -100,12 +98,12 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } } -func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(session.UserDataStream) + s.book.BindStream(session.MarketDataStream) s.liquidityOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.liquidityOrderBook.BindStream(session.UserDataStream) @@ -174,7 +172,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se return nil } -func (s *Strategy) preloadKLines(inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval) { +func (s *Strategy) preloadKLines( + inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval, +) { if store, ok := session.MarketDataStore(symbol); ok { if kLinesData, ok := store.KLinesOfInterval(interval); ok { for _, k := range *kLinesData { @@ -476,7 +476,9 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { log.Infof("%d liq orders are placed successfully", len(liqOrders)) } -func profitProtectedPrice(side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value) fixedpoint.Value { +func profitProtectedPrice( + side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value, +) fixedpoint.Value { switch side { case types.SideTypeSell: minProfitPrice := averageCost.Add( From d2dab58193009367a73977c77d770b23cee25f86 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 1 Nov 2023 17:05:10 +0800 Subject: [PATCH 160/422] scmaker: clean up scmaker risk control --- pkg/strategy/scmaker/strategy.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index d9eb8d7aa4..79e3f89569 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -5,14 +5,12 @@ import ( "fmt" "math" "sync" - "time" log "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" . "github.com/c9s/bbgo/pkg/indicator/v2" - "github.com/c9s/bbgo/pkg/risk/riskcontrol" "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -60,12 +58,6 @@ type Strategy struct { MinProfit fixedpoint.Value `json:"minProfit"` - // risk related parameters - PositionHardLimit fixedpoint.Value `json:"positionHardLimit"` - MaxPositionQuantity fixedpoint.Value `json:"maxPositionQuantity"` - CircuitBreakLossThreshold fixedpoint.Value `json:"circuitBreakLossThreshold"` - CircuitBreakEMA types.IntervalWindow `json:"circuitBreakEMA"` - liquidityOrderBook, adjustmentOrderBook *bbgo.ActiveOrderBook book *types.StreamOrderBook @@ -75,9 +67,6 @@ type Strategy struct { ewma *EWMAStream boll *BOLLStream intensity *IntensityStream - - positionRiskControl *riskcontrol.PositionRiskControl - circuitBreakRiskControl *riskcontrol.CircuitBreakRiskControl } func (s *Strategy) ID() string { @@ -111,21 +100,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.adjustmentOrderBook = bbgo.NewActiveOrderBook(s.Symbol) s.adjustmentOrderBook.BindStream(session.UserDataStream) - if !s.PositionHardLimit.IsZero() && !s.MaxPositionQuantity.IsZero() { - log.Infof("positionHardLimit and maxPositionQuantity are configured, setting up PositionRiskControl...") - s.positionRiskControl = riskcontrol.NewPositionRiskControl(s.OrderExecutor, s.PositionHardLimit, s.MaxPositionQuantity) - } - - if !s.CircuitBreakLossThreshold.IsZero() { - log.Infof("circuitBreakLossThreshold is configured, setting up CircuitBreakRiskControl...") - s.circuitBreakRiskControl = riskcontrol.NewCircuitBreakRiskControl( - s.Position, - session.Indicators(s.Symbol).EWMA(s.CircuitBreakEMA), - s.CircuitBreakLossThreshold, - s.ProfitStats, - 24*time.Hour) - } - scale, err := s.LiquiditySlideRule.Scale() if err != nil { return err @@ -282,7 +256,7 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { return } - if s.circuitBreakRiskControl != nil && s.circuitBreakRiskControl.IsHalted(ticker.Time) { + if s.IsHalted(ticker.Time) { log.Warn("circuitBreakRiskControl: trading halted") return } From dda2cfb73de27f6597cf924bf6fbfdf4b92ea47a Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 2 Nov 2023 11:59:47 +0800 Subject: [PATCH 161/422] liquiditymaker: first commit --- pkg/strategy/liquiditymaker/strategy.go | 440 ++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 pkg/strategy/liquiditymaker/strategy.go diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go new file mode 100644 index 0000000000..fcd8f17de1 --- /dev/null +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -0,0 +1,440 @@ +package liquiditymaker + +import ( + "context" + "fmt" + "math" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + . "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "liquiditymaker" + +type advancedOrderCancelApi interface { + CancelAllOrders(ctx context.Context) ([]types.Order, error) + CancelOrdersBySymbol(ctx context.Context, symbol string) ([]types.Order, error) +} + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +// Strategy is the strategy struct of LiquidityMaker +// liquidity maker does not care about the current price, it tries to place liquidity orders (limit maker orders) +// around the current mid price +// liquidity maker's target: +// - place enough total liquidity amount on the order book, for example, 20k USDT value liquidity on both sell and buy +// - ensure the spread by placing the orders from the mid price (or the last trade price) +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + + LiquidityUpdateInterval types.Interval `json:"liquidityUpdateInterval"` + + AdjustmentUpdateInterval types.Interval `json:"adjustmentUpdateInterval"` + + NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` + LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` + LiquidityLayerTickSize fixedpoint.Value `json:"liquidityLayerTickSize"` + LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` + LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + + Spread fixedpoint.Value `json:"spread"` + MaxPrice fixedpoint.Value `json:"maxPrice"` + MinPrice fixedpoint.Value `json:"minPrice"` + + MaxExposure fixedpoint.Value `json:"maxExposure"` + + MinProfit fixedpoint.Value `json:"minProfit"` + + liquidityOrderBook, adjustmentOrderBook *bbgo.ActiveOrderBook + book *types.StreamOrderBook + + liquidityScale bbgo.Scale +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.AdjustmentUpdateInterval}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.LiquidityUpdateInterval}) +} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + + s.book = types.NewStreamBook(s.Symbol) + s.book.BindStream(session.MarketDataStream) + + s.liquidityOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.liquidityOrderBook.BindStream(session.UserDataStream) + + s.adjustmentOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.adjustmentOrderBook.BindStream(session.UserDataStream) + + scale, err := s.LiquiditySlideRule.Scale() + if err != nil { + return err + } + + if err := scale.Solve(); err != nil { + return err + } + + if cancelApi, ok := session.Exchange.(advancedOrderCancelApi); ok { + _, _ = cancelApi.CancelOrdersBySymbol(ctx, s.Symbol) + } + + s.liquidityScale = scale + + session.UserDataStream.OnStart(func() { + s.placeLiquidityOrders(ctx) + }) + + session.MarketDataStream.OnKLineClosed(func(k types.KLine) { + if k.Interval == s.AdjustmentUpdateInterval { + s.placeAdjustmentOrders(ctx) + } + + if k.Interval == s.LiquidityUpdateInterval { + s.placeLiquidityOrders(ctx) + } + }) + + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + + if err := s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { + logErr(err, "unable to cancel liquidity orders") + } + + if err := s.adjustmentOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { + logErr(err, "unable to cancel adjustment orders") + } + }) + + return nil +} + +func (s *Strategy) placeAdjustmentOrders(ctx context.Context) { + _ = s.adjustmentOrderBook.GracefulCancel(ctx, s.Session.Exchange) + + if s.Position.IsDust() { + return + } + + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if logErr(err, "unable to query ticker") { + return + } + + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") + return + } + + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + + var adjOrders []types.SubmitOrder + + posSize := s.Position.Base.Abs() + tickSize := s.Market.TickSize + + if s.Position.IsShort() { + price := profitProtectedPrice(types.SideTypeBuy, s.Position.AverageCost, ticker.Sell.Add(tickSize.Neg()), s.Session.MakerFeeRate, s.MinProfit) + quoteQuantity := fixedpoint.Min(price.Mul(posSize), quoteBal.Available) + bidQuantity := quoteQuantity.Div(price) + + if s.Market.IsDustQuantity(bidQuantity, price) { + return + } + + adjOrders = append(adjOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimitMaker, + Side: types.SideTypeBuy, + Price: price, + Quantity: bidQuantity, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } else if s.Position.IsLong() { + price := profitProtectedPrice(types.SideTypeSell, s.Position.AverageCost, ticker.Buy.Add(tickSize), s.Session.MakerFeeRate, s.MinProfit) + askQuantity := fixedpoint.Min(posSize, baseBal.Available) + + if s.Market.IsDustQuantity(askQuantity, price) { + return + } + + adjOrders = append(adjOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimitMaker, + Side: types.SideTypeSell, + Price: price, + Quantity: askQuantity, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, adjOrders...) + if logErr(err, "unable to place liquidity orders") { + return + } + + s.adjustmentOrderBook.Add(createdOrders...) +} + +func (s *Strategy) placeLiquidityOrders(ctx context.Context) { + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if logErr(err, "unable to query ticker") { + return + } + + if s.IsHalted(ticker.Time) { + log.Warn("circuitBreakRiskControl: trading halted") + return + } + + err = s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) + if logErr(err, "unable to cancel orders") { + return + } + + if ticker.Buy.IsZero() && ticker.Sell.IsZero() { + ticker.Sell = ticker.Last.Add(s.Market.TickSize) + ticker.Buy = ticker.Last.Sub(s.Market.TickSize) + } else if ticker.Buy.IsZero() { + ticker.Buy = ticker.Sell.Sub(s.Market.TickSize) + } else if ticker.Sell.IsZero() { + ticker.Sell = ticker.Buy.Add(s.Market.TickSize) + } + + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") + return + } + + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + + lastTradedPrice := ticker.Last + midPrice := ticker.Sell.Add(ticker.Buy).Div(fixedpoint.Two) + currentSpread := ticker.Sell.Sub(ticker.Buy) + tickSize := fixedpoint.Max(s.LiquidityLayerTickSize, s.Market.TickSize) + sideSpread := s.Spread.Div(fixedpoint.Two) + + log.Infof("current: spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) + + ask1Price := midPrice.Mul(fixedpoint.One.Add(sideSpread)) + bid1Price := midPrice.Mul(fixedpoint.One.Sub(sideSpread)) + + askLastPrice := midPrice.Mul(fixedpoint.One.Add(s.LiquidityPriceRange)) + bidLastPrice := midPrice.Mul(fixedpoint.One.Sub(s.LiquidityPriceRange)) + log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", sideSpread.Float64(), + ask1Price.Float64(), askLastPrice.Float64(), + bid1Price.Float64(), bidLastPrice.Float64()) + + askLayerSpread := askLastPrice.Sub(ask1Price).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) + bidLayerSpread := bid1Price.Sub(bidLastPrice).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) + + if askLayerSpread.Compare(tickSize) < 0 { + askLayerSpread = tickSize + } + + if bidLayerSpread.Compare(tickSize) < 0 { + bidLayerSpread = tickSize + } + + sum := s.liquidityScale.Sum(1.0) + askSum := sum + bidSum := sum + log.Infof("liquidity sum: %f / %f", askSum, bidSum) + + skew := s.LiquiditySkew.Float64() + useSkew := !s.LiquiditySkew.IsZero() + if useSkew { + askSum = sum / skew + bidSum = sum * skew + log.Infof("adjusted liqudity skew: %f / %f", askSum, bidSum) + } + + var bidPrices []fixedpoint.Value + var askPrices []fixedpoint.Value + + // calculate and collect prices + for i := 0; i <= s.NumOfLiquidityLayers; i++ { + fi := fixedpoint.NewFromInt(int64(i)) + bidPrice := bid1Price.Sub(bidLayerSpread.Mul(fi)) + askPrice := ask1Price.Add(askLayerSpread.Mul(fi)) + + bidPrice = s.Market.TruncatePrice(bidPrice) + askPrice = s.Market.TruncatePrice(askPrice) + + bidPrices = append(bidPrices, bidPrice) + askPrices = append(askPrices, askPrice) + } + + availableBase := baseBal.Available + availableQuote := quoteBal.Available + + makerQuota := &bbgo.QuotaTransaction{} + makerQuota.QuoteAsset.Add(availableQuote) + makerQuota.BaseAsset.Add(availableBase) + + log.Infof("balances before liq orders: %s, %s", + baseBal.String(), + quoteBal.String()) + + if !s.Position.IsDust() { + if s.Position.IsLong() { + availableBase = availableBase.Sub(s.Position.Base) + availableBase = s.Market.RoundDownQuantityByPrecision(availableBase) + } else if s.Position.IsShort() { + posSizeInQuote := s.Position.Base.Mul(ticker.Sell) + availableQuote = availableQuote.Sub(posSizeInQuote) + } + } + + askX := availableBase.Float64() / askSum + bidX := availableQuote.Float64() / (bidSum * (fixedpoint.Sum(bidPrices).Float64())) + + askX = math.Trunc(askX*1e8) / 1e8 + bidX = math.Trunc(bidX*1e8) / 1e8 + + var liqOrders []types.SubmitOrder + for i := 0; i <= s.NumOfLiquidityLayers; i++ { + bidQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * bidX) + askQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * askX) + bidPrice := bidPrices[i] + askPrice := askPrices[i] + + log.Infof("liqudity layer #%d %f/%f = %f/%f", i, askPrice.Float64(), bidPrice.Float64(), askQuantity.Float64(), bidQuantity.Float64()) + + placeBuy := true + placeSell := true + averageCost := s.Position.AverageCost + // when long position, do not place sell orders below the average cost + if !s.Position.IsDust() { + if s.Position.IsLong() && askPrice.Compare(averageCost) < 0 { + placeSell = false + } + + if s.Position.IsShort() && bidPrice.Compare(averageCost) > 0 { + placeBuy = false + } + } + + quoteQuantity := bidQuantity.Mul(bidPrice) + + if s.Market.IsDustQuantity(bidQuantity, bidPrice) || !makerQuota.QuoteAsset.Lock(quoteQuantity) { + placeBuy = false + } + + if s.Market.IsDustQuantity(askQuantity, askPrice) || !makerQuota.BaseAsset.Lock(askQuantity) { + placeSell = false + } + + if placeBuy { + liqOrders = append(liqOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeLimitMaker, + Quantity: bidQuantity, + Price: bidPrice, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + + if placeSell { + liqOrders = append(liqOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeLimitMaker, + Quantity: askQuantity, + Price: askPrice, + Market: s.Market, + TimeInForce: types.TimeInForceGTC, + }) + } + } + + makerQuota.Commit() + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, liqOrders...) + if logErr(err, "unable to place liquidity orders") { + return + } + + s.liquidityOrderBook.Add(createdOrders...) + log.Infof("%d liq orders are placed successfully", len(liqOrders)) +} + +func profitProtectedPrice( + side types.SideType, averageCost, price, feeRate, minProfit fixedpoint.Value, +) fixedpoint.Value { + switch side { + case types.SideTypeSell: + minProfitPrice := averageCost.Add( + averageCost.Mul(feeRate.Add(minProfit))) + return fixedpoint.Max(minProfitPrice, price) + + case types.SideTypeBuy: + minProfitPrice := averageCost.Sub( + averageCost.Mul(feeRate.Add(minProfit))) + return fixedpoint.Min(minProfitPrice, price) + + } + return price +} + +func logErr(err error, msgAndArgs ...interface{}) bool { + if err == nil { + return false + } + + if len(msgAndArgs) == 0 { + log.WithError(err).Error(err.Error()) + } else if len(msgAndArgs) == 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Error(msg) + } else if len(msgAndArgs) > 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Errorf(msg, msgAndArgs[1:]...) + } + + return true +} + +func preloadKLines( + inc *KLineStream, session *bbgo.ExchangeSession, symbol string, interval types.Interval, +) { + if store, ok := session.MarketDataStore(symbol); ok { + if kLinesData, ok := store.KLinesOfInterval(interval); ok { + for _, k := range *kLinesData { + inc.EmitUpdate(k) + } + } + } +} From 533907894e954c8c42a2b39486e93b596f025283 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 15:37:12 +0800 Subject: [PATCH 162/422] liquiditymaker: implement order generator --- pkg/strategy/liquiditymaker/generator.go | 95 +++++++++++++++ pkg/strategy/liquiditymaker/generator_test.go | 114 ++++++++++++++++++ pkg/strategy/liquiditymaker/strategy.go | 3 + pkg/testing/testhelper/assert_priceside.go | 58 +++++++++ pkg/testing/testhelper/number.go | 18 +++ 5 files changed, 288 insertions(+) create mode 100644 pkg/strategy/liquiditymaker/generator.go create mode 100644 pkg/strategy/liquiditymaker/generator_test.go create mode 100644 pkg/testing/testhelper/assert_priceside.go create mode 100644 pkg/testing/testhelper/number.go diff --git a/pkg/strategy/liquiditymaker/generator.go b/pkg/strategy/liquiditymaker/generator.go new file mode 100644 index 0000000000..2b19b3f48e --- /dev/null +++ b/pkg/strategy/liquiditymaker/generator.go @@ -0,0 +1,95 @@ +package liquiditymaker + +import ( + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +// input: liquidityOrderGenerator( +// +// totalLiquidityAmount, +// startPrice, +// endPrice, +// numLayers, +// quantityScale) +// +// when side == sell +// +// priceAsk1 * scale(1) * f = amount1 +// priceAsk2 * scale(2) * f = amount2 +// priceAsk3 * scale(3) * f = amount3 +// +// totalLiquidityAmount = priceAsk1 * scale(1) * f + priceAsk2 * scale(2) * f + priceAsk3 * scale(3) * f + .... +// totalLiquidityAmount = f * (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) +// +// when side == buy +// +// priceBid1 * scale(1) * f = amount1 +type LiquidityOrderGenerator struct { + Symbol string + Market types.Market + + logger log.FieldLogger +} + +func (g *LiquidityOrderGenerator) Generate( + side types.SideType, totalAmount, startPrice, endPrice fixedpoint.Value, numLayers int, scale bbgo.Scale, +) (orders []types.SubmitOrder) { + + if g.logger == nil { + logger := log.New() + logger.SetLevel(log.ErrorLevel) + g.logger = logger + } + + layerSpread := endPrice.Sub(startPrice).Div(fixedpoint.NewFromInt(int64(numLayers - 1))) + switch side { + case types.SideTypeSell: + if layerSpread.Compare(g.Market.TickSize) < 0 { + layerSpread = g.Market.TickSize + } + + case types.SideTypeBuy: + if layerSpread.Compare(g.Market.TickSize.Neg()) > 0 { + layerSpread = g.Market.TickSize.Neg() + } + } + + quantityBase := 0.0 + var layerPrices []fixedpoint.Value + var layerScales []float64 + for i := 0; i < numLayers; i++ { + fi := fixedpoint.NewFromInt(int64(i)) + layerPrice := g.Market.TruncatePrice(startPrice.Add(layerSpread.Mul(fi))) + layerPrices = append(layerPrices, layerPrice) + + layerScale := scale.Call(float64(i + 1)) + layerScales = append(layerScales, layerScale) + + quantityBase += layerPrice.Float64() * layerScale + } + + factor := totalAmount.Float64() / quantityBase + + g.logger.Infof("liquidity amount base: %f, factor: %f", quantityBase, factor) + + for i := 0; i < numLayers; i++ { + price := layerPrices[i] + s := layerScales[i] + + quantity := factor * s + orders = append(orders, types.SubmitOrder{ + Symbol: g.Symbol, + Price: price, + Type: types.OrderTypeLimitMaker, + Quantity: g.Market.TruncateQuantity(fixedpoint.NewFromFloat(quantity)), + Side: side, + Market: g.Market, + }) + } + + return orders +} diff --git a/pkg/strategy/liquiditymaker/generator_test.go b/pkg/strategy/liquiditymaker/generator_test.go new file mode 100644 index 0000000000..d56700f6e2 --- /dev/null +++ b/pkg/strategy/liquiditymaker/generator_test.go @@ -0,0 +1,114 @@ +//go:build !dnum + +package liquiditymaker + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + . "github.com/c9s/bbgo/pkg/testing/testhelper" + "github.com/c9s/bbgo/pkg/types" +) + +func newTestMarket() types.Market { + return types.Market{ + BaseCurrency: "XML", + QuoteCurrency: "USDT", + TickSize: Number(0.0001), + StepSize: Number(0.01), + PricePrecision: 4, + VolumePrecision: 8, + MinNotional: Number(8.0), + MinQuantity: Number(40.0), + } +} + +func TestLiquidityOrderGenerator(t *testing.T) { + g := &LiquidityOrderGenerator{ + Symbol: "XMLUSDT", + Market: newTestMarket(), + } + + scale := &bbgo.ExponentialScale{ + Domain: [2]float64{1.0, 30.0}, + Range: [2]float64{1.0, 4.0}, + } + + err := scale.Solve() + assert.NoError(t, err) + assert.InDelta(t, 1.0, scale.Call(1.0), 0.00001) + assert.InDelta(t, 4.0, scale.Call(30.0), 0.00001) + + totalAmount := Number(200_000.0) + + t.Run("ask orders", func(t *testing.T) { + orders := g.Generate(types.SideTypeSell, totalAmount, Number(2.0), Number(2.04), 30, scale) + assert.Len(t, orders, 30) + + totalQuoteQuantity := fixedpoint.NewFromInt(0) + for _, o := range orders { + totalQuoteQuantity = totalQuoteQuantity.Add(o.Quantity.Mul(o.Price)) + } + assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("1513.40")}, + {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("1587.50")}, + {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("1665.23")}, + {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("1746.77")}, + {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("1832.30")}, + {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("1922.02")}, + {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("2016.13")}, + {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("2114.85")}, + {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("2218.40")}, + {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("2327.02")}, + {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("2440.96")}, + {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("2560.48")}, + {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("2685.86")}, + {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("2817.37")}, + {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("2955.32")}, + }, orders[0:15]) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("5771.04")}, + {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("6053.62")}, + }, orders[28:30]) + }) + + t.Run("bid orders", func(t *testing.T) { + orders := g.Generate(types.SideTypeBuy, totalAmount, Number(2.0), Number(1.96), 30, scale) + assert.Len(t, orders, 30) + + totalQuoteQuantity := fixedpoint.NewFromInt(0) + for _, o := range orders { + totalQuoteQuantity = totalQuoteQuantity.Add(o.Quantity.Mul(o.Price)) + } + assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("1551.37")}, + {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("1627.33")}, + {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("1707.01")}, + {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("1790.59")}, + {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("1878.27")}, + {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("1970.24")}, + {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("2066.71")}, + {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("2167.91")}, + {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("2274.06")}, + {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("2385.40")}, + {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("2502.20")}, + {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("2624.72")}, + {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("2753.24")}, + {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("2888.05")}, + {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("3029.46")}, + }, orders[0:15]) + + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("5915.83")}, + {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("6205.49")}, + }, orders[28:30]) + }) +} diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index fcd8f17de1..6dcb8ece0d 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -50,6 +50,9 @@ type Strategy struct { LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` + BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` + Spread fixedpoint.Value `json:"spread"` MaxPrice fixedpoint.Value `json:"maxPrice"` MinPrice fixedpoint.Value `json:"minPrice"` diff --git a/pkg/testing/testhelper/assert_priceside.go b/pkg/testing/testhelper/assert_priceside.go new file mode 100644 index 0000000000..7c45cdd9d8 --- /dev/null +++ b/pkg/testing/testhelper/assert_priceside.go @@ -0,0 +1,58 @@ +package testhelper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type PriceSideAssert struct { + Price fixedpoint.Value + Side types.SideType +} + +// AssertOrdersPriceSide asserts the orders with the given price and side (slice) +func AssertOrdersPriceSide(t *testing.T, asserts []PriceSideAssert, orders []types.SubmitOrder) { + for i, a := range asserts { + assert.Equalf(t, a.Price, orders[i].Price, "order #%d price should be %f", i+1, a.Price.Float64()) + assert.Equalf(t, a.Side, orders[i].Side, "order at price %f should be %s", a.Price.Float64(), a.Side) + } +} + +type PriceSideQuantityAssert struct { + Price fixedpoint.Value + Side types.SideType + Quantity fixedpoint.Value +} + +// AssertOrdersPriceSide asserts the orders with the given price and side (slice) +func AssertOrdersPriceSideQuantity( + t *testing.T, asserts []PriceSideQuantityAssert, orders []types.SubmitOrder, +) { + assert.Equalf(t, len(orders), len(asserts), "expecting %d orders", len(asserts)) + + var assertPrices, orderPrices fixedpoint.Slice + var assertPricesFloat, orderPricesFloat []float64 + for _, a := range asserts { + assertPrices = append(assertPrices, a.Price) + assertPricesFloat = append(assertPricesFloat, a.Price.Float64()) + } + + for _, o := range orders { + orderPrices = append(orderPrices, o.Price) + orderPricesFloat = append(orderPricesFloat, o.Price.Float64()) + } + + if !assert.Equalf(t, assertPricesFloat, orderPricesFloat, "assert prices") { + return + } + + for i, a := range asserts { + assert.Equalf(t, a.Price.Float64(), orders[i].Price.Float64(), "order #%d price should be %f", i+1, a.Price.Float64()) + assert.Equalf(t, a.Quantity.Float64(), orders[i].Quantity.Float64(), "order #%d quantity should be %f", i+1, a.Quantity.Float64()) + assert.Equalf(t, a.Side, orders[i].Side, "order at price %f should be %s", a.Price.Float64(), a.Side) + } +} diff --git a/pkg/testing/testhelper/number.go b/pkg/testing/testhelper/number.go new file mode 100644 index 0000000000..e57659a01a --- /dev/null +++ b/pkg/testing/testhelper/number.go @@ -0,0 +1,18 @@ +package testhelper + +import "github.com/c9s/bbgo/pkg/fixedpoint" + +func Number(a interface{}) fixedpoint.Value { + switch v := a.(type) { + case string: + return fixedpoint.MustNewFromString(v) + case int: + return fixedpoint.NewFromInt(int64(v)) + case int64: + return fixedpoint.NewFromInt(int64(v)) + case float64: + return fixedpoint.NewFromFloat(v) + } + + return fixedpoint.Zero +} From cc5c033af76c2dbd341140b8328c88c3f95bed6c Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 17:54:01 +0800 Subject: [PATCH 163/422] liquiditymaker: use order generator --- config/liquiditymaker.yaml | 54 +++++ pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/liquiditymaker/generator_test.go | 70 +++---- pkg/strategy/liquiditymaker/strategy.go | 185 +++++------------- 4 files changed, 142 insertions(+), 168 deletions(-) create mode 100644 config/liquiditymaker.yaml diff --git a/config/liquiditymaker.yaml b/config/liquiditymaker.yaml new file mode 100644 index 0000000000..85288d5b9e --- /dev/null +++ b/config/liquiditymaker.yaml @@ -0,0 +1,54 @@ +sessions: + max: + exchange: max + envVarPrefix: max + makerFeeRate: 0% + takerFeeRate: 0.025% + +#services: +# googleSpreadSheet: +# jsonTokenFile: ".credentials/google-cloud/service-account-json-token.json" +# spreadSheetId: "YOUR_SPREADSHEET_ID" + +exchangeStrategies: +- on: max + liquiditymaker: + symbol: &symbol USDTTWD + + ## adjustmentUpdateInterval is the interval for adjusting position + adjustmentUpdateInterval: 1m + + ## liquidityUpdateInterval is the interval for updating liquidity orders + liquidityUpdateInterval: 1h + + numOfLiquidityLayers: 30 + askLiquidityAmount: 20_000.0 + bidLiquidityAmount: 20_000.0 + liquidityPriceRange: 2% + useLastTradePrice: true + spread: 1.1% + + liquidityScale: + exp: + domain: [1, 30] + range: [1, 4] + + ## maxExposure controls how much balance should be used for placing the maker orders + maxExposure: 200_000 + minProfit: 0.01% + + +backtest: + sessions: + - max + startTime: "2023-05-20" + endTime: "2023-06-01" + symbols: + - *symbol + account: + max: + makerFeeRate: 0.0% + takerFeeRate: 0.025% + balances: + USDT: 5000 + TWD: 150_000 diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index d868e926ab..867c72dc2a 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -25,6 +25,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/irr" _ "github.com/c9s/bbgo/pkg/strategy/kline" _ "github.com/c9s/bbgo/pkg/strategy/linregmaker" + _ "github.com/c9s/bbgo/pkg/strategy/liquiditymaker" _ "github.com/c9s/bbgo/pkg/strategy/marketcap" _ "github.com/c9s/bbgo/pkg/strategy/pivotshort" _ "github.com/c9s/bbgo/pkg/strategy/pricealert" diff --git a/pkg/strategy/liquiditymaker/generator_test.go b/pkg/strategy/liquiditymaker/generator_test.go index d56700f6e2..995f33bd77 100644 --- a/pkg/strategy/liquiditymaker/generator_test.go +++ b/pkg/strategy/liquiditymaker/generator_test.go @@ -42,7 +42,7 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, 1.0, scale.Call(1.0), 0.00001) assert.InDelta(t, 4.0, scale.Call(30.0), 0.00001) - totalAmount := Number(200_000.0) + totalAmount := Number(20_000.0) t.Run("ask orders", func(t *testing.T) { orders := g.Generate(types.SideTypeSell, totalAmount, Number(2.0), Number(2.04), 30, scale) @@ -55,26 +55,26 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("1513.40")}, - {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("1587.50")}, - {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("1665.23")}, - {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("1746.77")}, - {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("1832.30")}, - {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("1922.02")}, - {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("2016.13")}, - {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("2114.85")}, - {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("2218.40")}, - {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("2327.02")}, - {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("2440.96")}, - {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("2560.48")}, - {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("2685.86")}, - {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("2817.37")}, - {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("2955.32")}, + {Side: types.SideTypeSell, Price: Number("2.0000"), Quantity: Number("151.34")}, + {Side: types.SideTypeSell, Price: Number("2.0013"), Quantity: Number("158.75")}, + {Side: types.SideTypeSell, Price: Number("2.0027"), Quantity: Number("166.52")}, + {Side: types.SideTypeSell, Price: Number("2.0041"), Quantity: Number("174.67")}, + {Side: types.SideTypeSell, Price: Number("2.0055"), Quantity: Number("183.23")}, + {Side: types.SideTypeSell, Price: Number("2.0068"), Quantity: Number("192.20")}, + {Side: types.SideTypeSell, Price: Number("2.0082"), Quantity: Number("201.61")}, + {Side: types.SideTypeSell, Price: Number("2.0096"), Quantity: Number("211.48")}, + {Side: types.SideTypeSell, Price: Number("2.0110"), Quantity: Number("221.84")}, + {Side: types.SideTypeSell, Price: Number("2.0124"), Quantity: Number("232.70")}, + {Side: types.SideTypeSell, Price: Number("2.0137"), Quantity: Number("244.09")}, + {Side: types.SideTypeSell, Price: Number("2.0151"), Quantity: Number("256.04")}, + {Side: types.SideTypeSell, Price: Number("2.0165"), Quantity: Number("268.58")}, + {Side: types.SideTypeSell, Price: Number("2.0179"), Quantity: Number("281.73")}, + {Side: types.SideTypeSell, Price: Number("2.0193"), Quantity: Number("295.53")}, }, orders[0:15]) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("5771.04")}, - {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("6053.62")}, + {Side: types.SideTypeSell, Price: Number("2.0386"), Quantity: Number("577.10")}, + {Side: types.SideTypeSell, Price: Number("2.0399"), Quantity: Number("605.36")}, }, orders[28:30]) }) @@ -89,26 +89,26 @@ func TestLiquidityOrderGenerator(t *testing.T) { assert.InDelta(t, totalAmount.Float64(), totalQuoteQuantity.Float64(), 1.0) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("1551.37")}, - {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("1627.33")}, - {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("1707.01")}, - {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("1790.59")}, - {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("1878.27")}, - {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("1970.24")}, - {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("2066.71")}, - {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("2167.91")}, - {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("2274.06")}, - {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("2385.40")}, - {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("2502.20")}, - {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("2624.72")}, - {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("2753.24")}, - {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("2888.05")}, - {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("3029.46")}, + {Side: types.SideTypeBuy, Price: Number("2.0000"), Quantity: Number("155.13")}, + {Side: types.SideTypeBuy, Price: Number("1.9986"), Quantity: Number("162.73")}, + {Side: types.SideTypeBuy, Price: Number("1.9972"), Quantity: Number("170.70")}, + {Side: types.SideTypeBuy, Price: Number("1.9958"), Quantity: Number("179.05")}, + {Side: types.SideTypeBuy, Price: Number("1.9944"), Quantity: Number("187.82")}, + {Side: types.SideTypeBuy, Price: Number("1.9931"), Quantity: Number("197.02")}, + {Side: types.SideTypeBuy, Price: Number("1.9917"), Quantity: Number("206.67")}, + {Side: types.SideTypeBuy, Price: Number("1.9903"), Quantity: Number("216.79")}, + {Side: types.SideTypeBuy, Price: Number("1.9889"), Quantity: Number("227.40")}, + {Side: types.SideTypeBuy, Price: Number("1.9875"), Quantity: Number("238.54")}, + {Side: types.SideTypeBuy, Price: Number("1.9862"), Quantity: Number("250.22")}, + {Side: types.SideTypeBuy, Price: Number("1.9848"), Quantity: Number("262.47")}, + {Side: types.SideTypeBuy, Price: Number("1.9834"), Quantity: Number("275.32")}, + {Side: types.SideTypeBuy, Price: Number("1.9820"), Quantity: Number("288.80")}, + {Side: types.SideTypeBuy, Price: Number("1.9806"), Quantity: Number("302.94")}, }, orders[0:15]) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ - {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("5915.83")}, - {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("6205.49")}, + {Side: types.SideTypeBuy, Price: Number("1.9613"), Quantity: Number("591.58")}, + {Side: types.SideTypeBuy, Price: Number("1.9600"), Quantity: Number("620.54")}, }, orders[28:30]) }) } diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 6dcb8ece0d..07a6517e7c 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -3,7 +3,6 @@ package liquiditymaker import ( "context" "fmt" - "math" "sync" log "github.com/sirupsen/logrus" @@ -44,18 +43,16 @@ type Strategy struct { AdjustmentUpdateInterval types.Interval `json:"adjustmentUpdateInterval"` - NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` - LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` - LiquidityLayerTickSize fixedpoint.Value `json:"liquidityLayerTickSize"` - LiquiditySkew fixedpoint.Value `json:"liquiditySkew"` - LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + NumOfLiquidityLayers int `json:"numOfLiquidityLayers"` + LiquiditySlideRule *bbgo.SlideRule `json:"liquidityScale"` + LiquidityPriceRange fixedpoint.Value `json:"liquidityPriceRange"` + AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` + BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` - AskLiquidityAmount fixedpoint.Value `json:"askLiquidityAmount"` - BidLiquidityAmount fixedpoint.Value `json:"bidLiquidityAmount"` - - Spread fixedpoint.Value `json:"spread"` - MaxPrice fixedpoint.Value `json:"maxPrice"` - MinPrice fixedpoint.Value `json:"minPrice"` + UseLastTradePrice bool `json:"useLastTradePrice"` + Spread fixedpoint.Value `json:"spread"` + MaxPrice fixedpoint.Value `json:"maxPrice"` + MinPrice fixedpoint.Value `json:"minPrice"` MaxExposure fixedpoint.Value `json:"maxExposure"` @@ -65,6 +62,8 @@ type Strategy struct { book *types.StreamOrderBook liquidityScale bbgo.Scale + + orderGenerator *LiquidityOrderGenerator } func (s *Strategy) ID() string { @@ -85,6 +84,11 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + s.orderGenerator = &LiquidityOrderGenerator{ + Symbol: s.Symbol, + Market: s.Market, + } + s.book = types.NewStreamBook(s.Symbol) s.book.BindStream(session.MarketDataStream) @@ -209,6 +213,11 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context) { } func (s *Strategy) placeLiquidityOrders(ctx context.Context) { + err := s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) + if logErr(err, "unable to cancel orders") { + return + } + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) if logErr(err, "unable to query ticker") { return @@ -219,11 +228,14 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { return } - err = s.liquidityOrderBook.GracefulCancel(ctx, s.Session.Exchange) - if logErr(err, "unable to cancel orders") { + if _, err := s.Session.UpdateAccount(ctx); err != nil { + logErr(err, "unable to update account") return } + baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) + quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + if ticker.Buy.IsZero() && ticker.Sell.IsZero() { ticker.Sell = ticker.Last.Add(s.Market.TickSize) ticker.Buy = ticker.Last.Sub(s.Market.TickSize) @@ -233,78 +245,32 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { ticker.Sell = ticker.Buy.Add(s.Market.TickSize) } - if _, err := s.Session.UpdateAccount(ctx); err != nil { - logErr(err, "unable to update account") - return - } - - baseBal, _ := s.Session.Account.Balance(s.Market.BaseCurrency) - quoteBal, _ := s.Session.Account.Balance(s.Market.QuoteCurrency) + log.Infof("ticker: %+v", ticker) lastTradedPrice := ticker.Last midPrice := ticker.Sell.Add(ticker.Buy).Div(fixedpoint.Two) currentSpread := ticker.Sell.Sub(ticker.Buy) - tickSize := fixedpoint.Max(s.LiquidityLayerTickSize, s.Market.TickSize) sideSpread := s.Spread.Div(fixedpoint.Two) - log.Infof("current: spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) + if s.UseLastTradePrice { + midPrice = lastTradedPrice + } + + log.Infof("current spread: %f lastTradedPrice: %f midPrice: %f", currentSpread.Float64(), lastTradedPrice.Float64(), midPrice.Float64()) ask1Price := midPrice.Mul(fixedpoint.One.Add(sideSpread)) bid1Price := midPrice.Mul(fixedpoint.One.Sub(sideSpread)) askLastPrice := midPrice.Mul(fixedpoint.One.Add(s.LiquidityPriceRange)) bidLastPrice := midPrice.Mul(fixedpoint.One.Sub(s.LiquidityPriceRange)) - log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", sideSpread.Float64(), + log.Infof("wanted side spread: %f askRange: %f ~ %f bidRange: %f ~ %f", + sideSpread.Float64(), ask1Price.Float64(), askLastPrice.Float64(), bid1Price.Float64(), bidLastPrice.Float64()) - askLayerSpread := askLastPrice.Sub(ask1Price).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) - bidLayerSpread := bid1Price.Sub(bidLastPrice).Div(fixedpoint.NewFromInt(int64(s.NumOfLiquidityLayers))) - - if askLayerSpread.Compare(tickSize) < 0 { - askLayerSpread = tickSize - } - - if bidLayerSpread.Compare(tickSize) < 0 { - bidLayerSpread = tickSize - } - - sum := s.liquidityScale.Sum(1.0) - askSum := sum - bidSum := sum - log.Infof("liquidity sum: %f / %f", askSum, bidSum) - - skew := s.LiquiditySkew.Float64() - useSkew := !s.LiquiditySkew.IsZero() - if useSkew { - askSum = sum / skew - bidSum = sum * skew - log.Infof("adjusted liqudity skew: %f / %f", askSum, bidSum) - } - - var bidPrices []fixedpoint.Value - var askPrices []fixedpoint.Value - - // calculate and collect prices - for i := 0; i <= s.NumOfLiquidityLayers; i++ { - fi := fixedpoint.NewFromInt(int64(i)) - bidPrice := bid1Price.Sub(bidLayerSpread.Mul(fi)) - askPrice := ask1Price.Add(askLayerSpread.Mul(fi)) - - bidPrice = s.Market.TruncatePrice(bidPrice) - askPrice = s.Market.TruncatePrice(askPrice) - - bidPrices = append(bidPrices, bidPrice) - askPrices = append(askPrices, askPrice) - } - availableBase := baseBal.Available availableQuote := quoteBal.Available - makerQuota := &bbgo.QuotaTransaction{} - makerQuota.QuoteAsset.Add(availableQuote) - makerQuota.BaseAsset.Add(availableBase) - log.Infof("balances before liq orders: %s, %s", baseBal.String(), quoteBal.String()) @@ -319,79 +285,32 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { } } - askX := availableBase.Float64() / askSum - bidX := availableQuote.Float64() / (bidSum * (fixedpoint.Sum(bidPrices).Float64())) + bidOrders := s.orderGenerator.Generate(types.SideTypeBuy, + fixedpoint.Min(s.BidLiquidityAmount, quoteBal.Available), + bid1Price, + bidLastPrice, + s.NumOfLiquidityLayers, + s.liquidityScale) - askX = math.Trunc(askX*1e8) / 1e8 - bidX = math.Trunc(bidX*1e8) / 1e8 + askOrders := s.orderGenerator.Generate(types.SideTypeSell, + s.AskLiquidityAmount, + ask1Price, + askLastPrice, + s.NumOfLiquidityLayers, + s.liquidityScale) - var liqOrders []types.SubmitOrder - for i := 0; i <= s.NumOfLiquidityLayers; i++ { - bidQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * bidX) - askQuantity := fixedpoint.NewFromFloat(s.liquidityScale.Call(float64(i)) * askX) - bidPrice := bidPrices[i] - askPrice := askPrices[i] + orderForms := append(bidOrders, askOrders...) - log.Infof("liqudity layer #%d %f/%f = %f/%f", i, askPrice.Float64(), bidPrice.Float64(), askQuantity.Float64(), bidQuantity.Float64()) - - placeBuy := true - placeSell := true - averageCost := s.Position.AverageCost - // when long position, do not place sell orders below the average cost - if !s.Position.IsDust() { - if s.Position.IsLong() && askPrice.Compare(averageCost) < 0 { - placeSell = false - } - - if s.Position.IsShort() && bidPrice.Compare(averageCost) > 0 { - placeBuy = false - } - } - - quoteQuantity := bidQuantity.Mul(bidPrice) - - if s.Market.IsDustQuantity(bidQuantity, bidPrice) || !makerQuota.QuoteAsset.Lock(quoteQuantity) { - placeBuy = false - } - - if s.Market.IsDustQuantity(askQuantity, askPrice) || !makerQuota.BaseAsset.Lock(askQuantity) { - placeSell = false - } - - if placeBuy { - liqOrders = append(liqOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeBuy, - Type: types.OrderTypeLimitMaker, - Quantity: bidQuantity, - Price: bidPrice, - Market: s.Market, - TimeInForce: types.TimeInForceGTC, - }) - } - - if placeSell { - liqOrders = append(liqOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeSell, - Type: types.OrderTypeLimitMaker, - Quantity: askQuantity, - Price: askPrice, - Market: s.Market, - TimeInForce: types.TimeInForceGTC, - }) - } - } - - makerQuota.Commit() - - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, liqOrders...) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) if logErr(err, "unable to place liquidity orders") { return } s.liquidityOrderBook.Add(createdOrders...) - log.Infof("%d liq orders are placed successfully", len(liqOrders)) + log.Infof("%d liq orders are placed successfully", len(orderForms)) + for _, o := range createdOrders { + log.Infof("liq order: %+v", o) + } } func profitProtectedPrice( From 3563c0b98601066aeefb0300ff02d6955ef84805 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 8 Nov 2023 20:19:05 +0800 Subject: [PATCH 164/422] liquiditymaker: filterAskOrders by base balance --- pkg/strategy/liquiditymaker/generator.go | 1 + pkg/strategy/liquiditymaker/strategy.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/strategy/liquiditymaker/generator.go b/pkg/strategy/liquiditymaker/generator.go index 2b19b3f48e..d21d79b835 100644 --- a/pkg/strategy/liquiditymaker/generator.go +++ b/pkg/strategy/liquiditymaker/generator.go @@ -24,6 +24,7 @@ import ( // // totalLiquidityAmount = priceAsk1 * scale(1) * f + priceAsk2 * scale(2) * f + priceAsk3 * scale(3) * f + .... // totalLiquidityAmount = f * (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) +// f = totalLiquidityAmount / (priceAsk1 * scale(1) + priceAsk2 * scale(2) + priceAsk3 * scale(3) + ....) // // when side == buy // diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 07a6517e7c..9d90e8fedf 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -299,6 +299,8 @@ func (s *Strategy) placeLiquidityOrders(ctx context.Context) { s.NumOfLiquidityLayers, s.liquidityScale) + askOrders = filterAskOrders(askOrders, baseBal.Available) + orderForms := append(bidOrders, askOrders...) createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) @@ -331,6 +333,20 @@ func profitProtectedPrice( return price } +func filterAskOrders(askOrders []types.SubmitOrder, available fixedpoint.Value) (out []types.SubmitOrder) { + usedBase := fixedpoint.Zero + for _, askOrder := range askOrders { + if usedBase.Add(askOrder.Quantity).Compare(available) > 0 { + return out + } + + usedBase = usedBase.Add(askOrder.Quantity) + out = append(out, askOrder) + } + + return out +} + func logErr(err error, msgAndArgs ...interface{}) bool { if err == nil { return false From 3d1de0ca058fd852740947621ff2f31472150aa0 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 165/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index cf60746a8b..605f188bc8 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index 973650f24c..ddd67acf42 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index f8c477e0b0..4c89aa619e 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 634db28630..014c4d5a87 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 77010d6d88..8d679a4543 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 2d0fcdfdc3..956e07afb6 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index a59a6bb681..42754dcf11 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 10b5a5942a..05d2851889 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index b18f69fc6d..5fe462deb8 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index b73d4bc4a2..947b9f0791 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index 684ca6e99e..eefb045b20 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index fca05d90be..9032b60974 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 6d44204538..6ea3daa8bd 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index fe99b1f648..645c604aa5 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index 707effa64d..e336f2531d 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index 1572edb082..b6f9b7aec7 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index e01fe74072..aaf0435e6c 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 3df96f1312..5dc938ed70 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index bb4f4d4436..fc712c940c 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index dd88e0a1a9..bbd40cef67 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index c7b28f2f39..7c0a5122b1 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 578af4b2e2..667b8c8c86 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 8a0c602929..4aeca4c37a 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 26650fc61f..7e0df13624 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index bd3edcbea1..f2c0c9ac11 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index 916e8d7dcb..02a156a824 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index cdc5373c65..dd86d5bd82 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index 72df8d23cd..0b83b12cb6 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 91e741f82f..e27f0988fc 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 2-Oct-2023 +###### Auto generated by spf13/cobra on 9-Nov-2023 From 31fb96c171fad3d1dc3192b2683ec0c2d2e64cb8 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 166/422] bump version to v1.53.0 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index df7940b6e7..5b92569c2f 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.52.0-2058ce80-dev" +const Version = "v1.53.0-4c701676-dev" -const VersionGitRef = "2058ce80" +const VersionGitRef = "4c701676" diff --git a/pkg/version/version.go b/pkg/version/version.go index 1be5d77039..cdefa39ba3 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.52.0-2058ce80" +const Version = "v1.53.0-4c701676" -const VersionGitRef = "2058ce80" +const VersionGitRef = "4c701676" From 28c8fda2dbac80c740a84ab0775e52eb2002e2b5 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 9 Nov 2023 12:56:18 +0800 Subject: [PATCH 167/422] add v1.53.0 release note --- doc/release/v1.53.0.md | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 doc/release/v1.53.0.md diff --git a/doc/release/v1.53.0.md b/doc/release/v1.53.0.md new file mode 100644 index 0000000000..8654d5f4f1 --- /dev/null +++ b/doc/release/v1.53.0.md @@ -0,0 +1,65 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.52.0...main) + + - [#1401](https://github.com/c9s/bbgo/pull/1401): STRATEGY: add liquidity maker + - [#1403](https://github.com/c9s/bbgo/pull/1403): FEATURE: [bybit] add assertion for API response + - [#1394](https://github.com/c9s/bbgo/pull/1394): FEATURE: [bitget] support query closed orders + - [#1392](https://github.com/c9s/bbgo/pull/1392): FEATURE: [bitget] add query open orders + - [#1396](https://github.com/c9s/bbgo/pull/1396): FEATURE: add ttl for position/grid2.profit stats persistence + - [#1395](https://github.com/c9s/bbgo/pull/1395): FIX: fix skip syncing active order + - [#1398](https://github.com/c9s/bbgo/pull/1398): FIX: [bybit] rm retry and add fee recover + - [#1397](https://github.com/c9s/bbgo/pull/1397): FEATURE: [bybit] to periodically fetch the fee rate + - [#1391](https://github.com/c9s/bbgo/pull/1391): FIX: [grid2] respect BaseGridNum and add a failing test case + - [#1390](https://github.com/c9s/bbgo/pull/1390): FIX: [rebalance] fix buy quantity + - [#1380](https://github.com/c9s/bbgo/pull/1380): FEATURE: [bitget] support kline subscription on stream + - [#1385](https://github.com/c9s/bbgo/pull/1385): FEATURE: [bitget] add query tickers api + - [#1376](https://github.com/c9s/bbgo/pull/1376): FEATURE: query trades from db page by page + - [#1386](https://github.com/c9s/bbgo/pull/1386): REFACTOR: [wall] refactor wall strategy with common.Strategy + - [#1382](https://github.com/c9s/bbgo/pull/1382): REFACTOR: [bitget] add rate limiter for account, ticker + - [#1384](https://github.com/c9s/bbgo/pull/1384): CHORE: minor improvements on backtest cmd + - [#1381](https://github.com/c9s/bbgo/pull/1381): DOC: grammatical errors in the README.md + - [#1377](https://github.com/c9s/bbgo/pull/1377): REFACTOR: [rebalance] submit one order at a time + - [#1378](https://github.com/c9s/bbgo/pull/1378): REFACTOR: [bitget] get symbol api + - [#1375](https://github.com/c9s/bbgo/pull/1375): DOC: grammatical error in the code_of_conduct file + - [#1374](https://github.com/c9s/bbgo/pull/1374): FIX: retry to get open orders only for 5 times and do not sync orders… + - [#1368](https://github.com/c9s/bbgo/pull/1368): FEATURE: merge grid recover and active orders recover logic + - [#1367](https://github.com/c9s/bbgo/pull/1367): DOC: fix typos in doc/development + - [#1372](https://github.com/c9s/bbgo/pull/1372): FIX: [bybit][kucoin] fix negative volume, price precision + - [#1373](https://github.com/c9s/bbgo/pull/1373): FEATURE: [xalign] adjust quantity by max amount + - [#1363](https://github.com/c9s/bbgo/pull/1363): FEATURE: [bitget] support ping/pong + - [#1370](https://github.com/c9s/bbgo/pull/1370): REFACTOR: [stream] move ping into stream level + - [#1361](https://github.com/c9s/bbgo/pull/1361): FEATURE: prepare query trades funtion for new recover + - [#1365](https://github.com/c9s/bbgo/pull/1365): FEATURE: [batch] add jumpIfEmpty opts to closed order batch query + - [#1364](https://github.com/c9s/bbgo/pull/1364): FEATURE: [batch] add a jumpIfEmpty to batch trade option + - [#1362](https://github.com/c9s/bbgo/pull/1362): DOC: Modified README.md file's language. + - [#1360](https://github.com/c9s/bbgo/pull/1360): DOC: Update CONTRIBUTING.md + - [#1351](https://github.com/c9s/bbgo/pull/1351): DOC: Update README.md + - [#1355](https://github.com/c9s/bbgo/pull/1355): REFACTOR: rename file and variable + - [#1358](https://github.com/c9s/bbgo/pull/1358): MINOR: [indicator] remove zero padding from RMA + - [#1357](https://github.com/c9s/bbgo/pull/1357): FIX: Fix duplicate RMA values and add test cases + - [#1356](https://github.com/c9s/bbgo/pull/1356): FIX: fix rma zero value issue + - [#1350](https://github.com/c9s/bbgo/pull/1350): FEATURE: [grid2] twin orderbook + - [#1353](https://github.com/c9s/bbgo/pull/1353): CHORE: go: update requestgen to v1.3.5 + - [#1349](https://github.com/c9s/bbgo/pull/1349): MINOR: remove profit entries from profit stats + - [#1352](https://github.com/c9s/bbgo/pull/1352): DOC: Fixed a typo in README.md + - [#1347](https://github.com/c9s/bbgo/pull/1347): FEATURE: [bitget] support market trade stream + - [#1344](https://github.com/c9s/bbgo/pull/1344): FEATURE: [bitget] support book stream on bitget + - [#1280](https://github.com/c9s/bbgo/pull/1280): FEATURE: [bitget] integrate QueryMarkets, QueryTicker and QueryAccount api + - [#1346](https://github.com/c9s/bbgo/pull/1346): FIX: [xnav] skip public only session + - [#1345](https://github.com/c9s/bbgo/pull/1345): FIX: [bbgo] check symbol length for injection + - [#1343](https://github.com/c9s/bbgo/pull/1343): FIX: [max] remove outdated margin fields + - [#1328](https://github.com/c9s/bbgo/pull/1328): FEATURE: recover active orders with open orders periodically + - [#1341](https://github.com/c9s/bbgo/pull/1341): REFACTOR: [random] remove adjustQuantity from config + - [#1342](https://github.com/c9s/bbgo/pull/1342): CHORE: make rightWindow possible to be set as zero + - [#1339](https://github.com/c9s/bbgo/pull/1339): FEATURE: [BYBIT] support order book depth 200 on bybit + - [#1340](https://github.com/c9s/bbgo/pull/1340): CHORE: update xfixedmaker config for backtest + - [#1335](https://github.com/c9s/bbgo/pull/1335): FEATURE: add custom private channel support to max + - [#1338](https://github.com/c9s/bbgo/pull/1338): FIX: [grid2] set max retries to 5 + - [#1337](https://github.com/c9s/bbgo/pull/1337): REFACTOR: rename randomtrader to random + - [#1327](https://github.com/c9s/bbgo/pull/1327): FIX: Fix duplicate orders caused by position risk control + - [#1331](https://github.com/c9s/bbgo/pull/1331): FEATURE: add xfixedmaker strategy + - [#1336](https://github.com/c9s/bbgo/pull/1336): FEATURE: add randomtrader strategy + - [#1332](https://github.com/c9s/bbgo/pull/1332): FEATURE: add supported interval for okex + - [#1232](https://github.com/c9s/bbgo/pull/1232): FEATURE: add forceOrder api for binance to show liquid info + - [#1334](https://github.com/c9s/bbgo/pull/1334): CHORE: [maxapi] change default http transport settings + - [#1330](https://github.com/c9s/bbgo/pull/1330): REFACTOR: Make fixedmaker simpler + - [#1312](https://github.com/c9s/bbgo/pull/1312): FEATURE: add QueryClosedOrders() and QueryTrades() for okex From 80ea46ca92bac25840bf759e832f75ebca764521 Mon Sep 17 00:00:00 2001 From: chiahung Date: Thu, 9 Nov 2023 16:19:53 +0800 Subject: [PATCH 168/422] FEATURE: use rest quote to place the last order when opening grid --- pkg/strategy/grid2/strategy.go | 26 +++++++++++++++++++++----- pkg/types/backtest_stream.go | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 620d91ce1f..f463b4ad62 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1362,7 +1362,11 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin ClientOrderID: s.newClientOrderID(), }) quoteQuantity := quantity.Mul(nextPrice) - usedQuote = usedQuote.Add(quoteQuantity) + + // because the precision issue, we need to round up quote quantity and add it into used quote + // e.g. quote we calculate : 8888.85, but it may lock 8888.9 due to their precision. + roundUpQuoteQuantity := quoteQuantity.Round(s.Market.VolumePrecision, fixedpoint.Up) + usedQuote = usedQuote.Add(roundUpQuoteQuantity) } } else { // if price spread is not enabled, and we have already placed a sell order index on the top of this price, @@ -1378,9 +1382,21 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin quoteQuantity := quantity.Mul(price) - if usedQuote.Add(quoteQuantity).Compare(totalQuote) > 0 { - s.logger.Warnf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) - continue + // because the precision issue, we need to round up quote quantity and add it into used quote + // e.g. quote we calculate : 8888.85, but it may lock 8888.9 due to their precision. + roundUpQuoteQuantity := quoteQuantity.Round(s.Market.VolumePrecision, fixedpoint.Up) + if usedQuote.Add(roundUpQuoteQuantity).Compare(totalQuote) > 0 { + if i > 0 { + s.logger.Errorf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) + return nil, fmt.Errorf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) + } else { + restQuote := totalQuote.Sub(usedQuote) + quantity = restQuote.Div(price).Round(s.Market.VolumePrecision, fixedpoint.Down) + if s.Market.MinQuantity.Compare(quantity) > 0 { + s.logger.Errorf("the round down quantity (%s) is less than min quantity (%s), we cannot place this order", quantity, s.Market.MinQuantity) + return nil, fmt.Errorf("the round down quantity (%s) is less than min quantity (%s), we cannot place this order", quantity, s.Market.MinQuantity) + } + } } submitOrders = append(submitOrders, types.SubmitOrder{ @@ -1395,7 +1411,7 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin GroupID: s.OrderGroupID, ClientOrderID: s.newClientOrderID(), }) - usedQuote = usedQuote.Add(quoteQuantity) + usedQuote = usedQuote.Add(roundUpQuoteQuantity) } } diff --git a/pkg/types/backtest_stream.go b/pkg/types/backtest_stream.go index ee46d31fcc..dadd0c1478 100644 --- a/pkg/types/backtest_stream.go +++ b/pkg/types/backtest_stream.go @@ -11,6 +11,7 @@ type BacktestStream struct { func (s *BacktestStream) Connect(ctx context.Context) error { s.EmitConnect() s.EmitStart() + s.EmitAuth() return nil } From c8c9659dd17713d90ce722f7fc3a893840049b8b Mon Sep 17 00:00:00 2001 From: chiahung Date: Thu, 9 Nov 2023 17:17:59 +0800 Subject: [PATCH 169/422] use PricePrecision for quote round up --- pkg/strategy/grid2/strategy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index f463b4ad62..cc08dc22f1 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1365,7 +1365,7 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin // because the precision issue, we need to round up quote quantity and add it into used quote // e.g. quote we calculate : 8888.85, but it may lock 8888.9 due to their precision. - roundUpQuoteQuantity := quoteQuantity.Round(s.Market.VolumePrecision, fixedpoint.Up) + roundUpQuoteQuantity := quoteQuantity.Round(s.Market.PricePrecision, fixedpoint.Up) usedQuote = usedQuote.Add(roundUpQuoteQuantity) } } else { @@ -1384,7 +1384,7 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin // because the precision issue, we need to round up quote quantity and add it into used quote // e.g. quote we calculate : 8888.85, but it may lock 8888.9 due to their precision. - roundUpQuoteQuantity := quoteQuantity.Round(s.Market.VolumePrecision, fixedpoint.Up) + roundUpQuoteQuantity := quoteQuantity.Round(s.Market.PricePrecision, fixedpoint.Up) if usedQuote.Add(roundUpQuoteQuantity).Compare(totalQuote) > 0 { if i > 0 { s.logger.Errorf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) From cb5e305fedafbc9a9855011c9af8a09b43ce79a8 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 6 Nov 2023 11:51:16 +0800 Subject: [PATCH 170/422] pkg/exchange: support submit order --- .../bitget/bitgetapi/v2/client_test.go | 15 +- .../bitgetapi/v2/place_order_request.go | 29 ++ .../v2/place_order_request_requestgen.go | 251 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 26 ++ pkg/exchange/bitget/convert_test.go | 26 ++ pkg/exchange/bitget/exchange.go | 113 +++++++- 6 files changed, 456 insertions(+), 4 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/place_order_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 3d56735e9e..7b178a6304 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -2,11 +2,12 @@ package bitgetapi import ( "context" - "github.com/stretchr/testify/assert" "os" "strconv" "testing" + "github.com/stretchr/testify/assert" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/testutil" ) @@ -45,4 +46,16 @@ func TestClient(t *testing.T) { t.Logf("place order resp: %+v", req) }) + + t.Run("PlaceOrderRequest", func(t *testing.T) { + req, err := client.NewPlaceOrderRequest().Symbol("APEUSDT").OrderType(OrderTypeLimit). + Side(SideTypeSell). + Price("2"). + Size("5"). + Force(OrderForceGTC). + Do(context.Background()) + assert.NoError(t, err) + + t.Logf("place order resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go b/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go new file mode 100644 index 0000000000..e36679decd --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/place_order_request.go @@ -0,0 +1,29 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" +) + +type PlaceOrderResponse struct { + OrderId string `json:"orderId"` + ClientOrderId string `json:"clientOrderId"` +} + +//go:generate PostRequest -url "/api/v2/spot/trade/place-order" -type PlaceOrderRequest -responseDataType .PlaceOrderResponse +type PlaceOrderRequest struct { + client requestgen.AuthenticatedAPIClient + symbol string `param:"symbol"` + orderType OrderType `param:"orderType"` + side SideType `param:"side"` + force OrderForce `param:"force"` + price *string `param:"price"` + size string `param:"size"` + clientOrderId *string `param:"clientOid"` +} + +func (c *Client) NewPlaceOrderRequest() *PlaceOrderRequest { + return &PlaceOrderRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go new file mode 100644 index 0000000000..a5c8c9d87c --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/place_order_request_requestgen.go @@ -0,0 +1,251 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/place-order -type PlaceOrderRequest -responseDataType .PlaceOrderResponse"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (p *PlaceOrderRequest) Symbol(symbol string) *PlaceOrderRequest { + p.symbol = symbol + return p +} + +func (p *PlaceOrderRequest) OrderType(orderType OrderType) *PlaceOrderRequest { + p.orderType = orderType + return p +} + +func (p *PlaceOrderRequest) Side(side SideType) *PlaceOrderRequest { + p.side = side + return p +} + +func (p *PlaceOrderRequest) Force(force OrderForce) *PlaceOrderRequest { + p.force = force + return p +} + +func (p *PlaceOrderRequest) Price(price string) *PlaceOrderRequest { + p.price = &price + return p +} + +func (p *PlaceOrderRequest) Size(size string) *PlaceOrderRequest { + p.size = size + return p +} + +func (p *PlaceOrderRequest) ClientOrderId(clientOrderId string) *PlaceOrderRequest { + p.clientOrderId = &clientOrderId + return p +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (p *PlaceOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (p *PlaceOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := p.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check orderType field -> json key orderType + orderType := p.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeLimit, OrderTypeMarket: + params["orderType"] = orderType + + default: + return nil, fmt.Errorf("orderType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["orderType"] = orderType + // check side field -> json key side + side := p.side + + // TEMPLATE check-valid-values + switch side { + case SideTypeBuy, SideTypeSell: + params["side"] = side + + default: + return nil, fmt.Errorf("side value %v is invalid", side) + + } + // END TEMPLATE check-valid-values + + // assign parameter of side + params["side"] = side + // check force field -> json key force + force := p.force + + // TEMPLATE check-valid-values + switch force { + case OrderForceGTC, OrderForcePostOnly, OrderForceFOK, OrderForceIOC: + params["force"] = force + + default: + return nil, fmt.Errorf("force value %v is invalid", force) + + } + // END TEMPLATE check-valid-values + + // assign parameter of force + params["force"] = force + // check price field -> json key price + if p.price != nil { + price := *p.price + + // assign parameter of price + params["price"] = price + } else { + } + // check size field -> json key size + size := p.size + + // assign parameter of size + params["size"] = size + // check clientOrderId field -> json key clientOid + if p.clientOrderId != nil { + clientOrderId := *p.clientOrderId + + // assign parameter of clientOrderId + params["clientOid"] = clientOrderId + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (p *PlaceOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := p.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if p.isVarSlice(_v) { + p.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (p *PlaceOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := p.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (p *PlaceOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (p *PlaceOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (p *PlaceOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (p *PlaceOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (p *PlaceOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := p.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (p *PlaceOrderRequest) Do(ctx context.Context) (*PlaceOrderResponse, error) { + + params, err := p.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + apiURL := "/api/v2/spot/trade/place-order" + + req, err := p.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := p.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + var data PlaceOrderResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 24785a81f1..a4026e1500 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -243,3 +243,29 @@ func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoin return fixedpoint.Zero, fmt.Errorf("failed to execute market buy quantity due to unexpected order status %s ", orderStatus) } } + +func toLocalOrderType(orderType types.OrderType) (v2.OrderType, error) { + switch orderType { + case types.OrderTypeLimit: + return v2.OrderTypeLimit, nil + + case types.OrderTypeMarket: + return v2.OrderTypeMarket, nil + + default: + return "", fmt.Errorf("order type %s not supported", orderType) + } +} + +func toLocalSide(side types.SideType) (v2.SideType, error) { + switch side { + case types.SideTypeSell: + return v2.SideTypeSell, nil + + case types.SideTypeBuy: + return v2.SideTypeBuy, nil + + default: + return "", fmt.Errorf("side type %s not supported", side) + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 1747980748..705eb88b8e 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -465,3 +465,29 @@ func Test_processMarketBuyQuantity(t *testing.T) { assert.ErrorContains(err, "xxx") }) } + +func Test_toLocalOrderType(t *testing.T) { + orderType, err := toLocalOrderType(types.OrderTypeLimit) + assert.NoError(t, err) + assert.Equal(t, v2.OrderTypeLimit, orderType) + + orderType, err = toLocalOrderType(types.OrderTypeMarket) + assert.NoError(t, err) + assert.Equal(t, v2.OrderTypeMarket, orderType) + + _, err = toLocalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toLocalSide(t *testing.T) { + orderType, err := toLocalSide(types.SideTypeSell) + assert.NoError(t, err) + assert.Equal(t, v2.SideTypeSell, orderType) + + orderType, err = toLocalSide(types.SideTypeBuy) + assert.NoError(t, err) + assert.Equal(t, v2.SideTypeBuy, orderType) + + _, err = toLocalOrderType("xxx") + assert.ErrorContains(t, err, "xxx") +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 298054e0ab..256cc74ef7 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -42,6 +42,8 @@ var ( queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) // closedQueryOrdersRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Get-History-Orders closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) + // submitOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order + submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -183,9 +185,114 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, return bals, nil } +// SubmitOrder submits an order. +// +// Remark: +// 1. We support only GTC for time-in-force, because the response from queryOrder does not include time-in-force information. +// 2. For market buy orders, the size unit is quote currency, whereas the unit for order.Quantity is in base currency. +// Therefore, we need to calculate the equivalent quote currency amount based on the ticker data. +// +// Note that there is a bug in Bitget where you can place a market order with the 'post_only' option successfully, +// which should not be possible. The issue has been reported. func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { - // TODO implement me - panic("implement me") + if len(order.Market.Symbol) == 0 { + return nil, fmt.Errorf("order.Market.Symbol is required: %+v", order) + } + + req := e.v2Client.NewPlaceOrderRequest() + req.Symbol(order.Market.Symbol) + + // set order type + orderType, err := toLocalOrderType(order.Type) + if err != nil { + return nil, err + } + req.OrderType(orderType) + + // set side + side, err := toLocalSide(order.Side) + if err != nil { + return nil, err + } + req.Side(side) + + // set quantity + qty := order.Quantity + // if the order is market buy, the quantity is quote coin, instead of base coin. so we need to convert it. + if order.Type == types.OrderTypeMarket && order.Side == types.SideTypeBuy { + ticker, err := e.QueryTicker(ctx, order.Market.Symbol) + if err != nil { + return nil, err + } + qty = order.Quantity.Mul(ticker.Buy) + } + req.Size(order.Market.FormatQuantity(qty)) + + // we support only GTC/PostOnly, this is because: + // 1. We support only SPOT trading. + // 2. The query oepn/closed order does not including the `force` in SPOT. + // If we support FOK/IOC, but you can't query them, that would be unreasonable. + // The other case to consider is 'PostOnly', which is a trade-off because we want to support 'xmaker'. + if order.TimeInForce != types.TimeInForceGTC { + return nil, fmt.Errorf("time-in-force %s not supported", order.TimeInForce) + } + req.Force(v2.OrderForceGTC) + // set price + if order.Type == types.OrderTypeLimit || order.Type == types.OrderTypeLimitMaker { + req.Price(order.Market.FormatPrice(order.Price)) + + if order.Type == types.OrderTypeLimitMaker { + req.Force(v2.OrderForcePostOnly) + } + } + + // set client order id + if len(order.ClientOrderID) > maxOrderIdLen { + return nil, fmt.Errorf("unexpected length of order id, got: %d", len(order.ClientOrderID)) + } + if len(order.ClientOrderID) > 0 { + req.ClientOrderId(order.ClientOrderID) + } + + if err := submitOrdersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("place order rate limiter wait error: %w", err) + } + res, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to place order, order: %#v, err: %w", order, err) + } + + if len(res.OrderId) == 0 || (len(order.ClientOrderID) != 0 && res.ClientOrderId != order.ClientOrderID) { + return nil, fmt.Errorf("unexpected order id, resp: %#v, order: %#v", res, order) + } + + orderId := res.OrderId + ordersResp, err := e.v2Client.NewGetUnfilledOrdersRequest().OrderId(orderId).Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query open order by order id: %s, err: %w", orderId, err) + } + + switch len(ordersResp) { + case 0: + // The market order will be executed immediately, so we cannot retrieve it through the NewGetUnfilledOrdersRequest API. + // Try to get the order from the NewGetHistoryOrdersRequest API. + ordersResp, err := e.v2Client.NewGetHistoryOrdersRequest().OrderId(orderId).Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query history order by order id: %s, err: %w", orderId, err) + } + + if len(ordersResp) != 1 { + return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) + } + + return toGlobalOrder(ordersResp[0]) + + case 1: + return unfilledOrderToGlobalOrder(ordersResp[0]) + + default: + return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) + } } func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { @@ -238,7 +345,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // ** Since and Until cannot exceed 90 days. ** // ** Since from the last 90 days can be queried. ** func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { - if since.Sub(time.Now()) > queryMaxDuration { + if time.Since(since) > queryMaxDuration { return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", since) } if until.Before(since) { From a26b1582308c22a15c50d67e4cc2ef3b8405bbcf Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 9 Nov 2023 10:35:53 +0800 Subject: [PATCH 171/422] pkg/exchange: support query trades --- .../bitget/bitgetapi/v2/client_test.go | 7 + .../bitget/bitgetapi/v2/get_trade_fills.go | 70 ++++++ .../v2/get_trade_fills_request_requestgen.go | 219 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 60 +++++ pkg/exchange/bitget/convert_test.go | 88 +++++++ pkg/exchange/bitget/exchange.go | 69 ++++++ 6 files changed, 513 insertions(+) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 7b178a6304..2e22f560df 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -58,4 +58,11 @@ func TestClient(t *testing.T) { t.Logf("place order resp: %+v", req) }) + + t.Run("GetTradeFillsRequest", func(t *testing.T) { + req, err := client.NewGetTradeFillsRequest().Symbol("APEUSDT").Do(ctx) + assert.NoError(t, err) + + t.Logf("get trade fills resp: %+v", req) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go new file mode 100644 index 0000000000..358bbda278 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go @@ -0,0 +1,70 @@ +package bitgetapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type TradeScope string + +const ( + TradeMaker TradeScope = "maker" + TradeTaker TradeScope = "taker" +) + +type DiscountStatus string + +const ( + DiscountYes DiscountStatus = "yes" + DiscountNo DiscountStatus = "no" +) + +type TradeFee struct { + // Discount or not + Deduction DiscountStatus `json:"deduction"` + // Transaction fee coin + FeeCoin string `json:"feeCoin"` + // Total transaction fee discount + TotalDeductionFee fixedpoint.Value `json:"totalDeductionFee"` + // Total transaction fee + TotalFee fixedpoint.Value `json:"totalFee"` +} + +type Trade struct { + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + OrderId types.StrInt64 `json:"orderId"` + TradeId types.StrInt64 `json:"tradeId"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + Size fixedpoint.Value `json:"size"` + Amount fixedpoint.Value `json:"amount"` + FeeDetail TradeFee `json:"feeDetail"` + TradeScope TradeScope `json:"tradeScope"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` +} + +//go:generate GetRequest -url "/api/v2/spot/trade/fills" -type GetTradeFillsRequest -responseDataType []Trade +type GetTradeFillsRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol string `param:"symbol,query"` + // Limit number default 100 max 100 + limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. + idLessThan *string `param:"idLessThan,query"` + startTime *int64 `param:"startTime,query"` + endTime *int64 `param:"endTime,query"` + orderId *string `param:"orderId,query"` +} + +func (s *Client) NewGetTradeFillsRequest() *GetTradeFillsRequest { + return &GetTradeFillsRequest{client: s.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go new file mode 100644 index 0000000000..2f19cf3786 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go @@ -0,0 +1,219 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/fills -type GetTradeFillsRequest -responseDataType []Trade"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (s *GetTradeFillsRequest) Symbol(symbol string) *GetTradeFillsRequest { + s.symbol = symbol + return s +} + +func (s *GetTradeFillsRequest) Limit(limit string) *GetTradeFillsRequest { + s.limit = &limit + return s +} + +func (s *GetTradeFillsRequest) IdLessThan(idLessThan string) *GetTradeFillsRequest { + s.idLessThan = &idLessThan + return s +} + +func (s *GetTradeFillsRequest) StartTime(startTime int64) *GetTradeFillsRequest { + s.startTime = &startTime + return s +} + +func (s *GetTradeFillsRequest) EndTime(endTime int64) *GetTradeFillsRequest { + s.endTime = &endTime + return s +} + +func (s *GetTradeFillsRequest) OrderId(orderId string) *GetTradeFillsRequest { + s.orderId = &orderId + return s +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (s *GetTradeFillsRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := s.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check limit field -> json key limit + if s.limit != nil { + limit := *s.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + // check idLessThan field -> json key idLessThan + if s.idLessThan != nil { + idLessThan := *s.idLessThan + + // assign parameter of idLessThan + params["idLessThan"] = idLessThan + } else { + } + // check startTime field -> json key startTime + if s.startTime != nil { + startTime := *s.startTime + + // assign parameter of startTime + params["startTime"] = startTime + } else { + } + // check endTime field -> json key endTime + if s.endTime != nil { + endTime := *s.endTime + + // assign parameter of endTime + params["endTime"] = endTime + } else { + } + // check orderId field -> json key orderId + if s.orderId != nil { + orderId := *s.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (s *GetTradeFillsRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (s *GetTradeFillsRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := s.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if s.isVarSlice(_v) { + s.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (s *GetTradeFillsRequest) GetParametersJSON() ([]byte, error) { + params, err := s.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (s *GetTradeFillsRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (s *GetTradeFillsRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (s *GetTradeFillsRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (s *GetTradeFillsRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (s *GetTradeFillsRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := s.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (s *GetTradeFillsRequest) Do(ctx context.Context) ([]Trade, error) { + + // no body params + var params interface{} + query, err := s.GetQueryParameters() + if err != nil { + return nil, err + } + + apiURL := "/api/v2/spot/trade/fills" + + req, err := s.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := s.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + var data []Trade + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a4026e1500..59f3357307 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -109,6 +109,66 @@ func toGlobalOrderStatus(status v2.OrderStatus) (types.OrderStatus, error) { } } +func isMaker(s v2.TradeScope) (bool, error) { + switch s { + case v2.TradeMaker: + return true, nil + + case v2.TradeTaker: + return false, nil + + default: + return false, fmt.Errorf("unexpected trade scope: %s", s) + } +} + +func isFeeDiscount(s v2.DiscountStatus) (bool, error) { + switch s { + case v2.DiscountYes: + return true, nil + + case v2.DiscountNo: + return false, nil + + default: + return false, fmt.Errorf("unexpected discount status: %s", s) + } +} + +func toGlobalTrade(trade v2.Trade) (*types.Trade, error) { + side, err := toGlobalSideType(trade.Side) + if err != nil { + return nil, err + } + + isMaker, err := isMaker(trade.TradeScope) + if err != nil { + return nil, err + } + + isDiscount, err := isFeeDiscount(trade.FeeDetail.Deduction) + if err != nil { + return nil, err + } + + return &types.Trade{ + ID: uint64(trade.TradeId), + OrderID: uint64(trade.OrderId), + Exchange: types.ExchangeBitget, + Price: trade.PriceAvg, + Quantity: trade.Size, + QuoteQuantity: trade.Amount, + Symbol: trade.Symbol, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: isMaker, + Time: types.Time(trade.CTime), + Fee: trade.FeeDetail.TotalFee.Abs(), + FeeCurrency: trade.FeeDetail.FeeCoin, + FeeDiscounted: isDiscount, + }, nil +} + // unfilledOrderToGlobalOrder convert the local order to global. // // Note that the quantity unit, according official document: Base coin when orderType=limit; Quote coin when orderType=market diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 705eb88b8e..470c63f311 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -491,3 +491,91 @@ func Test_toLocalSide(t *testing.T) { _, err = toLocalOrderType("xxx") assert.ErrorContains(t, err, "xxx") } + +func Test_isMaker(t *testing.T) { + isM, err := isMaker(v2.TradeTaker) + assert.NoError(t, err) + assert.False(t, isM) + + isM, err = isMaker(v2.TradeMaker) + assert.NoError(t, err) + assert.True(t, isM) + + _, err = isMaker("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_isFeeDiscount(t *testing.T) { + isDiscount, err := isFeeDiscount(v2.DiscountNo) + assert.NoError(t, err) + assert.False(t, isDiscount) + + isDiscount, err = isFeeDiscount(v2.DiscountYes) + assert.NoError(t, err) + assert.True(t, isDiscount) + + _, err = isFeeDiscount("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func Test_toGlobalTrade(t *testing.T) { + // { + // "userId":"8672173294", + // "symbol":"APEUSDT", + // "orderId":"1104337778433757184", + // "tradeId":"1104337778504044545", + // "orderType":"limit", + // "side":"sell", + // "priceAvg":"1.4001", + // "size":"5", + // "amount":"7.0005", + // "feeDetail":{ + // "deduction":"no", + // "feeCoin":"USDT", + // "totalDeductionFee":"", + // "totalFee":"-0.0070005" + // }, + // "tradeScope":"taker", + // "cTime":"1699020564676", + // "uTime":"1699020564687" + //} + trade := v2.Trade{ + UserId: types.StrInt64(8672173294), + Symbol: "APEUSDT", + OrderId: types.StrInt64(1104337778433757184), + TradeId: types.StrInt64(1104337778504044545), + OrderType: v2.OrderTypeLimit, + Side: v2.SideTypeSell, + PriceAvg: fixedpoint.NewFromFloat(1.4001), + Size: fixedpoint.NewFromFloat(5), + Amount: fixedpoint.NewFromFloat(7.0005), + FeeDetail: v2.TradeFee{ + Deduction: "no", + FeeCoin: "USDT", + TotalDeductionFee: fixedpoint.Zero, + TotalFee: fixedpoint.NewFromFloat(-0.0070005), + }, + TradeScope: v2.TradeTaker, + CTime: types.NewMillisecondTimestampFromInt(1699020564676), + UTime: types.NewMillisecondTimestampFromInt(1699020564687), + } + + res, err := toGlobalTrade(trade) + assert.NoError(t, err) + assert.Equal(t, &types.Trade{ + ID: uint64(1104337778504044545), + OrderID: uint64(1104337778433757184), + Exchange: types.ExchangeBitget, + Price: fixedpoint.NewFromFloat(1.4001), + Quantity: fixedpoint.NewFromFloat(5), + QuoteQuantity: fixedpoint.NewFromFloat(7.0005), + Symbol: "APEUSDT", + Side: types.SideTypeSell, + IsBuyer: false, + IsMaker: false, + Time: types.Time(types.NewMillisecondTimestampFromInt(1699020564676)), + Fee: fixedpoint.NewFromFloat(0.0070005), + FeeCurrency: "USDT", + FeeDiscounted: false, + }, res) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 256cc74ef7..4a5bf35ba5 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -2,6 +2,7 @@ package bitget import ( "context" + "errors" "fmt" "strconv" "time" @@ -44,6 +45,8 @@ var ( closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) // submitOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTradeRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Fills + queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -393,3 +396,69 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) erro // TODO implement me panic("implement me") } + +// QueryTrades queries fill trades. The trade of the response is in descending order. The time-based query are typically +// using (`CTime`) as the search criteria. +// If you need to retrieve all data, please utilize the function pkg/exchange/batch.TradeBatchQuery. +// +// ** StartTime is inclusive, EndTime is exclusive. If you use the EndTime, the StartTime is required. ** +// ** StartTime and EndTime cannot exceed 90 days. ** +func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { + if options.LastTradeID != 0 { + log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The trade of response is in descending order, so the last trade id not supported.") + } + + req := e.v2Client.NewGetTradeFillsRequest() + req.Symbol(symbol) + + if options.StartTime != nil { + if time.Since(*options.StartTime) > queryMaxDuration { + return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", options.StartTime) + } + req.StartTime(options.StartTime.UnixMilli()) + } + + if options.EndTime != nil { + if options.StartTime == nil { + return nil, errors.New("start time is required for query trades if you take end time") + } + if options.EndTime.Before(*options.StartTime) { + return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, *options.StartTime) + } + if options.EndTime.Sub(*options.StartTime) > queryMaxDuration { + return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", options.StartTime, options.EndTime) + } + req.EndTime(options.EndTime.UnixMilli()) + } + + limit := options.Limit + if limit > queryLimit || limit <= 0 { + log.Debugf("limtit is exceeded or zero, update to %d, got: %d", queryLimit, options.Limit) + limit = queryLimit + } + req.Limit(strconv.FormatInt(limit, 10)) + + if err := queryTradeRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("trade rate limiter wait error: %w", err) + } + response, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query trades, err: %w", err) + } + + var errs error + for _, trade := range response { + res, err := toGlobalTrade(trade) + if err != nil { + errs = multierr.Append(errs, err) + continue + } + trades = append(trades, *res) + } + + if errs != nil { + return nil, errs + } + + return trades, nil +} From 639947c8b76118a3431a61bc0726062e1efd1c50 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 9 Nov 2023 10:38:32 +0800 Subject: [PATCH 172/422] pkg/exchange: support cancel order --- .../bitgetapi/v2/cancel_order_request.go | 29 +++ .../v2/cancel_order_request_requestgen.go | 196 ++++++++++++++++++ .../bitget/bitgetapi/v2/client_test.go | 13 ++ pkg/exchange/bitget/exchange.go | 52 ++++- 4 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go new file mode 100644 index 0000000000..301212cd59 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request.go @@ -0,0 +1,29 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/types" +) + +type CancelOrder struct { + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` +} + +//go:generate PostRequest -url "/api/v2/spot/trade/cancel-order" -type CancelOrderRequest -responseDataType .CancelOrder +type CancelOrderRequest struct { + client requestgen.AuthenticatedAPIClient + + symbol string `param:"symbol"` + orderId *string `param:"orderId"` + clientOrderId *string `param:"clientOid"` +} + +func (c *Client) NewCancelOrderRequest() *CancelOrderRequest { + return &CancelOrderRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go new file mode 100644 index 0000000000..0be0e2ccb9 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/cancel_order_request_requestgen.go @@ -0,0 +1,196 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v2/spot/trade/cancel-order -type CancelOrderRequest -responseDataType .CancelOrder"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (c *CancelOrderRequest) Symbol(symbol string) *CancelOrderRequest { + c.symbol = symbol + return c +} + +func (c *CancelOrderRequest) OrderId(orderId string) *CancelOrderRequest { + c.orderId = &orderId + return c +} + +func (c *CancelOrderRequest) ClientOrderId(clientOrderId string) *CancelOrderRequest { + c.clientOrderId = &clientOrderId + return c +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (c *CancelOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (c *CancelOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := c.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check orderId field -> json key orderId + if c.orderId != nil { + orderId := *c.orderId + + // assign parameter of orderId + params["orderId"] = orderId + } else { + } + // check clientOrderId field -> json key clientOid + if c.clientOrderId != nil { + clientOrderId := *c.clientOrderId + + // assign parameter of clientOrderId + params["clientOid"] = clientOrderId + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (c *CancelOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := c.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if c.isVarSlice(_v) { + c.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (c *CancelOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := c.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (c *CancelOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (c *CancelOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (c *CancelOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (c *CancelOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (c *CancelOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := c.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (c *CancelOrderRequest) GetPath() string { + return "/api/v2/spot/trade/cancel-order" +} + +// Do generates the request object and send the request object to the API endpoint +func (c *CancelOrderRequest) Do(ctx context.Context) (*CancelOrder, error) { + + params, err := c.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + var apiURL string + + apiURL = c.GetPath() + + req, err := c.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := c.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data CancelOrder + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 2e22f560df..97b836f7fd 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -65,4 +65,17 @@ func TestClient(t *testing.T) { t.Logf("get trade fills resp: %+v", req) }) + + t.Run("CancelOrderRequest", func(t *testing.T) { + req, err := client.NewPlaceOrderRequest().Symbol("APEUSDT").OrderType(OrderTypeLimit). + Side(SideTypeSell). + Price("2"). + Size("5"). + Force(OrderForceGTC). + Do(context.Background()) + assert.NoError(t, err) + + resp, err := client.NewCancelOrderRequest().Symbol("APEUSDT").OrderId(req.OrderId).Do(ctx) + t.Logf("cancel order resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 4a5bf35ba5..05a10bb260 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -47,6 +47,8 @@ var ( submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) // queryTradeRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Fills queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // cancelOrderRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Cancel-Order + cancelOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) ) type Exchange struct { @@ -392,9 +394,53 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, return types.SortOrdersAscending(orders), nil } -func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) error { - // TODO implement me - panic("implement me") +func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (errs error) { + if len(orders) == 0 { + return nil + } + + for _, order := range orders { + req := e.client.NewCancelOrderRequest() + + reqId := "" + switch { + // use the OrderID first, then the ClientOrderID + case order.OrderID > 0: + req.OrderId(strconv.FormatUint(order.OrderID, 10)) + reqId = strconv.FormatUint(order.OrderID, 10) + + case len(order.ClientOrderID) != 0: + req.ClientOrderId(order.ClientOrderID) + reqId = order.ClientOrderID + + default: + errs = multierr.Append( + errs, + fmt.Errorf("the order uuid and client order id are empty, order: %#v", order), + ) + continue + } + + req.Symbol(order.Market.Symbol) + + if err := cancelOrderRateLimiter.Wait(ctx); err != nil { + errs = multierr.Append(errs, fmt.Errorf("cancel order rate limiter wait, order id: %s, error: %w", order.ClientOrderID, err)) + continue + } + res, err := req.Do(ctx) + if err != nil { + errs = multierr.Append(errs, fmt.Errorf("failed to cancel order id: %s, err: %w", order.ClientOrderID, err)) + continue + } + + // sanity check + if res.OrderId != reqId && res.ClientOrderId != reqId { + errs = multierr.Append(errs, fmt.Errorf("order id mismatch, exp: %s, respOrderId: %s, respClientOrderId: %s", reqId, res.OrderId, res.ClientOrderId)) + continue + } + } + + return errs } // QueryTrades queries fill trades. The trade of the response is in descending order. The time-based query are typically From 6c96d12d99e77178f83e5fd7bc233c57b0007f27 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 10 Nov 2023 21:56:18 +0800 Subject: [PATCH 173/422] pkg/exchange: add login method --- pkg/exchange/bitget/bitgetapi/client.go | 6 ++--- pkg/exchange/bitget/bitgetapi/v2/client.go | 4 +++ pkg/exchange/bitget/exchange.go | 3 +-- pkg/exchange/bitget/stream.go | 31 +++++++++++++++++++--- pkg/exchange/bitget/stream_test.go | 12 ++++++++- pkg/exchange/bitget/types.go | 8 +++++- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/client.go b/pkg/exchange/bitget/bitgetapi/client.go index 19b145d5d7..823f3a7c08 100644 --- a/pkg/exchange/bitget/bitgetapi/client.go +++ b/pkg/exchange/bitget/bitgetapi/client.go @@ -76,7 +76,7 @@ func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL } // See https://bitgetlimited.github.io/apidoc/en/spot/#signature - // sign( + // Sign( // timestamp + // method.toUpperCase() + // requestPath + "?" + queryString + @@ -94,7 +94,7 @@ func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL } signKey := timestamp + strings.ToUpper(method) + path + string(body) - signature := sign(signKey, c.secret) + signature := Sign(signKey, c.secret) req, err := http.NewRequestWithContext(ctx, method, pathURL.String(), bytes.NewReader(body)) if err != nil { @@ -110,7 +110,7 @@ func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL return req, nil } -func sign(payload string, secret string) string { +func Sign(payload string, secret string) string { var sig = hmac.New(sha256.New, []byte(secret)) _, err := sig.Write([]byte(payload)) if err != nil { diff --git a/pkg/exchange/bitget/bitgetapi/v2/client.go b/pkg/exchange/bitget/bitgetapi/v2/client.go index 3a2b2204d5..d15cd889bc 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client.go @@ -6,6 +6,10 @@ import ( "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" ) +const ( + PrivateWebSocketURL = "wss://ws.bitget.com/v2/ws/private" +) + type APIResponse = bitgetapi.APIResponse type Client struct { diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 05a10bb260..988b626d2b 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -83,8 +83,7 @@ func (e *Exchange) PlatformFeeCurrency() string { } func (e *Exchange) NewStream() types.Stream { - // TODO implement me - panic("implement me") + return NewStream(e.key, e.secret, e.passphrase) } func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 039c65127b..4636be4dbf 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -5,10 +5,14 @@ import ( "context" "encoding/json" "fmt" - "github.com/gorilla/websocket" + "strconv" "strings" + "time" + + "github.com/gorilla/websocket" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/types" ) @@ -21,6 +25,7 @@ var ( type Stream struct { types.StandardStream + key, secret, passphrase string bookEventCallbacks []func(o BookEvent) marketTradeEventCallbacks []func(o MarketTradeEvent) KLineEventCallbacks []func(o KLineEvent) @@ -28,10 +33,13 @@ type Stream struct { lastCandle map[string]types.KLine } -func NewStream() *Stream { +func NewStream(key, secret, passphrase string) *Stream { stream := &Stream{ StandardStream: types.NewStandardStream(), lastCandle: map[string]types.KLine{}, + key: key, + secret: secret, + passphrase: passphrase, } stream.SetEndpointCreator(stream.createEndpoint) @@ -89,7 +97,7 @@ func (s *Stream) createEndpoint(_ context.Context) (string, error) { if s.PublicOnly { url = bitgetapi.PublicWebSocketURL } else { - url = bitgetapi.PrivateWebSocketURL + url = v2.PrivateWebSocketURL } return url, nil } @@ -123,7 +131,22 @@ func (s *Stream) handlerConnect() { // errors are handled in the syncSubscriptions, so they are skipped here. _ = s.syncSubscriptions(WsEventSubscribe) } else { - log.Error("*** PRIVATE API NOT IMPLEMENTED ***") + timestamp := strconv.FormatInt(time.Now().Unix(), 10) + + if err := s.Conn.WriteJSON(WsOp{ + Op: WsEventLogin, + Args: []WsArg{ + { + ApiKey: s.key, + Passphrase: s.passphrase, + Timestamp: timestamp, + Sign: bitgetapi.Sign(fmt.Sprintf("%sGET/user/verify", timestamp), s.secret), + }, + }, + }); err != nil { + log.WithError(err).Error("failed to auth request") + return + } } } diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index b33e6afa20..d05cbd0371 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -19,7 +19,9 @@ func getTestClientOrSkip(t *testing.T) *Stream { t.Skip("skip test for CI") } - return NewStream() + return NewStream(os.Getenv("BITGET_API_KEY"), + os.Getenv("BITGET_API_SECRET"), + os.Getenv("BITGET_API_PASSPHRASE")) } func TestStream(t *testing.T) { @@ -122,6 +124,14 @@ func TestStream(t *testing.T) { <-c }) + t.Run("private test", func(t *testing.T) { + err := s.Connect(context.Background()) + assert.NoError(t, err) + + c := make(chan struct{}) + <-c + }) + } func TestStream_parseWebSocketEvent(t *testing.T) { diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index a1107cad66..f6fbfa06ce 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -34,6 +34,11 @@ type WsArg struct { Channel ChannelType `json:"channel"` // InstId Instrument ID. e.q. BTCUSDT, ETHUSDT InstId string `json:"instId"` + + ApiKey string `json:"apiKey"` + Passphrase string `json:"passphrase"` + Timestamp string `json:"timestamp"` + Sign string `json:"sign"` } type WsEventType string @@ -41,6 +46,7 @@ type WsEventType string const ( WsEventSubscribe WsEventType = "subscribe" WsEventUnsubscribe WsEventType = "unsubscribe" + WsEventLogin WsEventType = "login" WsEventError WsEventType = "error" ) @@ -76,7 +82,7 @@ func (w *WsEvent) IsValid() error { case WsEventError: return fmt.Errorf("websocket request error, op: %s, code: %d, msg: %s", w.Op, w.Code, w.Msg) - case WsEventSubscribe, WsEventUnsubscribe: + case WsEventSubscribe, WsEventUnsubscribe, WsEventLogin: // Actually, this code is unnecessary because the events are either `Subscribe` or `Unsubscribe`, But to avoid bugs // in the exchange, we still check. if w.Code != 0 || len(w.Msg) != 0 { From f49b14ac45f6d23708e547baa8216a7f3994ae77 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 10 Nov 2023 22:35:39 +0800 Subject: [PATCH 174/422] pkg/exchange: add balance event --- pkg/exchange/bitget/convert.go | 12 ++++++ pkg/exchange/bitget/convert_test.go | 19 +++++++++ pkg/exchange/bitget/stream.go | 51 +++++++++++++++++++++++++ pkg/exchange/bitget/stream_callbacks.go | 10 +++++ pkg/exchange/bitget/stream_test.go | 7 ++++ pkg/exchange/bitget/types.go | 29 +++++++++++++- 6 files changed, 127 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 59f3357307..b9d1a1ccbc 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -329,3 +329,15 @@ func toLocalSide(side types.SideType) (v2.SideType, error) { return "", fmt.Errorf("side type %s not supported", side) } } + +func toGlobalBalanceMap(balances []Balance) types.BalanceMap { + bm := types.BalanceMap{} + for _, obj := range balances { + bm[obj.Coin] = types.Balance{ + Currency: obj.Coin, + Available: obj.Available, + Locked: obj.Frozen.Add(obj.Locked), + } + } + return bm +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 470c63f311..19e759aa1b 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -579,3 +579,22 @@ func Test_toGlobalTrade(t *testing.T) { FeeDiscounted: false, }, res) } + +func Test_toGlobalBalanceMap(t *testing.T) { + assert.Equal(t, types.BalanceMap{ + "BTC": { + Currency: "BTC", + Available: fixedpoint.NewFromFloat(0.5), + Locked: fixedpoint.NewFromFloat(0.6 + 0.7), + }, + }, toGlobalBalanceMap([]Balance{ + { + Coin: "BTC", + Available: fixedpoint.NewFromFloat(0.5), + Frozen: fixedpoint.NewFromFloat(0.6), + Locked: fixedpoint.NewFromFloat(0.7), + LimitAvailable: fixedpoint.Zero, + UTime: types.NewMillisecondTimestampFromInt(1699020564676), + }, + })) +} diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 4636be4dbf..53e6964901 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -30,6 +30,8 @@ type Stream struct { marketTradeEventCallbacks []func(o MarketTradeEvent) KLineEventCallbacks []func(o KLineEvent) + accountEventCallbacks []func(e AccountEvent) + lastCandle map[string]types.KLine } @@ -51,6 +53,9 @@ func NewStream(key, secret, passphrase string) *Stream { stream.OnBookEvent(stream.handleBookEvent) stream.OnMarketTradeEvent(stream.handleMaretTradeEvent) stream.OnKLineEvent(stream.handleKLineEvent) + + stream.OnAuth(stream.handleAuth) + stream.OnAccountEvent(stream.handleAccountEvent) return stream } @@ -108,6 +113,9 @@ func (s *Stream) dispatchEvent(event interface{}) { if err := e.IsValid(); err != nil { log.Errorf("invalid event: %v", err) } + if e.IsAuthenticated() { + s.EmitAuth() + } case *BookEvent: s.EmitBookEvent(*e) @@ -118,6 +126,9 @@ func (s *Stream) dispatchEvent(event interface{}) { case *KLineEvent: s.EmitKLineEvent(*e) + case *AccountEvent: + s.EmitAccountEvent(*e) + case []byte: // We only handle the 'pong' case. Others are unexpected. if !bytes.Equal(e, pongBytes) { @@ -126,6 +137,22 @@ func (s *Stream) dispatchEvent(event interface{}) { } } +func (s *Stream) handleAuth() { + if err := s.Conn.WriteJSON(WsOp{ + Op: WsEventSubscribe, + Args: []WsArg{ + { + InstType: instSpV2, + Channel: ChannelAccount, + Coin: "default", // default all + }, + }, + }); err != nil { + log.WithError(err).Error("failed to send subscription request") + return + } +} + func (s *Stream) handlerConnect() { if s.PublicOnly { // errors are handled in the syncSubscriptions, so they are skipped here. @@ -236,6 +263,17 @@ func parseEvent(in []byte) (interface{}, error) { ch := event.Arg.Channel switch ch { + case ChannelAccount: + var acct AccountEvent + err = json.Unmarshal(event.Data, &acct.Balances) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into AccountEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + acct.actionType = event.Action + acct.instId = event.Arg.InstId + return &acct, nil + case ChannelOrderBook, ChannelOrderBook5, ChannelOrderBook15: var book BookEvent err = json.Unmarshal(event.Data, &book.Events) @@ -319,3 +357,16 @@ func (s *Stream) handleKLineEvent(k KLineEvent) { s.lastCandle[k.CacheKey()] = kLine } } + +func (s *Stream) handleAccountEvent(m AccountEvent) { + balanceMap := toGlobalBalanceMap(m.Balances) + if len(balanceMap) == 0 { + return + } + + if m.actionType == ActionTypeUpdate { + s.StandardStream.EmitBalanceUpdate(balanceMap) + return + } + s.StandardStream.EmitBalanceSnapshot(balanceMap) +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go index 82ef7beae3..44661171ec 100644 --- a/pkg/exchange/bitget/stream_callbacks.go +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -33,3 +33,13 @@ func (s *Stream) EmitKLineEvent(o KLineEvent) { cb(o) } } + +func (s *Stream) OnAccountEvent(cb func(e AccountEvent)) { + s.accountEventCallbacks = append(s.accountEventCallbacks, cb) +} + +func (s *Stream) EmitAccountEvent(e AccountEvent) { + for _, cb := range s.accountEventCallbacks { + cb(e) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index d05cbd0371..273941e7da 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -128,6 +128,13 @@ func TestStream(t *testing.T) { err := s.Connect(context.Background()) assert.NoError(t, err) + s.OnBalanceSnapshot(func(balances types.BalanceMap) { + t.Log("get balances", balances) + }) + s.OnBalanceUpdate(func(balances types.BalanceMap) { + t.Log("get update", balances) + }) + c := make(chan struct{}) <-c }) diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index f6fbfa06ce..0d5d5771ac 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -13,12 +13,14 @@ import ( type InstType string const ( - instSp InstType = "sp" + instSp InstType = "sp" + instSpV2 InstType = "SPOT" ) type ChannelType string const ( + ChannelAccount ChannelType = "account" // ChannelOrderBook snapshot and update might return less than 200 bids/asks as per symbol's orderbook various from // each other; The number of bids/asks is not a fixed value and may vary in the future ChannelOrderBook ChannelType = "books" @@ -34,6 +36,7 @@ type WsArg struct { Channel ChannelType `json:"channel"` // InstId Instrument ID. e.q. BTCUSDT, ETHUSDT InstId string `json:"instId"` + Coin string `json:"coin"` ApiKey string `json:"apiKey"` Passphrase string `json:"passphrase"` @@ -95,6 +98,10 @@ func (w *WsEvent) IsValid() error { } } +func (w *WsEvent) IsAuthenticated() bool { + return w.Event == WsEventLogin && w.Code == 0 +} + type ActionType string const ( @@ -398,3 +405,23 @@ func (k KLineEvent) CacheKey() string { // e.q: candle5m.BTCUSDT return fmt.Sprintf("%s.%s", k.channel, k.instId) } + +type Balance struct { + Coin string `json:"coin"` + Available fixedpoint.Value `json:"available"` + // Amount of frozen assets Usually frozen when the order is placed + Frozen fixedpoint.Value `json:"frozen"` + // Amount of locked assets Locked assests required to become a fiat merchants, etc. + Locked fixedpoint.Value `json:"locked"` + // Restricted availability For spot copy trading + LimitAvailable fixedpoint.Value `json:"limitAvailable"` + UTime types.MillisecondTimestamp `json:"uTime"` +} + +type AccountEvent struct { + Balances []Balance + + // internal use + actionType ActionType + instId string +} From b28b5e409791a061a2c8ed44a5e70a026d4b555b Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 11 Nov 2023 07:42:29 +0800 Subject: [PATCH 175/422] bbgo: add environment config for disabling some klines defaults --- pkg/bbgo/config.go | 7 +++++ pkg/bbgo/environment.go | 4 +-- pkg/bbgo/session.go | 66 ++++++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index 4cbc230489..591176484b 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -326,6 +326,11 @@ type ServiceConfig struct { GoogleSpreadSheetService *GoogleSpreadSheetServiceConfig `json:"googleSpreadSheet" yaml:"googleSpreadSheet"` } +type EnvironmentConfig struct { + DisableDefaultKLineSubscription bool `json:"disableDefaultKLineSubscription"` + DisableHistoryKLinePreload bool `json:"disableHistoryKLinePreload"` +} + type Config struct { Build *BuildConfig `json:"build,omitempty" yaml:"build,omitempty"` @@ -343,6 +348,8 @@ type Config struct { Service *ServiceConfig `json:"services,omitempty" yaml:"services,omitempty"` + Environment *EnvironmentConfig `json:"environment,omitempty" yaml:"environment,omitempty"` + Sessions map[string]*ExchangeSession `json:"sessions,omitempty" yaml:"sessions,omitempty"` RiskControls *RiskControls `json:"riskControls,omitempty" yaml:"riskControls,omitempty"` diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index 71ef121eb4..ce8b98fb50 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -108,13 +108,13 @@ type Environment struct { syncStatus SyncStatus syncConfig *SyncConfig - loggingConfig *LoggingConfig + loggingConfig *LoggingConfig + environmentConfig *EnvironmentConfig sessions map[string]*ExchangeSession } func NewEnvironment() *Environment { - now := time.Now() return &Environment{ // default trade scan time diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 3d2cc82e9c..101abd3f9c 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -472,49 +472,53 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ } if sub.Symbol == symbol { - klineSubscriptions[types.Interval(sub.Options.Interval)] = struct{}{} + klineSubscriptions[sub.Options.Interval] = struct{}{} } } } - // always subscribe the 1m kline so we can make sure the connection persists. - klineSubscriptions[minInterval] = struct{}{} + if !(environ.environmentConfig != nil && environ.environmentConfig.DisableDefaultKLineSubscription) { + // subscribe the 1m kline by default so we can make sure the connection persists. + klineSubscriptions[minInterval] = struct{}{} + } - for interval := range klineSubscriptions { - // avoid querying the last unclosed kline - endTime := environ.startTime - var i int64 - for i = 0; i < KLinePreloadLimit; i += 1000 { - var duration time.Duration = time.Duration(-i * int64(interval.Duration())) - e := endTime.Add(duration) + if !(environ.environmentConfig != nil && environ.environmentConfig.DisableHistoryKLinePreload) { + for interval := range klineSubscriptions { + // avoid querying the last unclosed kline + endTime := environ.startTime + var i int64 + for i = 0; i < KLinePreloadLimit; i += 1000 { + var duration time.Duration = time.Duration(-i * int64(interval.Duration())) + e := endTime.Add(duration) - kLines, err := session.Exchange.QueryKLines(ctx, symbol, interval, types.KLineQueryOptions{ - EndTime: &e, - Limit: 1000, // indicators need at least 100 - }) - if err != nil { - return err - } + kLines, err := session.Exchange.QueryKLines(ctx, symbol, interval, types.KLineQueryOptions{ + EndTime: &e, + Limit: 1000, // indicators need at least 100 + }) + if err != nil { + return err + } - if len(kLines) == 0 { - log.Warnf("no kline data for %s %s (end time <= %s)", symbol, interval, e) - continue - } + if len(kLines) == 0 { + log.Warnf("no kline data for %s %s (end time <= %s)", symbol, interval, e) + continue + } - // update last prices by the given kline - lastKLine := kLines[len(kLines)-1] - if interval == minInterval { - session.lastPrices[symbol] = lastKLine.Close - } + // update last prices by the given kline + lastKLine := kLines[len(kLines)-1] + if interval == minInterval { + session.lastPrices[symbol] = lastKLine.Close + } - for _, k := range kLines { - // let market data store trigger the update, so that the indicator could be updated too. - marketDataStore.AddKLine(k) + for _, k := range kLines { + // let market data store trigger the update, so that the indicator could be updated too. + marketDataStore.AddKLine(k) + } } } - } - log.Infof("%s last price: %v", symbol, session.lastPrices[symbol]) + log.Infof("%s last price: %v", symbol, session.lastPrices[symbol]) + } session.initializedSymbols[symbol] = struct{}{} return nil From 38507f4dd1ea7750220b67a57ec515dad36a6626 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 11 Nov 2023 07:59:44 +0800 Subject: [PATCH 176/422] bitget: add channel api code --- pkg/exchange/bitget/bitgetapi/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/bitgetapi/client.go b/pkg/exchange/bitget/bitgetapi/client.go index 19b145d5d7..f1a02710f1 100644 --- a/pkg/exchange/bitget/bitgetapi/client.go +++ b/pkg/exchange/bitget/bitgetapi/client.go @@ -51,7 +51,9 @@ func (c *RestClient) Auth(key, secret, passphrase string) { } // newAuthenticatedRequest creates new http request for authenticated routes. -func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL string, params url.Values, payload interface{}) (*http.Request, error) { +func (c *RestClient) NewAuthenticatedRequest( + ctx context.Context, method, refURL string, params url.Values, payload interface{}, +) (*http.Request, error) { if len(c.key) == 0 { return nil, errors.New("empty api key") } @@ -107,6 +109,7 @@ func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL req.Header.Add("ACCESS-SIGN", signature) req.Header.Add("ACCESS-TIMESTAMP", timestamp) req.Header.Add("ACCESS-PASSPHRASE", c.passphrase) + req.Header.Add("X-CHANNEL-API-CODE", "7575765263") return req, nil } From ef280077cd3d49a8f05419d50a558c6759454051 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 13 Nov 2023 11:53:41 +0800 Subject: [PATCH 177/422] pkg/exchange: print fee rate log --- pkg/exchange/bybit/market_info_poller.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/exchange/bybit/market_info_poller.go b/pkg/exchange/bybit/market_info_poller.go index 95664718f1..e35a1e4c21 100644 --- a/pkg/exchange/bybit/market_info_poller.go +++ b/pkg/exchange/bybit/market_info_poller.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "golang.org/x/time/rate" + "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" ) @@ -15,6 +17,10 @@ const ( feeRatePollingPeriod = time.Minute ) +var ( + pollFeeRateRateLimiter = rate.NewLimiter(rate.Every(10*time.Minute), 1) +) + type symbolFeeDetail struct { bybitapi.FeeRate @@ -78,6 +84,10 @@ func (p *feeRatePoller) poll(ctx context.Context) error { p.symbolFeeDetail = symbolFeeRate p.mu.Unlock() + if pollFeeRateRateLimiter.Allow() { + log.Infof("updated fee rate: %+v", p.symbolFeeDetail) + } + return nil } From 755ea5e427be734b503de6691e0a94d701820a2e Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 13 Nov 2023 19:21:58 +0800 Subject: [PATCH 178/422] pkg/exchange: implement query kline api --- .../bitget/bitgetapi/v2/client_test.go | 9 + .../get_history_orders_request_requestgen.go | 19 +- .../bitget/bitgetapi/v2/get_k_line.go | 83 +++++++ .../v2/get_k_line_request_requestgen.go | 224 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 24 ++ pkg/exchange/bitget/convert_test.go | 86 +++++++ pkg/exchange/bitget/exchange.go | 60 ++++- pkg/exchange/bitget/types.go | 35 +++ pkg/exchange/bitget/types_test.go | 59 +++-- 9 files changed, 562 insertions(+), 37 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_k_line.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 97b836f7fd..31909c7b70 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -5,6 +5,7 @@ import ( "os" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" @@ -78,4 +79,12 @@ func TestClient(t *testing.T) { resp, err := client.NewCancelOrderRequest().Symbol("APEUSDT").OrderId(req.OrderId).Do(ctx) t.Logf("cancel order resp: %+v", resp) }) + + t.Run("GetKLineRequest", func(t *testing.T) { + startTime := time.Date(2023, 8, 12, 0, 0, 0, 0, time.UTC) + endTime := time.Date(2023, 10, 14, 0, 0, 0, 0, time.UTC) + resp, err := client.NewGetKLineRequest().Symbol("APEUSDT").Granularity("30min").StartTime(startTime).EndTime(endTime).Limit("1000").Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go index 0a681f3228..398093399d 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go @@ -188,6 +188,12 @@ func (g *GetHistoryOrdersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetHistoryOrdersRequest) GetPath() string { + return "/api/v2/spot/trade/history-orders" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) { // no body params @@ -197,7 +203,9 @@ func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) return nil, err } - apiURL := "/api/v2/spot/trade/history-orders" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -214,6 +222,15 @@ func (g *GetHistoryOrdersRequest) Do(ctx context.Context) ([]OrderDetail, error) return nil, err } + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data []OrderDetail if err := json.Unmarshal(apiResponse.Data, &data); err != nil { return nil, err diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go b/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go new file mode 100644 index 0000000000..3a80a7c2f2 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_k_line.go @@ -0,0 +1,83 @@ +package bitgetapi + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type KLine struct { + // System timestamp, Unix millisecond timestamp, e.g. 1690196141868 + Ts types.MillisecondTimestamp + Open fixedpoint.Value + High fixedpoint.Value + Low fixedpoint.Value + Close fixedpoint.Value + // Trading volume in base currency, e.g. "BTC" in the "BTCUSD" pair. + Volume fixedpoint.Value + // Trading volume in quote currency, e.g. "USD" in the "BTCUSD" pair. + QuoteVolume fixedpoint.Value + // Trading volume in USDT + UsdtVolume fixedpoint.Value +} + +type KLineResponse []KLine + +const KLinesArrayLen = 8 + +func (k *KLine) UnmarshalJSON(data []byte) error { + var jsonArr []json.RawMessage + err := json.Unmarshal(data, &jsonArr) + if err != nil { + return fmt.Errorf("failed to unmarshal jsonRawMessage: %v, err: %w", string(data), err) + } + if len(jsonArr) != KLinesArrayLen { + return fmt.Errorf("unexpected K Lines array length: %d, exp: %d", len(jsonArr), KLinesArrayLen) + } + + err = json.Unmarshal(jsonArr[0], &k.Ts) + if err != nil { + return fmt.Errorf("failed to unmarshal resp index 0: %v, err: %w", string(jsonArr[0]), err) + } + + values := make([]fixedpoint.Value, len(jsonArr)-1) + for i, jsonRaw := range jsonArr[1:] { + err = json.Unmarshal(jsonRaw, &values[i]) + if err != nil { + return fmt.Errorf("failed to unmarshal resp index %d: %v, err: %w", i+1, string(jsonRaw), err) + } + } + k.Open = values[0] + k.High = values[1] + k.Low = values[2] + k.Close = values[3] + k.Volume = values[4] + k.QuoteVolume = values[5] + k.UsdtVolume = values[6] + + return nil +} + +//go:generate GetRequest -url "/api/v2/spot/market/candles" -type GetKLineRequest -responseDataType .KLineResponse +type GetKLineRequest struct { + client requestgen.APIClient + + symbol string `param:"symbol,query"` + granularity string `param:"granularity,query"` + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + // Limit number default 100 max 1000 + limit *string `param:"limit,query"` +} + +func (s *Client) NewGetKLineRequest() *GetKLineRequest { + return &GetKLineRequest{client: s.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go new file mode 100644 index 0000000000..2e4157ed24 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_k_line_request_requestgen.go @@ -0,0 +1,224 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/market/candles -type GetKLineRequest -responseDataType .KLineResponse"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetKLineRequest) Symbol(symbol string) *GetKLineRequest { + g.symbol = symbol + return g +} + +func (g *GetKLineRequest) Granularity(granularity string) *GetKLineRequest { + g.granularity = granularity + return g +} + +func (g *GetKLineRequest) StartTime(startTime time.Time) *GetKLineRequest { + g.startTime = &startTime + return g +} + +func (g *GetKLineRequest) EndTime(endTime time.Time) *GetKLineRequest { + g.endTime = &endTime + return g +} + +func (g *GetKLineRequest) Limit(limit string) *GetKLineRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetKLineRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + symbol := g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + // check granularity field -> json key granularity + granularity := g.granularity + + // assign parameter of granularity + params["granularity"] = granularity + // check startTime field -> json key startTime + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key endTime + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetKLineRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetKLineRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetKLineRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetKLineRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetKLineRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetKLineRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetKLineRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetKLineRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetKLineRequest) GetPath() string { + return "/api/v2/spot/market/candles" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetKLineRequest) Do(ctx context.Context) (KLineResponse, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data KLineResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index b9d1a1ccbc..a5e0e91a7c 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -6,6 +6,7 @@ import ( "math" "strconv" "strings" + "time" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" @@ -341,3 +342,26 @@ func toGlobalBalanceMap(balances []Balance) types.BalanceMap { } return bm } + +func toGlobalKLines(symbol string, interval types.Interval, kLines v2.KLineResponse) []types.KLine { + gKLines := make([]types.KLine, len(kLines)) + for i, kline := range kLines { + endTime := types.Time(kline.Ts.Time().Add(interval.Duration() - time.Millisecond)) + gKLines[i] = types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(kline.Ts), + EndTime: endTime, + Interval: interval, + Open: kline.Open, + Close: kline.Close, + High: kline.High, + Low: kline.Low, + Volume: kline.Volume, + QuoteVolume: kline.QuoteVolume, + // Bitget doesn't support close flag in REST API + Closed: false, + } + } + return gKLines +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 19e759aa1b..8bf66e9b71 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -3,6 +3,7 @@ package bitget import ( "strconv" "testing" + "time" "github.com/stretchr/testify/assert" @@ -598,3 +599,88 @@ func Test_toGlobalBalanceMap(t *testing.T) { }, })) } + +func Test_toGlobalKLines(t *testing.T) { + symbol := "BTCUSDT" + interval := types.Interval15m + + resp := v2.KLineResponse{ + /* + [ + { + "Ts": "1699816800000", + "OpenPrice": 29045.3, + "HighPrice": 29228.56, + "LowPrice": 29045.3, + "ClosePrice": 29228.56, + "Volume": 9.265593, + "QuoteVolume": 270447.43520753, + "UsdtVolume": 270447.43520753 + }, + { + "Ts": "1699816800000", + "OpenPrice": 29167.33, + "HighPrice": 29229.08, + "LowPrice": 29000, + "ClosePrice": 29045.3, + "Volume": 9.295508, + "QuoteVolume": 270816.87513775, + "UsdtVolume": 270816.87513775 + } + ] + */ + { + Ts: types.NewMillisecondTimestampFromInt(1691486100000), + Open: fixedpoint.NewFromFloat(29045.3), + High: fixedpoint.NewFromFloat(29228.56), + Low: fixedpoint.NewFromFloat(29045.3), + Close: fixedpoint.NewFromFloat(29228.56), + Volume: fixedpoint.NewFromFloat(9.265593), + QuoteVolume: fixedpoint.NewFromFloat(270447.43520753), + UsdtVolume: fixedpoint.NewFromFloat(270447.43520753), + }, + { + Ts: types.NewMillisecondTimestampFromInt(1691487000000), + Open: fixedpoint.NewFromFloat(29167.33), + High: fixedpoint.NewFromFloat(29229.08), + Low: fixedpoint.NewFromFloat(29000), + Close: fixedpoint.NewFromFloat(29045.3), + Volume: fixedpoint.NewFromFloat(9.295508), + QuoteVolume: fixedpoint.NewFromFloat(270816.87513775), + UsdtVolume: fixedpoint.NewFromFloat(270447.43520753), + }, + } + + expKlines := []types.KLine{ + { + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(resp[0].Ts.Time()), + EndTime: types.Time(resp[0].Ts.Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(29045.3), + Close: fixedpoint.NewFromFloat(29228.56), + High: fixedpoint.NewFromFloat(29228.56), + Low: fixedpoint.NewFromFloat(29045.3), + Volume: fixedpoint.NewFromFloat(9.265593), + QuoteVolume: fixedpoint.NewFromFloat(270447.43520753), + Closed: false, + }, + { + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(resp[1].Ts.Time()), + EndTime: types.Time(resp[1].Ts.Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(29167.33), + Close: fixedpoint.NewFromFloat(29045.3), + High: fixedpoint.NewFromFloat(29229.08), + Low: fixedpoint.NewFromFloat(29000), + Volume: fixedpoint.NewFromFloat(9.295508), + QuoteVolume: fixedpoint.NewFromFloat(270816.87513775), + Closed: false, + }, + } + + assert.Equal(t, toGlobalKLines(symbol, interval, resp), expKlines) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 988b626d2b..e941667738 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -21,9 +21,10 @@ const ( PlatformToken = "BGB" - queryLimit = 100 - maxOrderIdLen = 36 - queryMaxDuration = 90 * 24 * time.Hour + queryLimit = 100 + defaultKLineLimit = 100 + maxOrderIdLen = 36 + queryMaxDuration = 90 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -49,6 +50,8 @@ var ( queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) // cancelOrderRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Cancel-Order cancelOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // kLineRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/market/Get-Candle-Data + kLineOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) type Exchange struct { @@ -153,9 +156,56 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str return tickers, nil } +// QueryKLines queries the k line data by interval and time range...etc. +// +// If you provide only the start time, the system will return the latest data. +// If you provide both the start and end times, the system will return data within the specified range. +// If you provide only the end time, the system will return data that occurred before the end time. +// +// The end time has different limits. 1m, 5m can query for one month,15m can query for 52 days,30m can query for 62 days, +// 1H can query for 83 days,4H can query for 240 days,6H can query for 360 days. func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { - // TODO implement me - panic("implement me") + req := e.v2Client.NewGetKLineRequest().Symbol(symbol) + intervalStr, found := toLocalGranularity[interval] + if !found { + return nil, fmt.Errorf("%s not supported, supported granlarity: %+v", intervalStr, toLocalGranularity) + } + req.Granularity(intervalStr) + + limit := uint64(options.Limit) + if limit > defaultKLineLimit || limit <= 0 { + log.Debugf("limtit is exceeded or zero, update to %d, got: %d", defaultKLineLimit, options.Limit) + limit = defaultKLineLimit + } + req.Limit(strconv.FormatUint(limit, 10)) + + if options.StartTime != nil { + req.StartTime(*options.StartTime) + } + + if options.EndTime != nil { + if options.StartTime != nil && options.EndTime.Before(*options.StartTime) { + return nil, fmt.Errorf("end time %s before start time %s", *options.EndTime, *options.StartTime) + } + + ok, duration := hasMaxDuration(interval) + if ok && time.Since(*options.EndTime) > duration { + return nil, fmt.Errorf("end time %s are greater than max duration %s", *options.EndTime, duration) + } + req.EndTime(*options.EndTime) + } + + if err := kLineOrderRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query klines rate limiter wait error: %w", err) + } + + resp, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to call k line, err: %w", err) + } + + kLines := toGlobalKLines(symbol, interval, resp) + return types.SortKLinesAscending(kLines), nil } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index 0d5d5771ac..19c87c4846 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -299,8 +299,43 @@ var ( "candle1D": types.Interval1d, "candle1W": types.Interval1w, } + + // we align utc time zone + toLocalGranularity = map[types.Interval]string{ + types.Interval1m: "1min", + types.Interval5m: "5min", + types.Interval15m: "15min", + types.Interval30m: "30min", + types.Interval1h: "1h", + types.Interval4h: "4h", + types.Interval6h: "6Hutc", + types.Interval12h: "12Hutc", + types.Interval1d: "1Dutc", + types.Interval3d: "3Dutc", + types.Interval1w: "1Wutc", + types.Interval1mo: "1Mutc", + } ) +func hasMaxDuration(interval types.Interval) (bool, time.Duration) { + switch interval { + case types.Interval1m, types.Interval5m: + return true, 30 * 24 * time.Hour + case types.Interval15m: + return true, 52 * 24 * time.Hour + case types.Interval30m: + return true, 62 * 24 * time.Hour + case types.Interval1h: + return true, 83 * 24 * time.Hour + case types.Interval4h: + return true, 240 * 24 * time.Hour + case types.Interval6h: + return true, 360 * 24 * time.Hour + default: + return false, 0 * time.Duration(0) + } +} + type KLine struct { StartTime types.MillisecondTimestamp OpenPrice fixedpoint.Value diff --git a/pkg/exchange/bitget/types_test.go b/pkg/exchange/bitget/types_test.go index 1850566201..13cc344a78 100644 --- a/pkg/exchange/bitget/types_test.go +++ b/pkg/exchange/bitget/types_test.go @@ -6,38 +6,35 @@ import ( "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) -func TestKLine_ToGlobal(t *testing.T) { - startTime := int64(1698744600000) - interval := types.Interval1m - k := KLine{ - StartTime: types.NewMillisecondTimestampFromInt(startTime), - OpenPrice: fixedpoint.NewFromFloat(34361.49), - HighestPrice: fixedpoint.NewFromFloat(34458.98), - LowestPrice: fixedpoint.NewFromFloat(34355.53), - ClosePrice: fixedpoint.NewFromFloat(34416.41), - Volume: fixedpoint.NewFromFloat(99.6631), - } - - assert.Equal(t, types.KLine{ - Exchange: types.ExchangeBitget, - Symbol: "BTCUSDT", - StartTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time()), - EndTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time().Add(interval.Duration() - time.Millisecond)), - Interval: interval, - Open: fixedpoint.NewFromFloat(34361.49), - Close: fixedpoint.NewFromFloat(34416.41), - High: fixedpoint.NewFromFloat(34458.98), - Low: fixedpoint.NewFromFloat(34355.53), - Volume: fixedpoint.NewFromFloat(99.6631), - QuoteVolume: fixedpoint.Zero, - TakerBuyBaseAssetVolume: fixedpoint.Zero, - TakerBuyQuoteAssetVolume: fixedpoint.Zero, - LastTradeID: 0, - NumberOfTrades: 0, - Closed: false, - }, k.ToGlobal(interval, "BTCUSDT")) +func Test_hasMaxDuration(t *testing.T) { + ok, duration := hasMaxDuration(types.Interval1m) + assert.True(t, ok) + assert.Equal(t, 30*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval5m) + assert.True(t, ok) + assert.Equal(t, 30*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval15m) + assert.True(t, ok) + assert.Equal(t, 52*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval30m) + assert.True(t, ok) + assert.Equal(t, 62*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval1h) + assert.True(t, ok) + assert.Equal(t, 83*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval4h) + assert.True(t, ok) + assert.Equal(t, 240*24*time.Hour, duration) + + ok, duration = hasMaxDuration(types.Interval6h) + assert.True(t, ok) + assert.Equal(t, 360*24*time.Hour, duration) } From eb04eaeea44ee3ac27cb0cc6f303d3b5859d63dd Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 13 Nov 2023 12:51:19 +0800 Subject: [PATCH 179/422] pkg/exchange: types.kline end time should -1 time.Millisecond --- pkg/exchange/bitget/convert.go | 2 ++ pkg/exchange/bybit/convert.go | 2 +- pkg/exchange/bybit/convert_test.go | 4 ++-- pkg/types/kline.go | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index a5e0e91a7c..6c3c925464 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -346,6 +346,8 @@ func toGlobalBalanceMap(balances []Balance) types.BalanceMap { func toGlobalKLines(symbol string, interval types.Interval, kLines v2.KLineResponse) []types.KLine { gKLines := make([]types.KLine, len(kLines)) for i, kline := range kLines { + // follow the binance rule, to avoid endTime overlapping with the next startTime. So we subtract -1 time.Millisecond + // on endTime. endTime := types.Time(kline.Ts.Time().Add(interval.Duration() - time.Millisecond)) gKLines[i] = types.KLine{ Exchange: types.ExchangeBitget, diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index d89f194fea..8c2d915418 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -368,7 +368,7 @@ func toLocalInterval(interval types.Interval) (string, error) { func toGlobalKLines(symbol string, interval types.Interval, klines []bybitapi.KLine) []types.KLine { gKLines := make([]types.KLine, len(klines)) for i, kline := range klines { - endTime := types.Time(kline.StartTime.Time().Add(interval.Duration())) + endTime := types.Time(kline.StartTime.Time().Add(interval.Duration() - time.Millisecond)) gKLines[i] = types.KLine{ Exchange: types.ExchangeBybit, Symbol: symbol, diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index a5ddb08bc5..daac379e36 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -836,7 +836,7 @@ func Test_toGlobalKLines(t *testing.T) { Exchange: types.ExchangeBybit, Symbol: resp.Symbol, StartTime: types.Time(resp.List[0].StartTime.Time()), - EndTime: types.Time(resp.List[0].StartTime.Time().Add(interval.Duration())), + EndTime: types.Time(resp.List[0].StartTime.Time().Add(interval.Duration() - time.Millisecond)), Interval: interval, Open: fixedpoint.NewFromFloat(29045.3), Close: fixedpoint.NewFromFloat(29228.56), @@ -850,7 +850,7 @@ func Test_toGlobalKLines(t *testing.T) { Exchange: types.ExchangeBybit, Symbol: resp.Symbol, StartTime: types.Time(resp.List[1].StartTime.Time()), - EndTime: types.Time(resp.List[1].StartTime.Time().Add(interval.Duration())), + EndTime: types.Time(resp.List[1].StartTime.Time().Add(interval.Duration() - time.Millisecond)), Interval: interval, Open: fixedpoint.NewFromFloat(29167.33), Close: fixedpoint.NewFromFloat(29045.3), diff --git a/pkg/types/kline.go b/pkg/types/kline.go index 6bf863c30b..bd31ed9a29 100644 --- a/pkg/types/kline.go +++ b/pkg/types/kline.go @@ -54,7 +54,9 @@ type KLine struct { Symbol string `json:"symbol" db:"symbol"` StartTime Time `json:"startTime" db:"start_time"` - EndTime Time `json:"endTime" db:"end_time"` + // EndTime follows the binance rule, to avoid endTime overlapping with the next startTime. So if your end time (2023-01-01 01:00:00) + // are overlapping with next start time interval (2023-01-01 01:00:00), you should subtract -1 time.millisecond on EndTime. + EndTime Time `json:"endTime" db:"end_time"` Interval Interval `json:"interval" db:"interval"` From 5e5b8e1388c40b05c8dddba1caed9064d625e715 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 14:35:16 +0800 Subject: [PATCH 180/422] pkg/exchange: use v2 symbols --- .../bitget/bitgetapi/v2/client_test.go | 6 + .../bitgetapi/v2/get_symbols_request.go | 47 ++++++ .../v2/get_symbols_request_requestgen.go | 158 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 19 +-- pkg/exchange/bitget/convert_test.go | 60 +++---- pkg/exchange/bitget/exchange.go | 5 +- 6 files changed, 250 insertions(+), 45 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index 31909c7b70..d9feba7b19 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -87,4 +87,10 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("resp: %+v", resp) }) + + t.Run("GetSymbolsRequest", func(t *testing.T) { + resp, err := client.NewGetSymbolsRequest().Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go new file mode 100644 index 0000000000..cab2ed5541 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go @@ -0,0 +1,47 @@ +package bitgetapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type SymbolStatus string + +const ( + // SymbolOffline represent market is suspended, users cannot trade. + SymbolOffline SymbolStatus = "offline" + // SymbolGray represents market is online, but user trading is not available. + SymbolGray SymbolStatus = "gray" + // SymbolOnline trading begins, users can trade. + SymbolOnline SymbolStatus = "online" +) + +type Symbol struct { + Symbol string `json:"symbol"` + BaseCoin string `json:"baseCoin"` + QuoteCoin string `json:"quoteCoin"` + MinTradeAmount fixedpoint.Value `json:"minTradeAmount"` + MaxTradeAmount fixedpoint.Value `json:"maxTradeAmount"` + TakerFeeRate fixedpoint.Value `json:"takerFeeRate"` + MakerFeeRate fixedpoint.Value `json:"makerFeeRate"` + PricePrecision fixedpoint.Value `json:"pricePrecision"` + QuantityPrecision fixedpoint.Value `json:"quantityPrecision"` + QuotePrecision fixedpoint.Value `json:"quotePrecision"` + MinTradeUSDT fixedpoint.Value `json:"minTradeUSDT"` + Status SymbolStatus `json:"status"` + BuyLimitPriceRatio fixedpoint.Value `json:"buyLimitPriceRatio"` + SellLimitPriceRatio fixedpoint.Value `json:"sellLimitPriceRatio"` +} + +//go:generate GetRequest -url "/api/v2/spot/public/symbols" -type GetSymbolsRequest -responseDataType []Symbol +type GetSymbolsRequest struct { + client requestgen.APIClient +} + +func (c *Client) NewGetSymbolsRequest() *GetSymbolsRequest { + return &GetSymbolsRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go new file mode 100644 index 0000000000..7005830947 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go @@ -0,0 +1,158 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/public/symbols -type GetSymbolsRequest -responseDataType []Symbol"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetSymbolsRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetSymbolsRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetSymbolsRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetSymbolsRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetSymbolsRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetSymbolsRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetSymbolsRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetSymbolsRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetSymbolsRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetSymbolsRequest) GetPath() string { + return "/api/v2/spot/public/symbols" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetSymbolsRequest) Do(ctx context.Context) ([]Symbol, error) { + + // no body params + var params interface{} + query := url.Values{} + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []Symbol + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 6c3c925464..afe228c5c9 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "strconv" - "strings" "time" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" @@ -14,10 +13,6 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func toGlobalSymbol(s string) string { - return strings.ToUpper(s) -} - func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { return types.Balance{ Currency: asset.CoinName, @@ -30,23 +25,23 @@ func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { } } -func toGlobalMarket(s bitgetapi.Symbol) types.Market { - if s.Status != bitgetapi.SymbolOnline { +func toGlobalMarket(s v2.Symbol) types.Market { + if s.Status != v2.SymbolOnline { log.Warnf("The symbol %s is not online", s.Symbol) } return types.Market{ - Symbol: s.SymbolName, + Symbol: s.Symbol, LocalSymbol: s.Symbol, - PricePrecision: s.PriceScale.Int(), - VolumePrecision: s.QuantityScale.Int(), + PricePrecision: s.PricePrecision.Int(), + VolumePrecision: s.QuantityPrecision.Int(), QuoteCurrency: s.QuoteCoin, BaseCurrency: s.BaseCoin, MinNotional: s.MinTradeUSDT, MinAmount: s.MinTradeUSDT, MinQuantity: s.MinTradeAmount, MaxQuantity: s.MaxTradeAmount, - StepSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.QuantityScale.Int())), - TickSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.PriceScale.Int())), + StepSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.QuantityPrecision.Int())), + TickSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.PricePrecision.Int())), MinPrice: fixedpoint.Zero, MaxPrice: fixedpoint.Zero, } diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 8bf66e9b71..a6feab7e2f 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -46,53 +46,53 @@ func Test_toGlobalBalance(t *testing.T) { func Test_toGlobalMarket(t *testing.T) { // sample: //{ - // "symbol":"BTCUSDT_SPBL", - // "symbolName":"BTCUSDT", - // "baseCoin":"BTC", - // "quoteCoin":"USDT", - // "minTradeAmount":"0.0001", - // "maxTradeAmount":"10000", - // "takerFeeRate":"0.001", - // "makerFeeRate":"0.001", - // "priceScale":"4", - // "quantityScale":"8", - // "minTradeUSDT":"5", - // "status":"online", - // "buyLimitPriceRatio": "0.05", - // "sellLimitPriceRatio": "0.05" - // } - inst := bitgetapi.Symbol{ - Symbol: "BTCUSDT_SPBL", - SymbolName: "BTCUSDT", + // "symbol":"BTCUSDT", + // "baseCoin":"BTC", + // "quoteCoin":"USDT", + // "minTradeAmount":"0", + // "maxTradeAmount":"10000000000", + // "takerFeeRate":"0.002", + // "makerFeeRate":"0.002", + // "pricePrecision":"2", + // "quantityPrecision":"4", + // "quotePrecision":"6", + // "status":"online", + // "minTradeUSDT":"5", + // "buyLimitPriceRatio":"0.05", + // "sellLimitPriceRatio":"0.05" + //} + inst := v2.Symbol{ + Symbol: "BTCUSDT", BaseCoin: "BTC", QuoteCoin: "USDT", - MinTradeAmount: fixedpoint.NewFromFloat(0.0001), - MaxTradeAmount: fixedpoint.NewFromFloat(10000), - TakerFeeRate: fixedpoint.NewFromFloat(0.001), - MakerFeeRate: fixedpoint.NewFromFloat(0.001), - PriceScale: fixedpoint.NewFromFloat(4), - QuantityScale: fixedpoint.NewFromFloat(8), + MinTradeAmount: fixedpoint.NewFromFloat(0), + MaxTradeAmount: fixedpoint.NewFromFloat(10000000000), + TakerFeeRate: fixedpoint.NewFromFloat(0.002), + MakerFeeRate: fixedpoint.NewFromFloat(0.002), + PricePrecision: fixedpoint.NewFromFloat(2), + QuantityPrecision: fixedpoint.NewFromFloat(4), + QuotePrecision: fixedpoint.NewFromFloat(6), MinTradeUSDT: fixedpoint.NewFromFloat(5), - Status: bitgetapi.SymbolOnline, + Status: v2.SymbolOnline, BuyLimitPriceRatio: fixedpoint.NewFromFloat(0.05), SellLimitPriceRatio: fixedpoint.NewFromFloat(0.05), } exp := types.Market{ - Symbol: inst.SymbolName, + Symbol: inst.Symbol, LocalSymbol: inst.Symbol, - PricePrecision: 4, - VolumePrecision: 8, + PricePrecision: 2, + VolumePrecision: 4, QuoteCurrency: inst.QuoteCoin, BaseCurrency: inst.BaseCoin, MinNotional: inst.MinTradeUSDT, MinAmount: inst.MinTradeUSDT, MinQuantity: inst.MinTradeAmount, MaxQuantity: inst.MaxTradeAmount, - StepSize: fixedpoint.NewFromFloat(0.00000001), + StepSize: fixedpoint.NewFromFloat(0.0001), MinPrice: fixedpoint.Zero, MaxPrice: fixedpoint.Zero, - TickSize: fixedpoint.NewFromFloat(0.0001), + TickSize: fixedpoint.NewFromFloat(0.01), } assert.Equal(t, toGlobalMarket(inst), exp) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index e941667738..7750062f9b 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -94,7 +94,7 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { return nil, fmt.Errorf("markets rate limiter wait error: %w", err) } - req := e.client.NewGetSymbolsRequest() + req := e.v2Client.NewGetSymbolsRequest() symbols, err := req.Do(ctx) if err != nil { return nil, err @@ -102,8 +102,7 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { markets := types.MarketMap{} for _, s := range symbols { - symbol := toGlobalSymbol(s.SymbolName) - markets[symbol] = toGlobalMarket(s) + markets[s.Symbol] = toGlobalMarket(s) } return markets, nil From 737f2fc86da16d459cd38625ad5f3acb927e83b1 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 15:26:07 +0800 Subject: [PATCH 181/422] pkg/exchange: add response validator --- pkg/exchange/bitget/bitgetapi/client.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/exchange/bitget/bitgetapi/client.go b/pkg/exchange/bitget/bitgetapi/client.go index 8fa7f4404a..a85b07af3f 100644 --- a/pkg/exchange/bitget/bitgetapi/client.go +++ b/pkg/exchange/bitget/bitgetapi/client.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" + "fmt" "net/http" "net/url" "strconv" @@ -164,3 +165,17 @@ type APIResponse struct { Message string `json:"msg"` Data json.RawMessage `json:"data"` } + +func (a APIResponse) Validate() error { + // v1, v2 use the same success code. + // https://www.bitget.com/api-doc/spot/error-code/restapi + // https://bitgetlimited.github.io/apidoc/en/mix/#restapi-error-codes + if a.Code != "00000" { + return a.Error() + } + return nil +} + +func (a APIResponse) Error() error { + return fmt.Errorf("code: %s, msg: %s, data: %q", a.Code, a.Message, a.Data) +} From 53bce6d5c1f721eca0ca33c37aa867ff8abae5f2 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 14:57:07 +0800 Subject: [PATCH 182/422] pkg/exchange: use v2 query ticker --- .../bitget/bitgetapi/v2/client_test.go | 6 + .../bitgetapi/v2/get_tickers_request.go | 40 ++++ .../v2/get_tickers_request_requestgen.go | 174 ++++++++++++++++++ pkg/exchange/bitget/convert.go | 12 +- pkg/exchange/bitget/convert_test.go | 68 +++---- pkg/exchange/bitget/exchange.go | 9 +- 6 files changed, 267 insertions(+), 42 deletions(-) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_tickers_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_tickers_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index d9feba7b19..afefe7c9e1 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -93,4 +93,10 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("resp: %+v", resp) }) + + t.Run("GetTickersRequest", func(t *testing.T) { + resp, err := client.NewGetTickersRequest().Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request.go new file mode 100644 index 0000000000..02e5d07b57 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request.go @@ -0,0 +1,40 @@ +package bitgetapi + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type Ticker struct { + Symbol string `json:"symbol"` + High24H fixedpoint.Value `json:"high24h"` + Open fixedpoint.Value `json:"open"` + Low24H fixedpoint.Value `json:"low24h"` + LastPr fixedpoint.Value `json:"lastPr"` + QuoteVolume fixedpoint.Value `json:"quoteVolume"` + BaseVolume fixedpoint.Value `json:"baseVolume"` + UsdtVolume fixedpoint.Value `json:"usdtVolume"` + BidPr fixedpoint.Value `json:"bidPr"` + AskPr fixedpoint.Value `json:"askPr"` + BidSz fixedpoint.Value `json:"bidSz"` + AskSz fixedpoint.Value `json:"askSz"` + OpenUtc fixedpoint.Value `json:"openUtc"` + Ts types.MillisecondTimestamp `json:"ts"` + ChangeUtc24H fixedpoint.Value `json:"changeUtc24h"` + Change24H fixedpoint.Value `json:"change24h"` +} + +//go:generate GetRequest -url "/api/v2/spot/market/tickers" -type GetTickersRequest -responseDataType []Ticker +type GetTickersRequest struct { + client requestgen.APIClient + + symbol *string `param:"symbol,query"` +} + +func (s *Client) NewGetTickersRequest() *GetTickersRequest { + return &GetTickersRequest{client: s.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request_requestgen.go new file mode 100644 index 0000000000..6e60c21068 --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_tickers_request_requestgen.go @@ -0,0 +1,174 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/market/tickers -type GetTickersRequest -responseDataType []Ticker"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (g *GetTickersRequest) Symbol(symbol string) *GetTickersRequest { + g.symbol = &symbol + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTickersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check symbol field -> json key symbol + if g.symbol != nil { + symbol := *g.symbol + + // assign parameter of symbol + params["symbol"] = symbol + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTickersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTickersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTickersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTickersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTickersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTickersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTickersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTickersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetTickersRequest) GetPath() string { + return "/api/v2/spot/market/tickers" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetTickersRequest) Do(ctx context.Context) ([]Ticker, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []Ticker + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index afe228c5c9..3a8a875dce 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -47,16 +47,16 @@ func toGlobalMarket(s v2.Symbol) types.Market { } } -func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker { +func toGlobalTicker(ticker v2.Ticker) types.Ticker { return types.Ticker{ Time: ticker.Ts.Time(), - Volume: ticker.BaseVol, - Last: ticker.Close, - Open: ticker.OpenUtc0, + Volume: ticker.BaseVolume, + Last: ticker.LastPr, + Open: ticker.Open, High: ticker.High24H, Low: ticker.Low24H, - Buy: ticker.BuyOne, - Sell: ticker.SellOne, + Buy: ticker.BidPr, + Sell: ticker.AskPr, } } diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index a6feab7e2f..dff77b4360 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -100,39 +100,41 @@ func Test_toGlobalMarket(t *testing.T) { func Test_toGlobalTicker(t *testing.T) { // sample: - // { - // "symbol": "BTCUSDT", - // "high24h": "24175.65", - // "low24h": "23677.75", - // "close": "24014.11", - // "quoteVol": "177689342.3025", - // "baseVol": "7421.5009", - // "usdtVol": "177689342.302407", - // "ts": "1660704288118", - // "buyOne": "24013.94", - // "sellOne": "24014.06", - // "bidSz": "0.0663", - // "askSz": "0.0119", - // "openUtc0": "23856.72", - // "changeUtc":"0.00301", - // "change":"0.00069" - // } - ticker := bitgetapi.Ticker{ - Symbol: "BTCUSDT", - High24H: fixedpoint.NewFromFloat(24175.65), - Low24H: fixedpoint.NewFromFloat(23677.75), - Close: fixedpoint.NewFromFloat(24014.11), - QuoteVol: fixedpoint.NewFromFloat(177689342.3025), - BaseVol: fixedpoint.NewFromFloat(7421.5009), - UsdtVol: fixedpoint.NewFromFloat(177689342.302407), - Ts: types.NewMillisecondTimestampFromInt(1660704288118), - BuyOne: fixedpoint.NewFromFloat(24013.94), - SellOne: fixedpoint.NewFromFloat(24014.06), - BidSz: fixedpoint.NewFromFloat(0.0663), - AskSz: fixedpoint.NewFromFloat(0.0119), - OpenUtc0: fixedpoint.NewFromFloat(23856.72), - ChangeUtc: fixedpoint.NewFromFloat(0.00301), - Change: fixedpoint.NewFromFloat(0.00069), + //{ + // "open":"36465.96", + // "symbol":"BTCUSDT", + // "high24h":"37040.25", + // "low24h":"36202.65", + // "lastPr":"36684.42", + // "quoteVolume":"311893591.2805", + // "baseVolume":"8507.3684", + // "usdtVolume":"311893591.280427", + // "ts":"1699947106122", + // "bidPr":"36684.49", + // "askPr":"36684.51", + // "bidSz":"0.3812", + // "askSz":"0.0133", + // "openUtc":"36465.96", + // "changeUtc24h":"0.00599", + // "change24h":"-0.00426" + //} + ticker := v2.Ticker{ + Symbol: "BTCUSDT", + High24H: fixedpoint.NewFromFloat(24175.65), + Low24H: fixedpoint.NewFromFloat(23677.75), + LastPr: fixedpoint.NewFromFloat(24014.11), + QuoteVolume: fixedpoint.NewFromFloat(177689342.3025), + BaseVolume: fixedpoint.NewFromFloat(7421.5009), + UsdtVolume: fixedpoint.NewFromFloat(177689342.302407), + Ts: types.NewMillisecondTimestampFromInt(1660704288118), + BidPr: fixedpoint.NewFromFloat(24013.94), + AskPr: fixedpoint.NewFromFloat(24014.06), + BidSz: fixedpoint.NewFromFloat(0.0663), + AskSz: fixedpoint.NewFromFloat(0.0119), + OpenUtc: fixedpoint.NewFromFloat(23856.72), + ChangeUtc24H: fixedpoint.NewFromFloat(0.00301), + Change24H: fixedpoint.NewFromFloat(0.00069), + Open: fixedpoint.NewFromFloat(23856.72), } assert.Equal(t, types.Ticker{ diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 7750062f9b..86e891c319 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -113,14 +113,17 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke return nil, fmt.Errorf("ticker rate limiter wait error: %w", err) } - req := e.client.NewGetTickerRequest() + req := e.v2Client.NewGetTickersRequest() req.Symbol(symbol) resp, err := req.Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query ticker: %w", err) } + if len(resp) != 1 { + return nil, fmt.Errorf("unexpected length of query single symbol: %+v", resp) + } - ticker := toGlobalTicker(*resp) + ticker := toGlobalTicker(resp[1]) return &ticker, nil } @@ -143,7 +146,7 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str return nil, fmt.Errorf("tickers rate limiter wait error: %w", err) } - resp, err := e.client.NewGetAllTickersRequest().Do(ctx) + resp, err := e.v2Client.NewGetTickersRequest().Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query tickers: %w", err) } From 5808e0184b228462e4a14d7e5b1d30db33a20e22 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 18:13:54 +0800 Subject: [PATCH 183/422] pkg/types: skip pong event on emitting raw message --- pkg/exchange/bitget/stream.go | 12 ++----- pkg/exchange/bybit/stream.go | 12 +++++-- pkg/exchange/bybit/types.go | 7 ++++ pkg/exchange/bybit/types_test.go | 56 +++++--------------------------- pkg/types/stream.go | 31 +++++++++++++----- 5 files changed, 49 insertions(+), 69 deletions(-) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 53e6964901..00ecae4263 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -129,11 +129,6 @@ func (s *Stream) dispatchEvent(event interface{}) { case *AccountEvent: s.EmitAccountEvent(*e) - case []byte: - // We only handle the 'pong' case. Others are unexpected. - if !bytes.Equal(e, pongBytes) { - log.Errorf("invalid event: %q", e) - } } } @@ -240,10 +235,9 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { func parseWebSocketEvent(in []byte) (interface{}, error) { switch { case bytes.Equal(in, pongBytes): - // Return the original raw data may seem redundant because we can validate the string and return nil, - // but we cannot return nil to a lower level handler. This can cause confusion in the next handler, such as - // the dispatch handler. Therefore, I return the original raw data. - return in, nil + // return global pong event to avoid emit raw message + return types.WebsocketPongEvent{}, nil + default: return parseEvent(in) } diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 8911125673..47581f2aed 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -159,9 +159,6 @@ func (s *Stream) createEndpoint(_ context.Context) (string, error) { func (s *Stream) dispatchEvent(event interface{}) { switch e := event.(type) { case *WebSocketOpEvent: - if err := e.IsValid(); err != nil { - log.Errorf("invalid event: %v", err) - } if e.IsAuthenticated() { s.EmitAuth() } @@ -197,6 +194,15 @@ func (s *Stream) parseWebSocketEvent(in []byte) (interface{}, error) { switch { case e.IsOp(): + if err = e.IsValid(); err != nil { + log.Errorf("invalid event: %+v, err: %s", e, err) + return nil, err + } + + // return global pong event to avoid emit raw message + if ok, pongEvent := e.toGlobalPongEventIfValid(); ok { + return pongEvent, nil + } return e.WebSocketOpEvent, nil case e.IsTopic(): diff --git a/pkg/exchange/bybit/types.go b/pkg/exchange/bybit/types.go index 7ae55bd857..1c95f638c8 100644 --- a/pkg/exchange/bybit/types.go +++ b/pkg/exchange/bybit/types.go @@ -88,6 +88,13 @@ func (w *WebSocketOpEvent) IsValid() error { } } +func (w *WebSocketOpEvent) toGlobalPongEventIfValid() (bool, *types.WebsocketPongEvent) { + if w.Op == WsOpTypePing || w.Op == WsOpTypePong { + return true, &types.WebsocketPongEvent{} + } + return false, nil +} + func (w *WebSocketOpEvent) IsAuthenticated() bool { return w.Op == WsOpTypeAuth && w.Success } diff --git a/pkg/exchange/bybit/types_test.go b/pkg/exchange/bybit/types_test.go index 5f32abd776..603c49f092 100644 --- a/pkg/exchange/bybit/types_test.go +++ b/pkg/exchange/bybit/types_test.go @@ -20,19 +20,9 @@ func Test_parseWebSocketEvent(t *testing.T) { raw, err := s.parseWebSocketEvent([]byte(msg)) assert.NoError(t, err) - expRetMsg := string(WsOpTypePong) - e, ok := raw.(*WebSocketOpEvent) + e, ok := raw.(*types.WebsocketPongEvent) assert.True(t, ok) - assert.Equal(t, &WebSocketOpEvent{ - Success: true, - RetMsg: expRetMsg, - ConnId: "a806f6c4-3608-4b6d-a225-9f5da975bc44", - ReqId: "", - Op: WsOpTypePing, - Args: nil, - }, e) - - assert.NoError(t, e.IsValid()) + assert.Equal(t, &types.WebsocketPongEvent{}, e) }) t.Run("[public] PingEvent with req id", func(t *testing.T) { @@ -41,20 +31,9 @@ func Test_parseWebSocketEvent(t *testing.T) { raw, err := s.parseWebSocketEvent([]byte(msg)) assert.NoError(t, err) - expRetMsg := string(WsOpTypePong) - expReqId := "b26704da-f5af-44c2-bdf7-935d6739e1a0" - e, ok := raw.(*WebSocketOpEvent) + e, ok := raw.(*types.WebsocketPongEvent) assert.True(t, ok) - assert.Equal(t, &WebSocketOpEvent{ - Success: true, - RetMsg: expRetMsg, - ConnId: "a806f6c4-3608-4b6d-a225-9f5da975bc44", - ReqId: expReqId, - Op: WsOpTypePing, - Args: nil, - }, e) - - assert.NoError(t, e.IsValid()) + assert.Equal(t, &types.WebsocketPongEvent{}, e) }) t.Run("[private] PingEvent without req id", func(t *testing.T) { @@ -63,18 +42,9 @@ func Test_parseWebSocketEvent(t *testing.T) { raw, err := s.parseWebSocketEvent([]byte(msg)) assert.NoError(t, err) - e, ok := raw.(*WebSocketOpEvent) + e, ok := raw.(*types.WebsocketPongEvent) assert.True(t, ok) - assert.Equal(t, &WebSocketOpEvent{ - Success: false, - RetMsg: "", - ConnId: "civn4p1dcjmtvb69ome0-yrt1", - ReqId: "", - Op: WsOpTypePong, - Args: []string{"1690884539181"}, - }, e) - - assert.NoError(t, e.IsValid()) + assert.Equal(t, &types.WebsocketPongEvent{}, e) }) t.Run("[private] PingEvent with req id", func(t *testing.T) { @@ -83,19 +53,9 @@ func Test_parseWebSocketEvent(t *testing.T) { raw, err := s.parseWebSocketEvent([]byte(msg)) assert.NoError(t, err) - expReqId := "78d36b57-a142-47b7-9143-5843df77d44d" - e, ok := raw.(*WebSocketOpEvent) + e, ok := raw.(*types.WebsocketPongEvent) assert.True(t, ok) - assert.Equal(t, &WebSocketOpEvent{ - Success: false, - RetMsg: "", - ConnId: "civn4p1dcjmtvb69ome0-yrt1", - ReqId: expReqId, - Op: WsOpTypePong, - Args: []string{"1690884539181"}, - }, e) - - assert.NoError(t, e.IsValid()) + assert.Equal(t, &types.WebsocketPongEvent{}, e) }) } diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 4ce8c161fd..96c66c4eae 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -62,6 +62,8 @@ type HeartBeat func(conn *websocket.Conn) error type BeforeConnect func(ctx context.Context) error +type WebsocketPongEvent struct{} + //go:generate callbackgen -type StandardStream -interface type StandardStream struct { parser Parser @@ -226,6 +228,9 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel // flag format: debug-{component}-{message type} debugRawMessage := viper.GetBool("debug-websocket-raw-message") + hasParser := s.parser != nil + hasDispatcher := s.dispatcher != nil + for { select { @@ -276,22 +281,30 @@ func (s *StandardStream) Read(ctx context.Context, conn *websocket.Conn, cancel continue } - s.EmitRawMessage(message) - if debugRawMessage { log.Info(string(message)) } + if !hasParser { + s.EmitRawMessage(message) + continue + } + var e interface{} - if s.parser != nil { - e, err = s.parser(message) - if err != nil { - log.WithError(err).Errorf("websocket event parse error, message: %s", message) - continue - } + e, err = s.parser(message) + if err != nil { + log.WithError(err).Errorf("websocket event parse error, message: %s", message) + // emit raw message even if occurs error, because we want anything can be detected + s.EmitRawMessage(message) + continue + } + + // skip pong event to avoid the message like spam + if _, ok := e.(*WebsocketPongEvent); !ok { + s.EmitRawMessage(message) } - if s.dispatcher != nil { + if hasDispatcher { s.dispatcher(e) } } From 562f85af75ca4b537afcbb384ab5e2876124bc91 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 20:42:11 +0800 Subject: [PATCH 184/422] pkg/exchange: rename v2Client -> v2client --- pkg/exchange/bitget/exchange.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 86e891c319..064a303e08 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -58,7 +58,7 @@ type Exchange struct { key, secret, passphrase string client *bitgetapi.RestClient - v2Client *v2.Client + v2client *v2.Client } func New(key, secret, passphrase string) *Exchange { @@ -73,7 +73,7 @@ func New(key, secret, passphrase string) *Exchange { secret: secret, passphrase: passphrase, client: client, - v2Client: v2.NewClient(client), + v2client: v2.NewClient(client), } } @@ -94,7 +94,7 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { return nil, fmt.Errorf("markets rate limiter wait error: %w", err) } - req := e.v2Client.NewGetSymbolsRequest() + req := e.v2client.NewGetSymbolsRequest() symbols, err := req.Do(ctx) if err != nil { return nil, err @@ -113,7 +113,7 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke return nil, fmt.Errorf("ticker rate limiter wait error: %w", err) } - req := e.v2Client.NewGetTickersRequest() + req := e.v2client.NewGetTickersRequest() req.Symbol(symbol) resp, err := req.Do(ctx) if err != nil { @@ -146,7 +146,7 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str return nil, fmt.Errorf("tickers rate limiter wait error: %w", err) } - resp, err := e.v2Client.NewGetTickersRequest().Do(ctx) + resp, err := e.v2client.NewGetTickersRequest().Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query tickers: %w", err) } @@ -167,7 +167,7 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str // The end time has different limits. 1m, 5m can query for one month,15m can query for 52 days,30m can query for 62 days, // 1H can query for 83 days,4H can query for 240 days,6H can query for 360 days. func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { - req := e.v2Client.NewGetKLineRequest().Symbol(symbol) + req := e.v2client.NewGetKLineRequest().Symbol(symbol) intervalStr, found := toLocalGranularity[interval] if !found { return nil, fmt.Errorf("%s not supported, supported granlarity: %+v", intervalStr, toLocalGranularity) @@ -255,7 +255,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr return nil, fmt.Errorf("order.Market.Symbol is required: %+v", order) } - req := e.v2Client.NewPlaceOrderRequest() + req := e.v2client.NewPlaceOrderRequest() req.Symbol(order.Market.Symbol) // set order type @@ -323,7 +323,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr } orderId := res.OrderId - ordersResp, err := e.v2Client.NewGetUnfilledOrdersRequest().OrderId(orderId).Do(ctx) + ordersResp, err := e.v2client.NewGetUnfilledOrdersRequest().OrderId(orderId).Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query open order by order id: %s, err: %w", orderId, err) } @@ -332,7 +332,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr case 0: // The market order will be executed immediately, so we cannot retrieve it through the NewGetUnfilledOrdersRequest API. // Try to get the order from the NewGetHistoryOrdersRequest API. - ordersResp, err := e.v2Client.NewGetHistoryOrdersRequest().OrderId(orderId).Do(ctx) + ordersResp, err := e.v2client.NewGetHistoryOrdersRequest().OrderId(orderId).Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query history order by order id: %s, err: %w", orderId, err) } @@ -358,7 +358,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return nil, fmt.Errorf("open order rate limiter wait error: %w", err) } - req := e.v2Client.NewGetUnfilledOrdersRequest(). + req := e.v2client.NewGetUnfilledOrdersRequest(). Symbol(symbol). Limit(strconv.FormatInt(queryLimit, 10)) if nextCursor != 0 { @@ -417,7 +417,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, if err := closedQueryOrdersRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("query closed order rate limiter wait error: %w", err) } - res, err := e.v2Client.NewGetHistoryOrdersRequest(). + res, err := e.v2client.NewGetHistoryOrdersRequest(). Symbol(symbol). Limit(strconv.Itoa(queryLimit)). StartTime(since.UnixMilli()). @@ -505,7 +505,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The trade of response is in descending order, so the last trade id not supported.") } - req := e.v2Client.NewGetTradeFillsRequest() + req := e.v2client.NewGetTradeFillsRequest() req.Symbol(symbol) if options.StartTime != nil { From 720fe2e12ee124bf0fa4976b1113f65b1b06c4a7 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 11:22:42 +0800 Subject: [PATCH 185/422] pkg/bbgo, pkg/types: add new interface PrivateChannelSymbolSetter --- pkg/bbgo/session.go | 9 +++++++++ pkg/types/stream.go | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 101abd3f9c..617abef2f2 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -58,6 +58,10 @@ type ExchangeSession struct { // This option is exchange specific PrivateChannels []string `json:"privateChannels,omitempty" yaml:"privateChannels,omitempty"` + // PrivateChannelSymbols is used for filtering the private user data channel, .e.g, order symbol subscription. + // This option is exchange specific + PrivateChannelSymbols []string `json:"privateChannelSymbols,omitempty" yaml:"privateChannelSymbols,omitempty"` + Margin bool `json:"margin,omitempty" yaml:"margin"` IsolatedMargin bool `json:"isolatedMargin,omitempty" yaml:"isolatedMargin,omitempty"` IsolatedMarginSymbol string `json:"isolatedMarginSymbol,omitempty" yaml:"isolatedMarginSymbol,omitempty"` @@ -248,6 +252,11 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) setter.SetPrivateChannels(session.PrivateChannels) } } + if len(session.PrivateChannelSymbols) > 0 { + if setter, ok := session.UserDataStream.(types.PrivateChannelSymbolSetter); ok { + setter.SetPrivateChannelSymbols(session.PrivateChannelSymbols) + } + } logger.Infof("querying account balances...") account, err := session.Exchange.QueryAccount(ctx) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index 96c66c4eae..f8aa209e13 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -46,6 +46,10 @@ type PrivateChannelSetter interface { SetPrivateChannels(channels []string) } +type PrivateChannelSymbolSetter interface { + SetPrivateChannelSymbols(symbols []string) +} + type Unsubscriber interface { // Unsubscribe unsubscribes the all subscriptions. Unsubscribe() From fdc4c12ac186517f0a5b6d08349c541c7e42719e Mon Sep 17 00:00:00 2001 From: narumi Date: Sat, 11 Nov 2023 10:16:55 +0800 Subject: [PATCH 186/422] add wise rate api --- pkg/datasource/wise/README.md | 25 ++ pkg/datasource/wise/client.go | 80 ++++++ pkg/datasource/wise/group.go | 9 + pkg/datasource/wise/rate.go | 10 + pkg/datasource/wise/rate_request.go | 27 +++ .../wise/rate_request_requestgen.go | 229 ++++++++++++++++++ pkg/datasource/wise/time.go | 34 +++ 7 files changed, 414 insertions(+) create mode 100644 pkg/datasource/wise/README.md create mode 100644 pkg/datasource/wise/client.go create mode 100644 pkg/datasource/wise/group.go create mode 100644 pkg/datasource/wise/rate.go create mode 100644 pkg/datasource/wise/rate_request.go create mode 100644 pkg/datasource/wise/rate_request_requestgen.go create mode 100644 pkg/datasource/wise/time.go diff --git a/pkg/datasource/wise/README.md b/pkg/datasource/wise/README.md new file mode 100644 index 0000000000..1b6a04f808 --- /dev/null +++ b/pkg/datasource/wise/README.md @@ -0,0 +1,25 @@ +# Wise + +[Wise API Docs](https://docs.wise.com/api-docs) + +```go +c := wise.NewClient() +c.Auth(os.Getenv("WISE_TOKEN")) + +ctx := context.Background() +rates, err := c.QueryRate(ctx, "USD", "TWD") +if err != nil { + panic(err) +} +fmt.Printf("%+v\n", rates) + +// or +now := time.Now() +rates, err = c.QueryRateHistory(ctx, "USD", "TWD", now.Add(-time.Hour*24*7), now, types.Interval1h) +if err != nil { + panic(err) +} +for _, rate := range rates { + fmt.Printf("%+v\n", rate) +} +``` diff --git a/pkg/datasource/wise/client.go b/pkg/datasource/wise/client.go new file mode 100644 index 0000000000..7f0fee7b55 --- /dev/null +++ b/pkg/datasource/wise/client.go @@ -0,0 +1,80 @@ +package wise + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +const ( + defaultHTTPTimeout = time.Second * 15 + defaultBaseURL = "https://api.transferwise.com" + sandboxBaseURL = "https://api.sandbox.transferwise.tech" +) + +type Client struct { + requestgen.BaseAPIClient + + token string +} + +func NewClient() *Client { + u, err := url.Parse(defaultBaseURL) + if err != nil { + panic(err) + } + + return &Client{ + BaseAPIClient: requestgen.BaseAPIClient{ + BaseURL: u, + HttpClient: &http.Client{ + Timeout: defaultHTTPTimeout, + }, + }, + } +} + +func (c *Client) Auth(token string) { + c.token = token +} + +func (c *Client) NewAuthenticatedRequest(ctx context.Context, method, refURL string, params url.Values, payload interface{}) (*http.Request, error) { + req, err := c.NewRequest(ctx, method, refURL, params, payload) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "application/json") + + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + return req, nil +} + +func (c *Client) QueryRate(ctx context.Context, source string, target string) ([]Rate, error) { + req := c.NewRateRequest().Source(source).Target(target) + return req.Do(ctx) +} + +func (c *Client) QueryRateHistory(ctx context.Context, source string, target string, from time.Time, to time.Time, interval types.Interval) ([]Rate, error) { + req := c.NewRateRequest().Source(source).Target(target).From(from).To(to) + + switch interval { + case types.Interval1h: + req.Group(GroupHour) + case types.Interval1d: + req.Group(GroupDay) + case types.Interval1m: + req.Group(GroupMinute) + default: + return nil, fmt.Errorf("unsupported interval: %s", interval) + } + + return req.Do(ctx) +} diff --git a/pkg/datasource/wise/group.go b/pkg/datasource/wise/group.go new file mode 100644 index 0000000000..2770237b56 --- /dev/null +++ b/pkg/datasource/wise/group.go @@ -0,0 +1,9 @@ +package wise + +type Group string + +const ( + GroupMinute = Group("minute") + GroupHour = Group("hour") + GroupDay = Group("day") +) diff --git a/pkg/datasource/wise/rate.go b/pkg/datasource/wise/rate.go new file mode 100644 index 0000000000..7a901626f9 --- /dev/null +++ b/pkg/datasource/wise/rate.go @@ -0,0 +1,10 @@ +package wise + +import "github.com/c9s/bbgo/pkg/fixedpoint" + +type Rate struct { + Value fixedpoint.Value `json:"rate"` + Target string `json:"target"` + Source string `json:"source"` + Time Time `json:"time"` +} diff --git a/pkg/datasource/wise/rate_request.go b/pkg/datasource/wise/rate_request.go new file mode 100644 index 0000000000..f8d8561b09 --- /dev/null +++ b/pkg/datasource/wise/rate_request.go @@ -0,0 +1,27 @@ +package wise + +import ( + "time" + + "github.com/c9s/requestgen" +) + +// https://docs.wise.com/api-docs/api-reference/rate + +//go:generate requestgen -method GET -url "/v1/rates" -type RateRequest -responseType []Rate +type RateRequest struct { + client requestgen.AuthenticatedAPIClient + + source string `param:"source"` + target string `param:"target"` + time *time.Time `param:"time" timeFormat:"2006-01-02T15:04:05-0700"` + from *time.Time `param:"from" timeFormat:"2006-01-02T15:04:05-0700"` + to *time.Time `param:"to" timeFormat:"2006-01-02T15:04:05-0700"` + group *Group `param:"group"` +} + +func (c *Client) NewRateRequest() *RateRequest { + return &RateRequest{ + client: c, + } +} diff --git a/pkg/datasource/wise/rate_request_requestgen.go b/pkg/datasource/wise/rate_request_requestgen.go new file mode 100644 index 0000000000..7869491989 --- /dev/null +++ b/pkg/datasource/wise/rate_request_requestgen.go @@ -0,0 +1,229 @@ +// Code generated by "requestgen -method GET -url /v1/rates -type RateRequest -responseType []Rate"; DO NOT EDIT. + +package wise + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" + "time" +) + +func (r *RateRequest) Source(source string) *RateRequest { + r.source = source + return r +} + +func (r *RateRequest) Target(target string) *RateRequest { + r.target = target + return r +} + +func (r *RateRequest) Time(time time.Time) *RateRequest { + r.time = &time + return r +} + +func (r *RateRequest) From(from time.Time) *RateRequest { + r.from = &from + return r +} + +func (r *RateRequest) To(to time.Time) *RateRequest { + r.to = &to + return r +} + +func (r *RateRequest) Group(group Group) *RateRequest { + r.group = &group + return r +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (r *RateRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (r *RateRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check source field -> json key source + source := r.source + + // assign parameter of source + params["source"] = source + // check target field -> json key target + target := r.target + + // assign parameter of target + params["target"] = target + // check time field -> json key time + if r.time != nil { + time := *r.time + + // assign parameter of time + params["time"] = time.Format("2006-01-02T15:04:05-0700") + } else { + } + // check from field -> json key from + if r.from != nil { + from := *r.from + + // assign parameter of from + params["from"] = from.Format("2006-01-02T15:04:05-0700") + } else { + } + // check to field -> json key to + if r.to != nil { + to := *r.to + + // assign parameter of to + params["to"] = to.Format("2006-01-02T15:04:05-0700") + } else { + } + // check group field -> json key group + if r.group != nil { + group := *r.group + + // assign parameter of group + params["group"] = group + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (r *RateRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := r.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if r.isVarSlice(_v) { + r.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (r *RateRequest) GetParametersJSON() ([]byte, error) { + params, err := r.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (r *RateRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (r *RateRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (r *RateRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (r *RateRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (r *RateRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := r.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (r *RateRequest) GetPath() string { + return "/v1/rates" +} + +// Do generates the request object and send the request object to the API endpoint +func (r *RateRequest) Do(ctx context.Context) ([]Rate, error) { + + // empty params for GET operation + var params interface{} + query, err := r.GetParametersQuery() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = r.GetPath() + + req, err := r.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := r.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse []Rate + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + return apiResponse, nil +} diff --git a/pkg/datasource/wise/time.go b/pkg/datasource/wise/time.go new file mode 100644 index 0000000000..ba12abd7d3 --- /dev/null +++ b/pkg/datasource/wise/time.go @@ -0,0 +1,34 @@ +package wise + +import ( + "encoding/json" + "time" +) + +const layout = "2006-01-02T15:04:05-0700" + +type Time time.Time + +func (t Time) Time() time.Time { + return time.Time(t) +} + +func (t Time) String() string { + return time.Time(t).Format(layout) +} + +func (t *Time) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + parsed, err := time.Parse(layout, s) + if err != nil { + return err + } + + *t = Time(parsed) + return nil +} From b0476e40ed3e19c9980995e7da775fa0511a32c3 Mon Sep 17 00:00:00 2001 From: narumi Date: Sat, 11 Nov 2023 10:27:06 +0800 Subject: [PATCH 187/422] go mod tidy --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index 56327767ef..dbd2e6cef5 100644 --- a/go.sum +++ b/go.sum @@ -82,10 +82,6 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/c-bata/goptuna v0.8.1 h1:25+n1MLv0yvCsD56xv4nqIus3oLHL9GuPAZDLIqmX1U= github.com/c-bata/goptuna v0.8.1/go.mod h1:knmS8+Iyq5PPy1YUeIEq0pMFR4Y6x7z/CySc9HlZTCY= -github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY= -github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4= -github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs= -github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/requestgen v1.3.6 h1:ul7dZ2uwGYjNBjreooRfSY10WTXvQmQSjZsHebz6QfE= github.com/c9s/requestgen v1.3.6/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs= From 4f94f7acc0838c2f7043ee28bf880699b0473eb5 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 14 Nov 2023 11:23:30 +0800 Subject: [PATCH 188/422] pkg/exchange: implement order trade user stream --- pkg/exchange/bitget/convert.go | 117 ++++++ pkg/exchange/bitget/convert_test.go | 449 ++++++++++++++++++++++++ pkg/exchange/bitget/stream.go | 78 +++- pkg/exchange/bitget/stream_callbacks.go | 10 + pkg/exchange/bitget/stream_test.go | 6 + pkg/exchange/bitget/types.go | 62 ++++ 6 files changed, 718 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 3a8a875dce..4098e89102 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -362,3 +362,120 @@ func toGlobalKLines(symbol string, interval types.Interval, kLines v2.KLineRespo } return gKLines } + +func toGlobalTimeInForce(force v2.OrderForce) (types.TimeInForce, error) { + switch force { + case v2.OrderForceFOK: + return types.TimeInForceFOK, nil + + case v2.OrderForceGTC, v2.OrderForcePostOnly: + return types.TimeInForceGTC, nil + + case v2.OrderForceIOC: + return types.TimeInForceIOC, nil + + default: + return "", fmt.Errorf("unexpected time-in-force: %s", force) + } +} + +func (o *Order) processMarketBuyQuantity() (fixedpoint.Value, error) { + switch o.Status { + case v2.OrderStatusLive, v2.OrderStatusNew, v2.OrderStatusInit, v2.OrderStatusCancelled: + return fixedpoint.Zero, nil + + case v2.OrderStatusPartialFilled: + if o.FillPrice.IsZero() { + return fixedpoint.Zero, fmt.Errorf("fillPrice for a partialFilled should not be zero") + } + return o.Size.Div(o.FillPrice), nil + + case v2.OrderStatusFilled: + return o.AccBaseVolume, nil + + default: + return fixedpoint.Zero, fmt.Errorf("unexpected status: %s", o.Status) + } +} + +func (o *Order) toGlobalOrder() (types.Order, error) { + side, err := toGlobalSideType(o.Side) + if err != nil { + return types.Order{}, err + } + + orderType, err := toGlobalOrderType(o.OrderType) + if err != nil { + return types.Order{}, err + } + + timeInForce, err := toGlobalTimeInForce(o.Force) + if err != nil { + return types.Order{}, err + } + + status, err := toGlobalOrderStatus(o.Status) + if err != nil { + return types.Order{}, err + } + + qty := o.Size + if orderType == types.OrderTypeMarket && side == types.SideTypeBuy { + qty, err = o.processMarketBuyQuantity() + if err != nil { + return types.Order{}, err + } + } + + return types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: o.ClientOrderId, + Symbol: o.InstId, + Side: side, + Type: orderType, + Quantity: qty, + Price: o.PriceAvg, + TimeInForce: timeInForce, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(o.OrderId), + UUID: strconv.FormatInt(int64(o.OrderId), 10), + Status: status, + ExecutedQuantity: o.AccBaseVolume, + IsWorking: o.Status.IsWorking(), + CreationTime: types.Time(o.CTime.Time()), + UpdateTime: types.Time(o.UTime.Time()), + }, nil +} + +func (o *Order) toGlobalTrade() (types.Trade, error) { + if o.Status != v2.OrderStatusPartialFilled { + return types.Trade{}, fmt.Errorf("failed to convert to global trade, unexpected status: %s", o.Status) + } + + side, err := toGlobalSideType(o.Side) + if err != nil { + return types.Trade{}, err + } + + isMaker, err := o.isMaker() + if err != nil { + return types.Trade{}, err + } + + return types.Trade{ + ID: uint64(o.TradeId), + OrderID: uint64(o.OrderId), + Exchange: types.ExchangeBitget, + Price: o.FillPrice, + Quantity: o.BaseVolume, + QuoteQuantity: o.FillPrice.Mul(o.BaseVolume), + Symbol: o.InstId, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: isMaker, + Time: types.Time(o.FillTime), + Fee: o.FillFee.Abs(), + FeeCurrency: o.FillFeeCoin, + }, nil +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index dff77b4360..57b5366fdc 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -686,3 +686,452 @@ func Test_toGlobalKLines(t *testing.T) { assert.Equal(t, toGlobalKLines(symbol, interval, resp), expKlines) } + +func Test_toGlobalTimeInForce(t *testing.T) { + force, err := toGlobalTimeInForce(v2.OrderForceFOK) + assert.NoError(t, err) + assert.Equal(t, types.TimeInForceFOK, force) + + force, err = toGlobalTimeInForce(v2.OrderForceGTC) + assert.NoError(t, err) + assert.Equal(t, types.TimeInForceGTC, force) + + force, err = toGlobalTimeInForce(v2.OrderForcePostOnly) + assert.NoError(t, err) + assert.Equal(t, types.TimeInForceGTC, force) + + force, err = toGlobalTimeInForce(v2.OrderForceIOC) + assert.NoError(t, err) + assert.Equal(t, types.TimeInForceIOC, force) + + _, err = toGlobalTimeInForce("xxx") + assert.ErrorContains(t, err, "xxx") +} + +func TestOrder_processMarketBuyQuantity(t *testing.T) { + t.Run("zero qty", func(t *testing.T) { + o := Order{} + for _, s := range []v2.OrderStatus{v2.OrderStatusLive, v2.OrderStatusNew, v2.OrderStatusInit, v2.OrderStatusCancelled} { + o.Status = s + qty, err := o.processMarketBuyQuantity() + assert.NoError(t, err) + assert.Equal(t, fixedpoint.Zero, qty) + } + }) + + t.Run("calculate qty", func(t *testing.T) { + o := Order{ + Size: fixedpoint.NewFromFloat(2), + Trade: Trade{ + FillPrice: fixedpoint.NewFromFloat(1), + }, + Status: v2.OrderStatusPartialFilled, + } + qty, err := o.processMarketBuyQuantity() + assert.NoError(t, err) + assert.Equal(t, fixedpoint.NewFromFloat(2), qty) + }) + + t.Run("return accumulated balance", func(t *testing.T) { + o := Order{ + AccBaseVolume: fixedpoint.NewFromFloat(5), + Status: v2.OrderStatusFilled, + } + qty, err := o.processMarketBuyQuantity() + assert.NoError(t, err) + assert.Equal(t, fixedpoint.NewFromFloat(5), qty) + }) + + t.Run("unexpected status", func(t *testing.T) { + o := Order{ + Status: "xxx", + } + _, err := o.processMarketBuyQuantity() + assert.ErrorContains(t, err, "xxx") + }) +} + +func TestOrder_toGlobalOrder(t *testing.T) { + o := Order{ + Trade: Trade{ + FillPrice: fixedpoint.NewFromFloat(0.49016), + TradeId: types.StrInt64(1107950490073112582), + BaseVolume: fixedpoint.NewFromFloat(33.6558), + FillTime: types.NewMillisecondTimestampFromInt(1699881902235), + FillFee: fixedpoint.NewFromFloat(-0.0336558), + FillFeeCoin: "BGB", + TradeScope: "T", + }, + InstId: "BGBUSDT", + OrderId: types.StrInt64(1107950489998626816), + ClientOrderId: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Size: fixedpoint.NewFromFloat(39.0), + Notional: fixedpoint.NewFromFloat(39.0), + OrderType: v2.OrderTypeMarket, + Force: v2.OrderForceGTC, + Side: v2.SideTypeBuy, + AccBaseVolume: fixedpoint.NewFromFloat(33.6558), + PriceAvg: fixedpoint.NewFromFloat(0.49016), + Status: v2.OrderStatusPartialFilled, + CTime: types.NewMillisecondTimestampFromInt(1699881902217), + UTime: types.NewMillisecondTimestampFromInt(1699881902248), + FeeDetail: nil, + EnterPointSource: "API", + } + + // market buy example: + // { + // "instId":"BGBUSDT", + // "orderId":"1107950489998626816", + // "clientOid":"cc73aab9-1e44-4022-8458-60d8c6a08753", + // "size":"39.0000", + // "notional":"39.000000", + // "orderType":"market", + // "force":"gtc", + // "side":"buy", + // "fillPrice":"0.49016", + // "tradeId":"1107950490073112582", + // "baseVolume":"33.6558", + // "fillTime":"1699881902235", + // "fillFee":"-0.0336558", + // "fillFeeCoin":"BGB", + // "tradeScope":"T", + // "accBaseVolume":"33.6558", + // "priceAvg":"0.49016", + // "status":"partially_filled", + // "cTime":"1699881902217", + // "uTime":"1699881902248", + // "feeDetail":[ + // { + // "feeCoin":"BGB", + // "fee":"-0.0336558" + // } + // ], + // "enterPointSource":"API" + // } + t.Run("market buy", func(t *testing.T) { + newO := o + res, err := newO.toGlobalOrder() + assert.NoError(t, err) + assert.Equal(t, types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Symbol: "BGBUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeMarket, + Quantity: newO.Size.Div(newO.FillPrice), + Price: newO.PriceAvg, + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(newO.OrderId), + UUID: strconv.FormatInt(int64(newO.OrderId), 10), + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: newO.AccBaseVolume, + IsWorking: newO.Status.IsWorking(), + CreationTime: types.Time(newO.CTime), + UpdateTime: types.Time(newO.UTime), + }, res) + }) + + // market sell example: + // { + // "instId":"BGBUSDT", + // "orderId":"1107940456212631553", + // "clientOid":"088bb971-858e-48e2-b503-85c3274edd89", + // "size":"285.0000", + // "orderType":"market", + // "force":"gtc", + // "side":"sell", + // "fillPrice":"0.48706", + // "tradeId":"1107940456278728706", + // "baseVolume":"22.5840", + // "fillTime":"1699879509992", + // "fillFee":"-0.01099976304", + // "fillFeeCoin":"USDT", + // "tradeScope":"T", + // "accBaseVolume":"45.1675", + // "priceAvg":"0.48706", + // "status":"partially_filled", + // "cTime":"1699879509976", + // "uTime":"1699879510007", + // "feeDetail":[ + // { + // "feeCoin":"USDT", + // "fee":"-0.02199928255" + // } + // ], + // "enterPointSource":"API" + // } + t.Run("market sell", func(t *testing.T) { + newO := o + newO.OrderType = v2.OrderTypeMarket + newO.Side = v2.SideTypeSell + + res, err := newO.toGlobalOrder() + assert.NoError(t, err) + assert.Equal(t, types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Symbol: "BGBUSDT", + Side: types.SideTypeSell, + Type: types.OrderTypeMarket, + Quantity: newO.Size, + Price: newO.PriceAvg, + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(newO.OrderId), + UUID: strconv.FormatInt(int64(newO.OrderId), 10), + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: newO.AccBaseVolume, + IsWorking: newO.Status.IsWorking(), + CreationTime: types.Time(newO.CTime), + UpdateTime: types.Time(newO.UTime), + }, res) + }) + + // limit buy example: + // { + // "instId":"BGBUSDT", + // "orderId":"1107955329902481408", + // "clientOid":"c578164a-bf34-44ba-8bb7-a1538f33b1b8", + // "price":"0.49998", + // "size":"24.9990", + // "notional":"24.999000", + // "orderType":"limit", + // "force":"gtc", + // "side":"buy", + // "fillPrice":"0.49998", + // "tradeId":"1107955401758285828", + // "baseVolume":"15.9404", + // "fillTime":"1699883073272", + // "fillFee":"-0.0159404", + // "fillFeeCoin":"BGB", + // "tradeScope":"M", + // "accBaseVolume":"15.9404", + // "priceAvg":"0.49998", + // "status":"partially_filled", + // "cTime":"1699883056140", + // "uTime":"1699883073285", + // "feeDetail":[ + // { + // "feeCoin":"BGB", + // "fee":"-0.0159404" + // } + // ], + // "enterPointSource":"API" + // } + t.Run("limit buy", func(t *testing.T) { + newO := o + newO.OrderType = v2.OrderTypeLimit + + res, err := newO.toGlobalOrder() + assert.NoError(t, err) + assert.Equal(t, types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Symbol: "BGBUSDT", + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: newO.Size, + Price: newO.PriceAvg, + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(newO.OrderId), + UUID: strconv.FormatInt(int64(newO.OrderId), 10), + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: newO.AccBaseVolume, + IsWorking: newO.Status.IsWorking(), + CreationTime: types.Time(newO.CTime), + UpdateTime: types.Time(newO.UTime), + }, res) + }) + + // limit sell example: + // { + // "instId":"BGBUSDT", + // "orderId":"1107936497259417600", + // "clientOid":"02d4592e-091c-4b5a-aef3-6a7cf57b5e82", + // "price":"0.48710", + // "size":"280.0000", + // "orderType":"limit", + // "force":"gtc", + // "side":"sell", + // "fillPrice":"0.48710", + // "tradeId":"1107937053540556809", + // "baseVolume":"41.0593", + // "fillTime":"1699878698716", + // "fillFee":"-0.01999998503", + // "fillFeeCoin":"USDT", + // "tradeScope":"M", + // "accBaseVolume":"146.3209", + // "priceAvg":"0.48710", + // "status":"partially_filled", + // "cTime":"1699878566088", + // "uTime":"1699878698746", + // "feeDetail":[ + // { + // "feeCoin":"USDT", + // "fee":"-0.07127291039" + // } + // ], + // "enterPointSource":"API" + // } + t.Run("limit sell", func(t *testing.T) { + newO := o + newO.OrderType = v2.OrderTypeLimit + newO.Side = v2.SideTypeSell + + res, err := newO.toGlobalOrder() + assert.NoError(t, err) + assert.Equal(t, types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Symbol: "BGBUSDT", + Side: types.SideTypeSell, + Type: types.OrderTypeLimit, + Quantity: newO.Size, + Price: newO.PriceAvg, + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeBitget, + OrderID: uint64(newO.OrderId), + UUID: strconv.FormatInt(int64(newO.OrderId), 10), + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: newO.AccBaseVolume, + IsWorking: newO.Status.IsWorking(), + CreationTime: types.Time(newO.CTime), + UpdateTime: types.Time(newO.UTime), + }, res) + }) + + t.Run("unexpected status", func(t *testing.T) { + newO := o + newO.Status = "xxx" + _, err := newO.toGlobalOrder() + assert.ErrorContains(t, err, "xxx") + }) + + t.Run("unexpected time-in-force", func(t *testing.T) { + newO := o + newO.Force = "xxx" + _, err := newO.toGlobalOrder() + assert.ErrorContains(t, err, "xxx") + }) + + t.Run("unexpected order type", func(t *testing.T) { + newO := o + newO.OrderType = "xxx" + _, err := newO.toGlobalOrder() + assert.ErrorContains(t, err, "xxx") + }) + + t.Run("unexpected side", func(t *testing.T) { + newO := o + newO.Side = "xxx" + _, err := newO.toGlobalOrder() + assert.ErrorContains(t, err, "xxx") + }) +} + +func TestOrder_toGlobalTrade(t *testing.T) { + // market buy example: + // { + // "instId":"BGBUSDT", + // "orderId":"1107950489998626816", + // "clientOid":"cc73aab9-1e44-4022-8458-60d8c6a08753", + // "size":"39.0000", + // "notional":"39.000000", + // "orderType":"market", + // "force":"gtc", + // "side":"buy", + // "fillPrice":"0.49016", + // "tradeId":"1107950490073112582", + // "baseVolume":"33.6558", + // "fillTime":"1699881902235", + // "fillFee":"-0.0336558", + // "fillFeeCoin":"BGB", + // "tradeScope":"T", + // "accBaseVolume":"33.6558", + // "priceAvg":"0.49016", + // "status":"partially_filled", + // "cTime":"1699881902217", + // "uTime":"1699881902248", + // "feeDetail":[ + // { + // "feeCoin":"BGB", + // "fee":"-0.0336558" + // } + // ], + // "enterPointSource":"API" + // } + o := Order{ + Trade: Trade{ + FillPrice: fixedpoint.NewFromFloat(0.49016), + TradeId: types.StrInt64(1107950490073112582), + BaseVolume: fixedpoint.NewFromFloat(33.6558), + FillTime: types.NewMillisecondTimestampFromInt(1699881902235), + FillFee: fixedpoint.NewFromFloat(-0.0336558), + FillFeeCoin: "BGB", + TradeScope: "T", + }, + InstId: "BGBUSDT", + OrderId: types.StrInt64(1107950489998626816), + ClientOrderId: "cc73aab9-1e44-4022-8458-60d8c6a08753", + Size: fixedpoint.NewFromFloat(39.0), + Notional: fixedpoint.NewFromFloat(39.0), + OrderType: v2.OrderTypeMarket, + Force: v2.OrderForceGTC, + Side: v2.SideTypeBuy, + AccBaseVolume: fixedpoint.NewFromFloat(33.6558), + PriceAvg: fixedpoint.NewFromFloat(0.49016), + Status: v2.OrderStatusPartialFilled, + CTime: types.NewMillisecondTimestampFromInt(1699881902217), + UTime: types.NewMillisecondTimestampFromInt(1699881902248), + FeeDetail: nil, + EnterPointSource: "API", + } + + t.Run("succeeds", func(t *testing.T) { + res, err := o.toGlobalTrade() + assert.NoError(t, err) + assert.Equal(t, types.Trade{ + ID: uint64(o.TradeId), + OrderID: uint64(o.OrderId), + Exchange: types.ExchangeBitget, + Price: o.FillPrice, + Quantity: o.BaseVolume, + QuoteQuantity: o.FillPrice.Mul(o.BaseVolume), + Symbol: "BGBUSDT", + Side: types.SideTypeBuy, + IsBuyer: true, + IsMaker: false, + Time: types.Time(o.FillTime), + Fee: o.FillFee.Abs(), + FeeCurrency: "BGB", + }, res) + }) + + t.Run("unexpected trade scope", func(t *testing.T) { + newO := o + newO.TradeScope = "xxx" + _, err := newO.toGlobalTrade() + assert.ErrorContains(t, err, "xxx") + }) + + t.Run("unexpected side type", func(t *testing.T) { + newO := o + newO.Side = "xxx" + _, err := newO.toGlobalTrade() + assert.ErrorContains(t, err, "xxx") + }) + + t.Run("unexpected side type", func(t *testing.T) { + newO := o + newO.Status = "xxx" + _, err := newO.toGlobalTrade() + assert.ErrorContains(t, err, "xxx") + }) +} diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 00ecae4263..bdab89774c 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -25,12 +25,15 @@ var ( type Stream struct { types.StandardStream + privateChannelSymbols []string + key, secret, passphrase string bookEventCallbacks []func(o BookEvent) marketTradeEventCallbacks []func(o MarketTradeEvent) KLineEventCallbacks []func(o KLineEvent) - accountEventCallbacks []func(e AccountEvent) + accountEventCallbacks []func(e AccountEvent) + orderTradeEventCallbacks []func(e OrderTradeEvent) lastCandle map[string]types.KLine } @@ -56,6 +59,7 @@ func NewStream(key, secret, passphrase string) *Stream { stream.OnAuth(stream.handleAuth) stream.OnAccountEvent(stream.handleAccountEvent) + stream.OnOrderTradeEvent(stream.handleOrderTradeEvent) return stream } @@ -129,25 +133,52 @@ func (s *Stream) dispatchEvent(event interface{}) { case *AccountEvent: s.EmitAccountEvent(*e) + case *OrderTradeEvent: + s.EmitOrderTradeEvent(*e) + + case []byte: + // We only handle the 'pong' case. Others are unexpected. + if !bytes.Equal(e, pongBytes) { + log.Errorf("invalid event: %q", e) + } } } +// handleAuth subscribe private stream channels. Because Bitget doesn't allow authentication and subscription to be used +// consecutively, we subscribe after authentication confirmation. func (s *Stream) handleAuth() { - if err := s.Conn.WriteJSON(WsOp{ + op := WsOp{ Op: WsEventSubscribe, Args: []WsArg{ { InstType: instSpV2, Channel: ChannelAccount, - Coin: "default", // default all + Coin: "default", // all coins }, }, - }); err != nil { + } + if len(s.privateChannelSymbols) > 0 { + for _, symbol := range s.privateChannelSymbols { + op.Args = append(op.Args, WsArg{ + InstType: instSpV2, + Channel: ChannelOrders, + InstId: symbol, + }) + } + } else { + log.Warnf("you have not subscribed to any order channels") + } + + if err := s.Conn.WriteJSON(op); err != nil { log.WithError(err).Error("failed to send subscription request") return } } +func (s *Stream) SetPrivateChannelSymbols(symbols []string) { + s.privateChannelSymbols = symbols +} + func (s *Stream) handlerConnect() { if s.PublicOnly { // errors are handled in the syncSubscriptions, so they are skipped here. @@ -279,6 +310,17 @@ func parseEvent(in []byte) (interface{}, error) { book.instId = event.Arg.InstId return &book, nil + case ChannelOrders: + var order OrderTradeEvent + err = json.Unmarshal(event.Data, &order.Orders) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into OrderTradeEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + order.actionType = event.Action + order.instId = event.Arg.InstId + return &order, nil + case ChannelTrade: var trade MarketTradeEvent err = json.Unmarshal(event.Data, &trade.Events) @@ -364,3 +406,31 @@ func (s *Stream) handleAccountEvent(m AccountEvent) { } s.StandardStream.EmitBalanceSnapshot(balanceMap) } + +func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { + if len(m.Orders) == 0 { + return + } + + for _, order := range m.Orders { + globalOrder, err := order.toGlobalOrder() + if err != nil { + log.Errorf("failed to convert order to global: %s", err) + continue + } + // The bitget support only snapshot on orders channel, so we use snapshot as update to emit data. + if m.actionType != ActionTypeSnapshot { + continue + } + s.StandardStream.EmitOrderUpdate(globalOrder) + + if globalOrder.Status == types.OrderStatusPartiallyFilled { + trade, err := order.toGlobalTrade() + if err != nil { + log.Errorf("failed to convert trade to global: %s", err) + continue + } + s.StandardStream.EmitTradeUpdate(trade) + } + } +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go index 44661171ec..01dea0f5fe 100644 --- a/pkg/exchange/bitget/stream_callbacks.go +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -43,3 +43,13 @@ func (s *Stream) EmitAccountEvent(e AccountEvent) { cb(e) } } + +func (s *Stream) OnOrderTradeEvent(cb func(e OrderTradeEvent)) { + s.orderTradeEventCallbacks = append(s.orderTradeEventCallbacks, cb) +} + +func (s *Stream) EmitOrderTradeEvent(e OrderTradeEvent) { + for _, cb := range s.orderTradeEventCallbacks { + cb(e) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index 273941e7da..011210abeb 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -134,6 +134,12 @@ func TestStream(t *testing.T) { s.OnBalanceUpdate(func(balances types.BalanceMap) { t.Log("get update", balances) }) + s.OnOrderUpdate(func(order types.Order) { + t.Log("order update", order) + }) + s.OnTradeUpdate(func(trade types.Trade) { + t.Log("trade update", trade) + }) c := make(chan struct{}) <-c diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index 19c87c4846..06425c64fc 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -29,6 +30,7 @@ const ( // ChannelOrderBook15 top 15 order book of "books" that begins from bid1/ask1 ChannelOrderBook15 ChannelType = "books15" ChannelTrade ChannelType = "trade" + ChannelOrders ChannelType = "orders" ) type WsArg struct { @@ -460,3 +462,63 @@ type AccountEvent struct { actionType ActionType instId string } + +type Trade struct { + // Latest filled price + FillPrice fixedpoint.Value `json:"fillPrice"` + TradeId types.StrInt64 `json:"tradeId"` + // Number of latest filled orders + BaseVolume fixedpoint.Value `json:"baseVolume"` + FillTime types.MillisecondTimestamp `json:"fillTime"` + // Transaction fee of the latest transaction, negative value + FillFee fixedpoint.Value `json:"fillFee"` + // Currency of transaction fee of the latest transaction + FillFeeCoin string `json:"fillFeeCoin"` + // Direction of liquidity of the latest transaction + TradeScope string `json:"tradeScope"` +} + +type Order struct { + Trade + + InstId string `json:"instId"` + // OrderId are always numeric. It's confirmed with official customer service. https://t.me/bitgetOpenapi/24172 + OrderId types.StrInt64 `json:"orderId"` + ClientOrderId string `json:"clientOid"` + // Size is base coin when orderType=limit; quote coin when orderType=market + Size fixedpoint.Value `json:"size"` + // Buy amount, returned when buying at market price + Notional fixedpoint.Value `json:"notional"` + OrderType v2.OrderType `json:"orderType"` + Force v2.OrderForce `json:"force"` + Side v2.SideType `json:"side"` + AccBaseVolume fixedpoint.Value `json:"accBaseVolume"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + Status v2.OrderStatus `json:"status"` + CTime types.MillisecondTimestamp `json:"cTime"` + UTime types.MillisecondTimestamp `json:"uTime"` + FeeDetail []struct { + FeeCoin string `json:"feeCoin"` + Fee string `json:"fee"` + } `json:"feeDetail"` + EnterPointSource string `json:"enterPointSource"` +} + +func (o *Order) isMaker() (bool, error) { + switch o.TradeScope { + case "T": + return false, nil + case "M": + return true, nil + default: + return false, fmt.Errorf("unexpected trade scope: %s", o.TradeScope) + } +} + +type OrderTradeEvent struct { + Orders []Order + + // internal use + actionType ActionType + instId string +} From cf527a6f05873ddd8a0554f916b226888305d6b3 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 15 Nov 2023 10:46:18 +0800 Subject: [PATCH 189/422] pkg/exchange: make the CTime and UTime to qualified name --- .../v2/get_history_orders_request.go | 4 +- .../v2/get_history_orders_request_test.go | 8 ++-- .../bitget/bitgetapi/v2/get_trade_fills.go | 26 ++++++------ .../v2/get_unfilled_orders_request.go | 4 +- pkg/exchange/bitget/convert.go | 14 +++---- pkg/exchange/bitget/convert_test.go | 40 +++++++++---------- pkg/exchange/bitget/exchange.go | 4 +- pkg/exchange/bitget/types.go | 6 +-- 8 files changed, 53 insertions(+), 53 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go index 51807eb5ed..ac6f52ac1a 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go @@ -51,8 +51,8 @@ type OrderDetail struct { // The value is json string, so we unmarshal it after unmarshal OrderDetail FeeDetailRaw string `json:"feeDetail"` OrderSource string `json:"orderSource"` - CTime types.MillisecondTimestamp `json:"cTime"` - UTime types.MillisecondTimestamp `json:"uTime"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` FeeDetail FeeDetail } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go index 07e319faba..2e3cf9600a 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_test.go @@ -53,8 +53,8 @@ func TestOrderDetail_UnmarshalJSON(t *testing.T) { EnterPointSource: "API", FeeDetailRaw: "", OrderSource: "normal", - CTime: types.NewMillisecondTimestampFromInt(1699021576683), - UTime: types.NewMillisecondTimestampFromInt(1699021649099), + CreatedTime: types.NewMillisecondTimestampFromInt(1699021576683), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699021649099), FeeDetail: FeeDetail{}, }, od) }) @@ -98,8 +98,8 @@ func TestOrderDetail_UnmarshalJSON(t *testing.T) { EnterPointSource: "API", FeeDetailRaw: `{"newFees":{"c":0,"d":0,"deduction":false,"r":-0.0070005,"t":-0.0070005,"totalDeductionFee":0},"USDT":{"deduction":false,"feeCoinCode":"USDT","totalDeductionFee":0,"totalFee":-0.007000500000}}`, OrderSource: "normal", - CTime: types.NewMillisecondTimestampFromInt(1699020564659), - UTime: types.NewMillisecondTimestampFromInt(1699020564688), + CreatedTime: types.NewMillisecondTimestampFromInt(1699020564659), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699020564688), FeeDetail: FeeDetail{ NewFees: struct { DeductedByCoupon fixedpoint.Value `json:"c"` diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go index 358bbda278..59111360cb 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go @@ -36,19 +36,19 @@ type TradeFee struct { } type Trade struct { - UserId types.StrInt64 `json:"userId"` - Symbol string `json:"symbol"` - OrderId types.StrInt64 `json:"orderId"` - TradeId types.StrInt64 `json:"tradeId"` - OrderType OrderType `json:"orderType"` - Side SideType `json:"side"` - PriceAvg fixedpoint.Value `json:"priceAvg"` - Size fixedpoint.Value `json:"size"` - Amount fixedpoint.Value `json:"amount"` - FeeDetail TradeFee `json:"feeDetail"` - TradeScope TradeScope `json:"tradeScope"` - CTime types.MillisecondTimestamp `json:"cTime"` - UTime types.MillisecondTimestamp `json:"uTime"` + UserId types.StrInt64 `json:"userId"` + Symbol string `json:"symbol"` + OrderId types.StrInt64 `json:"orderId"` + TradeId types.StrInt64 `json:"tradeId"` + OrderType OrderType `json:"orderType"` + Side SideType `json:"side"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + Size fixedpoint.Value `json:"size"` + Amount fixedpoint.Value `json:"amount"` + FeeDetail TradeFee `json:"feeDetail"` + TradeScope TradeScope `json:"tradeScope"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` } //go:generate GetRequest -url "/api/v2/spot/trade/fills" -type GetTradeFillsRequest -responseDataType []Trade diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go index 178b31bbac..2af0aac17b 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go @@ -27,8 +27,8 @@ type UnfilledOrder struct { QuoteVolume fixedpoint.Value `json:"quoteVolume"` EnterPointSource string `json:"enterPointSource"` OrderSource string `json:"orderSource"` - CTime types.MillisecondTimestamp `json:"cTime"` - UTime types.MillisecondTimestamp `json:"uTime"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` } //go:generate GetRequest -url "/api/v2/spot/trade/unfilled-orders" -type GetUnfilledOrdersRequest -responseDataType []UnfilledOrder diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 4098e89102..0b6a4b9216 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -158,7 +158,7 @@ func toGlobalTrade(trade v2.Trade) (*types.Trade, error) { Side: side, IsBuyer: side == types.SideTypeBuy, IsMaker: isMaker, - Time: types.Time(trade.CTime), + Time: types.Time(trade.CreatedTime), Fee: trade.FeeDetail.TotalFee.Abs(), FeeCurrency: trade.FeeDetail.FeeCoin, FeeDiscounted: isDiscount, @@ -211,8 +211,8 @@ func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { Status: status, ExecutedQuantity: order.BaseVolume, IsWorking: order.Status.IsWorking(), - CreationTime: types.Time(order.CTime.Time()), - UpdateTime: types.Time(order.UTime.Time()), + CreationTime: types.Time(order.CreatedTime.Time()), + UpdateTime: types.Time(order.UpdatedTime.Time()), }, nil } @@ -262,8 +262,8 @@ func toGlobalOrder(order v2.OrderDetail) (*types.Order, error) { Status: status, ExecutedQuantity: order.BaseVolume, IsWorking: order.Status.IsWorking(), - CreationTime: types.Time(order.CTime.Time()), - UpdateTime: types.Time(order.UTime.Time()), + CreationTime: types.Time(order.CreatedTime.Time()), + UpdateTime: types.Time(order.UpdatedTime.Time()), }, nil } @@ -443,8 +443,8 @@ func (o *Order) toGlobalOrder() (types.Order, error) { Status: status, ExecutedQuantity: o.AccBaseVolume, IsWorking: o.Status.IsWorking(), - CreationTime: types.Time(o.CTime.Time()), - UpdateTime: types.Time(o.UTime.Time()), + CreationTime: types.Time(o.CreatedTime.Time()), + UpdateTime: types.Time(o.UpdatedTime.Time()), }, nil } diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index 57b5366fdc..da083929f5 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -222,8 +222,8 @@ func Test_unfilledOrderToGlobalOrder(t *testing.T) { QuoteVolume: fixedpoint.NewFromFloat(0), EnterPointSource: "API", OrderSource: "normal", - CTime: types.NewMillisecondTimestampFromInt(1660704288118), - UTime: types.NewMillisecondTimestampFromInt(1660704288118), + CreatedTime: types.NewMillisecondTimestampFromInt(1660704288118), + UpdatedTime: types.NewMillisecondTimestampFromInt(1660704288118), } ) @@ -296,8 +296,8 @@ func Test_toGlobalOrder(t *testing.T) { EnterPointSource: "API", FeeDetailRaw: `{\"newFees\":{\"c\":0,\"d\":0,\"deduction\":false,\"r\":-0.0070005,\"t\":-0.0070005,\"totalDeductionFee\":0},\"USDT\":{\"deduction\":false,\"feeCoinCode\":\"USDT\",\"totalDeductionFee\":0,\"totalFee\":-0.007000500000}}`, OrderSource: "normal", - CTime: types.NewMillisecondTimestampFromInt(1660704288118), - UTime: types.NewMillisecondTimestampFromInt(1660704288118), + CreatedTime: types.NewMillisecondTimestampFromInt(1660704288118), + UpdatedTime: types.NewMillisecondTimestampFromInt(1660704288118), } expOrder = &types.Order{ @@ -558,9 +558,9 @@ func Test_toGlobalTrade(t *testing.T) { TotalDeductionFee: fixedpoint.Zero, TotalFee: fixedpoint.NewFromFloat(-0.0070005), }, - TradeScope: v2.TradeTaker, - CTime: types.NewMillisecondTimestampFromInt(1699020564676), - UTime: types.NewMillisecondTimestampFromInt(1699020564687), + TradeScope: v2.TradeTaker, + CreatedTime: types.NewMillisecondTimestampFromInt(1699020564676), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699020564687), } res, err := toGlobalTrade(trade) @@ -597,7 +597,7 @@ func Test_toGlobalBalanceMap(t *testing.T) { Frozen: fixedpoint.NewFromFloat(0.6), Locked: fixedpoint.NewFromFloat(0.7), LimitAvailable: fixedpoint.Zero, - UTime: types.NewMillisecondTimestampFromInt(1699020564676), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699020564676), }, })) } @@ -773,8 +773,8 @@ func TestOrder_toGlobalOrder(t *testing.T) { AccBaseVolume: fixedpoint.NewFromFloat(33.6558), PriceAvg: fixedpoint.NewFromFloat(0.49016), Status: v2.OrderStatusPartialFilled, - CTime: types.NewMillisecondTimestampFromInt(1699881902217), - UTime: types.NewMillisecondTimestampFromInt(1699881902248), + CreatedTime: types.NewMillisecondTimestampFromInt(1699881902217), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699881902248), FeeDetail: nil, EnterPointSource: "API", } @@ -829,8 +829,8 @@ func TestOrder_toGlobalOrder(t *testing.T) { Status: types.OrderStatusPartiallyFilled, ExecutedQuantity: newO.AccBaseVolume, IsWorking: newO.Status.IsWorking(), - CreationTime: types.Time(newO.CTime), - UpdateTime: types.Time(newO.UTime), + CreationTime: types.Time(newO.CreatedTime), + UpdateTime: types.Time(newO.UpdatedTime), }, res) }) @@ -886,8 +886,8 @@ func TestOrder_toGlobalOrder(t *testing.T) { Status: types.OrderStatusPartiallyFilled, ExecutedQuantity: newO.AccBaseVolume, IsWorking: newO.Status.IsWorking(), - CreationTime: types.Time(newO.CTime), - UpdateTime: types.Time(newO.UTime), + CreationTime: types.Time(newO.CreatedTime), + UpdateTime: types.Time(newO.UpdatedTime), }, res) }) @@ -944,8 +944,8 @@ func TestOrder_toGlobalOrder(t *testing.T) { Status: types.OrderStatusPartiallyFilled, ExecutedQuantity: newO.AccBaseVolume, IsWorking: newO.Status.IsWorking(), - CreationTime: types.Time(newO.CTime), - UpdateTime: types.Time(newO.UTime), + CreationTime: types.Time(newO.CreatedTime), + UpdateTime: types.Time(newO.UpdatedTime), }, res) }) @@ -1002,8 +1002,8 @@ func TestOrder_toGlobalOrder(t *testing.T) { Status: types.OrderStatusPartiallyFilled, ExecutedQuantity: newO.AccBaseVolume, IsWorking: newO.Status.IsWorking(), - CreationTime: types.Time(newO.CTime), - UpdateTime: types.Time(newO.UTime), + CreationTime: types.Time(newO.CreatedTime), + UpdateTime: types.Time(newO.UpdatedTime), }, res) }) @@ -1088,8 +1088,8 @@ func TestOrder_toGlobalTrade(t *testing.T) { AccBaseVolume: fixedpoint.NewFromFloat(33.6558), PriceAvg: fixedpoint.NewFromFloat(0.49016), Status: v2.OrderStatusPartialFilled, - CTime: types.NewMillisecondTimestampFromInt(1699881902217), - UTime: types.NewMillisecondTimestampFromInt(1699881902248), + CreatedTime: types.NewMillisecondTimestampFromInt(1699881902217), + UpdatedTime: types.NewMillisecondTimestampFromInt(1699881902248), FeeDetail: nil, EnterPointSource: "API", } diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 064a303e08..3f731402aa 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -394,7 +394,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return orders, nil } -// QueryClosedOrders queries closed order by time range(`CTime`) and id. The order of the response is in descending order. +// QueryClosedOrders queries closed order by time range(`CreatedTime`) and id. The order of the response is in descending order. // If you need to retrieve all data, please utilize the function pkg/exchange/batch.ClosedOrderBatchQuery. // // ** Since is inclusive, Until is exclusive. If you use a time range to query, you must provide both a start time and an end time. ** @@ -495,7 +495,7 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err } // QueryTrades queries fill trades. The trade of the response is in descending order. The time-based query are typically -// using (`CTime`) as the search criteria. +// using (`CreatedTime`) as the search criteria. // If you need to retrieve all data, please utilize the function pkg/exchange/batch.TradeBatchQuery. // // ** StartTime is inclusive, EndTime is exclusive. If you use the EndTime, the StartTime is required. ** diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index 06425c64fc..c9c8468f5e 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -452,7 +452,7 @@ type Balance struct { Locked fixedpoint.Value `json:"locked"` // Restricted availability For spot copy trading LimitAvailable fixedpoint.Value `json:"limitAvailable"` - UTime types.MillisecondTimestamp `json:"uTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` } type AccountEvent struct { @@ -495,8 +495,8 @@ type Order struct { AccBaseVolume fixedpoint.Value `json:"accBaseVolume"` PriceAvg fixedpoint.Value `json:"priceAvg"` Status v2.OrderStatus `json:"status"` - CTime types.MillisecondTimestamp `json:"cTime"` - UTime types.MillisecondTimestamp `json:"uTime"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` FeeDetail []struct { FeeCoin string `json:"feeCoin"` Fee string `json:"fee"` From 687ffe985c3592b61a524b6fe1e166a436ba2934 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 15 Nov 2023 22:20:26 +0800 Subject: [PATCH 190/422] pkg/exchange: use time.Time instead of int64 to represent time --- .../bitget/bitgetapi/v2/client_test.go | 10 +++--- .../v2/get_history_orders_request.go | 9 +++--- .../get_history_orders_request_requestgen.go | 12 ++++--- .../bitget/bitgetapi/v2/get_trade_fills.go | 10 +++--- .../v2/get_trade_fills_request_requestgen.go | 31 +++++++++++++++--- .../v2/get_unfilled_orders_request.go | 10 +++--- .../get_unfilled_orders_request_requestgen.go | 32 ++++++++++++++++--- pkg/exchange/bitget/exchange.go | 8 ++--- 8 files changed, 88 insertions(+), 34 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index afefe7c9e1..f8eef7862b 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -34,15 +34,16 @@ func TestClient(t *testing.T) { ctx := context.Background() t.Run("GetUnfilledOrdersRequest", func(t *testing.T) { - req := client.NewGetUnfilledOrdersRequest().StartTime(1) + startTime := time.Now().Add(-30 * 24 * time.Hour) + req := client.NewGetUnfilledOrdersRequest().StartTime(startTime) resp, err := req.Do(ctx) assert.NoError(t, err) t.Logf("resp: %+v", resp) }) t.Run("GetHistoryOrdersRequest", func(t *testing.T) { - // market buy - req, err := client.NewGetHistoryOrdersRequest().Symbol("APEUSDT").Do(ctx) + startTime := time.Now().Add(-30 * 24 * time.Hour) + req, err := client.NewGetHistoryOrdersRequest().Symbol("APEUSDT").StartTime(startTime).Do(ctx) assert.NoError(t, err) t.Logf("place order resp: %+v", req) @@ -61,7 +62,8 @@ func TestClient(t *testing.T) { }) t.Run("GetTradeFillsRequest", func(t *testing.T) { - req, err := client.NewGetTradeFillsRequest().Symbol("APEUSDT").Do(ctx) + startTime := time.Now().Add(-30 * 24 * time.Hour) + req, err := client.NewGetTradeFillsRequest().Symbol("APEUSDT").StartTime(startTime).Do(ctx) assert.NoError(t, err) t.Logf("get trade fills resp: %+v", req) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go index ac6f52ac1a..478d802e88 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request.go @@ -6,6 +6,7 @@ package bitgetapi import ( "encoding/json" "fmt" + "time" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -91,10 +92,10 @@ type GetHistoryOrdersRequest struct { // Limit number default 100 max 100 limit *string `param:"limit,query"` // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. - idLessThan *string `param:"idLessThan,query"` - startTime *int64 `param:"startTime,query"` - endTime *int64 `param:"endTime,query"` - orderId *string `param:"orderId,query"` + idLessThan *string `param:"idLessThan,query"` + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + orderId *string `param:"orderId,query"` } func (c *Client) NewGetHistoryOrdersRequest() *GetHistoryOrdersRequest { diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go index 398093399d..f5fb447d31 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_history_orders_request_requestgen.go @@ -10,6 +10,8 @@ import ( "net/url" "reflect" "regexp" + "strconv" + "time" ) func (g *GetHistoryOrdersRequest) Symbol(symbol string) *GetHistoryOrdersRequest { @@ -27,12 +29,12 @@ func (g *GetHistoryOrdersRequest) IdLessThan(idLessThan string) *GetHistoryOrder return g } -func (g *GetHistoryOrdersRequest) StartTime(startTime int64) *GetHistoryOrdersRequest { +func (g *GetHistoryOrdersRequest) StartTime(startTime time.Time) *GetHistoryOrdersRequest { g.startTime = &startTime return g } -func (g *GetHistoryOrdersRequest) EndTime(endTime int64) *GetHistoryOrdersRequest { +func (g *GetHistoryOrdersRequest) EndTime(endTime time.Time) *GetHistoryOrdersRequest { g.endTime = &endTime return g } @@ -74,7 +76,8 @@ func (g *GetHistoryOrdersRequest) GetQueryParameters() (url.Values, error) { startTime := *g.startTime // assign parameter of startTime - params["startTime"] = startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check endTime field -> json key endTime @@ -82,7 +85,8 @@ func (g *GetHistoryOrdersRequest) GetQueryParameters() (url.Values, error) { endTime := *g.endTime // assign parameter of endTime - params["endTime"] = endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check orderId field -> json key orderId diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go index 59111360cb..fb91bbfc1a 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills.go @@ -1,6 +1,8 @@ package bitgetapi import ( + "time" + "github.com/c9s/requestgen" "github.com/c9s/bbgo/pkg/fixedpoint" @@ -59,10 +61,10 @@ type GetTradeFillsRequest struct { // Limit number default 100 max 100 limit *string `param:"limit,query"` // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. - idLessThan *string `param:"idLessThan,query"` - startTime *int64 `param:"startTime,query"` - endTime *int64 `param:"endTime,query"` - orderId *string `param:"orderId,query"` + idLessThan *string `param:"idLessThan,query"` + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + orderId *string `param:"orderId,query"` } func (s *Client) NewGetTradeFillsRequest() *GetTradeFillsRequest { diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go index 2f19cf3786..b445834fa8 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_trade_fills_request_requestgen.go @@ -10,6 +10,8 @@ import ( "net/url" "reflect" "regexp" + "strconv" + "time" ) func (s *GetTradeFillsRequest) Symbol(symbol string) *GetTradeFillsRequest { @@ -27,12 +29,12 @@ func (s *GetTradeFillsRequest) IdLessThan(idLessThan string) *GetTradeFillsReque return s } -func (s *GetTradeFillsRequest) StartTime(startTime int64) *GetTradeFillsRequest { +func (s *GetTradeFillsRequest) StartTime(startTime time.Time) *GetTradeFillsRequest { s.startTime = &startTime return s } -func (s *GetTradeFillsRequest) EndTime(endTime int64) *GetTradeFillsRequest { +func (s *GetTradeFillsRequest) EndTime(endTime time.Time) *GetTradeFillsRequest { s.endTime = &endTime return s } @@ -71,7 +73,8 @@ func (s *GetTradeFillsRequest) GetQueryParameters() (url.Values, error) { startTime := *s.startTime // assign parameter of startTime - params["startTime"] = startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check endTime field -> json key endTime @@ -79,7 +82,8 @@ func (s *GetTradeFillsRequest) GetQueryParameters() (url.Values, error) { endTime := *s.endTime // assign parameter of endTime - params["endTime"] = endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check orderId field -> json key orderId @@ -185,6 +189,12 @@ func (s *GetTradeFillsRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (s *GetTradeFillsRequest) GetPath() string { + return "/api/v2/spot/trade/fills" +} + +// Do generates the request object and send the request object to the API endpoint func (s *GetTradeFillsRequest) Do(ctx context.Context) ([]Trade, error) { // no body params @@ -194,7 +204,9 @@ func (s *GetTradeFillsRequest) Do(ctx context.Context) ([]Trade, error) { return nil, err } - apiURL := "/api/v2/spot/trade/fills" + var apiURL string + + apiURL = s.GetPath() req, err := s.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -211,6 +223,15 @@ func (s *GetTradeFillsRequest) Do(ctx context.Context) ([]Trade, error) { return nil, err } + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data []Trade if err := json.Unmarshal(apiResponse.Data, &data); err != nil { return nil, err diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go index 2af0aac17b..d597df1d06 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go @@ -4,6 +4,8 @@ package bitgetapi //go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data import ( + "time" + "github.com/c9s/requestgen" "github.com/c9s/bbgo/pkg/fixedpoint" @@ -39,10 +41,10 @@ type GetUnfilledOrdersRequest struct { // Limit number default 100 max 100 limit *string `param:"limit,query"` // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. - idLessThan *string `param:"idLessThan,query"` - startTime *int64 `param:"startTime,query"` - endTime *int64 `param:"endTime,query"` - orderId *string `param:"orderId,query"` + idLessThan *string `param:"idLessThan,query"` + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + orderId *string `param:"orderId,query"` } func (c *Client) NewGetUnfilledOrdersRequest() *GetUnfilledOrdersRequest { diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go index a3bb598193..18305340b7 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request_requestgen.go @@ -10,6 +10,8 @@ import ( "net/url" "reflect" "regexp" + "strconv" + "time" ) func (g *GetUnfilledOrdersRequest) Symbol(symbol string) *GetUnfilledOrdersRequest { @@ -27,12 +29,12 @@ func (g *GetUnfilledOrdersRequest) IdLessThan(idLessThan string) *GetUnfilledOrd return g } -func (g *GetUnfilledOrdersRequest) StartTime(startTime int64) *GetUnfilledOrdersRequest { +func (g *GetUnfilledOrdersRequest) StartTime(startTime time.Time) *GetUnfilledOrdersRequest { g.startTime = &startTime return g } -func (g *GetUnfilledOrdersRequest) EndTime(endTime int64) *GetUnfilledOrdersRequest { +func (g *GetUnfilledOrdersRequest) EndTime(endTime time.Time) *GetUnfilledOrdersRequest { g.endTime = &endTime return g } @@ -74,7 +76,8 @@ func (g *GetUnfilledOrdersRequest) GetQueryParameters() (url.Values, error) { startTime := *g.startTime // assign parameter of startTime - params["startTime"] = startTime + // convert time.Time to milliseconds time stamp + params["startTime"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check endTime field -> json key endTime @@ -82,7 +85,8 @@ func (g *GetUnfilledOrdersRequest) GetQueryParameters() (url.Values, error) { endTime := *g.endTime // assign parameter of endTime - params["endTime"] = endTime + // convert time.Time to milliseconds time stamp + params["endTime"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) } else { } // check orderId field -> json key orderId @@ -188,6 +192,12 @@ func (g *GetUnfilledOrdersRequest) GetSlugsMap() (map[string]string, error) { return slugs, nil } +// GetPath returns the request path of the API +func (g *GetUnfilledOrdersRequest) GetPath() string { + return "/api/v2/spot/trade/unfilled-orders" +} + +// Do generates the request object and send the request object to the API endpoint func (g *GetUnfilledOrdersRequest) Do(ctx context.Context) ([]UnfilledOrder, error) { // no body params @@ -197,7 +207,9 @@ func (g *GetUnfilledOrdersRequest) Do(ctx context.Context) ([]UnfilledOrder, err return nil, err } - apiURL := "/api/v2/spot/trade/unfilled-orders" + var apiURL string + + apiURL = g.GetPath() req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) if err != nil { @@ -213,6 +225,16 @@ func (g *GetUnfilledOrdersRequest) Do(ctx context.Context) ([]UnfilledOrder, err if err := response.DecodeJSON(&apiResponse); err != nil { return nil, err } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } var data []UnfilledOrder if err := json.Unmarshal(apiResponse.Data, &data); err != nil { return nil, err diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 3f731402aa..b290aaefe1 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -420,8 +420,8 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, res, err := e.v2client.NewGetHistoryOrdersRequest(). Symbol(symbol). Limit(strconv.Itoa(queryLimit)). - StartTime(since.UnixMilli()). - EndTime(until.UnixMilli()). + StartTime(since). + EndTime(until). Do(ctx) if err != nil { return nil, fmt.Errorf("failed to call get order histories error: %w", err) @@ -512,7 +512,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type if time.Since(*options.StartTime) > queryMaxDuration { return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", options.StartTime) } - req.StartTime(options.StartTime.UnixMilli()) + req.StartTime(*options.StartTime) } if options.EndTime != nil { @@ -525,7 +525,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type if options.EndTime.Sub(*options.StartTime) > queryMaxDuration { return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", options.StartTime, options.EndTime) } - req.EndTime(options.EndTime.UnixMilli()) + req.EndTime(*options.EndTime) } limit := options.Limit From 6d39c9a5d179842c49625b3d5dab9be55abb2b08 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 15 Nov 2023 22:18:09 +0800 Subject: [PATCH 191/422] pkg/exchange: use the now - 90 days instead of return err if since is 90 days earlier --- pkg/exchange/bitget/exchange.go | 48 +++++++++++++++++++-------------- pkg/types/strint.go | 7 +++++ 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index b290aaefe1..e8736cd653 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -397,18 +397,22 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // QueryClosedOrders queries closed order by time range(`CreatedTime`) and id. The order of the response is in descending order. // If you need to retrieve all data, please utilize the function pkg/exchange/batch.ClosedOrderBatchQuery. // +// REMARK: If your start time is 90 days earlier, we will update it to now - 90 days. // ** Since is inclusive, Until is exclusive. If you use a time range to query, you must provide both a start time and an end time. ** // ** Since and Until cannot exceed 90 days. ** -// ** Since from the last 90 days can be queried. ** +// ** Since from the last 90 days can be queried ** func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { - if time.Since(since) > queryMaxDuration { - return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", since) + newSince := since + + if time.Since(newSince) > queryMaxDuration { + newSince = time.Now().Add(-queryMaxDuration) + log.Warnf("!!!BITGET EXCHANGE API NOTICE!!! The closed order API cannot query data beyond 90 days from the current date, update %s -> %s", since, newSince) } - if until.Before(since) { - return nil, fmt.Errorf("end time %s before start %s", until, since) + if until.Before(newSince) { + return nil, fmt.Errorf("end time %s before start %s", until, newSince) } - if until.Sub(since) > queryMaxDuration { - return nil, fmt.Errorf("the start time %s and end time %s cannot exceed 90 days", since, until) + if until.Sub(newSince) > queryMaxDuration { + return nil, fmt.Errorf("the start time %s and end time %s cannot exceed 90 days", newSince, until) } if lastOrderID != 0 { log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The order of response is in descending order, so the last order id not supported.") @@ -420,7 +424,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, res, err := e.v2client.NewGetHistoryOrdersRequest(). Symbol(symbol). Limit(strconv.Itoa(queryLimit)). - StartTime(since). + StartTime(newSince). EndTime(until). Do(ctx) if err != nil { @@ -451,7 +455,7 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err } for _, order := range orders { - req := e.client.NewCancelOrderRequest() + req := e.v2client.NewCancelOrderRequest() reqId := "" switch { @@ -472,7 +476,7 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err continue } - req.Symbol(order.Market.Symbol) + req.Symbol(order.Symbol) if err := cancelOrderRateLimiter.Wait(ctx); err != nil { errs = multierr.Append(errs, fmt.Errorf("cancel order rate limiter wait, order id: %s, error: %w", order.ClientOrderID, err)) @@ -485,8 +489,8 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err } // sanity check - if res.OrderId != reqId && res.ClientOrderId != reqId { - errs = multierr.Append(errs, fmt.Errorf("order id mismatch, exp: %s, respOrderId: %s, respClientOrderId: %s", reqId, res.OrderId, res.ClientOrderId)) + if res.OrderId.String() != reqId && res.ClientOrderId != reqId { + errs = multierr.Append(errs, fmt.Errorf("order id mismatch, exp: %s, respOrderId: %d, respClientOrderId: %s", reqId, res.OrderId, res.ClientOrderId)) continue } } @@ -498,6 +502,7 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err // using (`CreatedTime`) as the search criteria. // If you need to retrieve all data, please utilize the function pkg/exchange/batch.TradeBatchQuery. // +// REMARK: If your start time is 90 days earlier, we will update it to now - 90 days. // ** StartTime is inclusive, EndTime is exclusive. If you use the EndTime, the StartTime is required. ** // ** StartTime and EndTime cannot exceed 90 days. ** func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { @@ -508,22 +513,25 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type req := e.v2client.NewGetTradeFillsRequest() req.Symbol(symbol) + var newStartTime time.Time if options.StartTime != nil { - if time.Since(*options.StartTime) > queryMaxDuration { - return nil, fmt.Errorf("start time from the last 90 days can be queried, got: %s", options.StartTime) + newStartTime = *options.StartTime + if time.Since(newStartTime) > queryMaxDuration { + newStartTime = time.Now().Add(-queryMaxDuration) + log.Warnf("!!!BITGET EXCHANGE API NOTICE!!! The trade API cannot query data beyond 90 days from the current date, update %s -> %s", *options.StartTime, newStartTime) } - req.StartTime(*options.StartTime) + req.StartTime(newStartTime) } if options.EndTime != nil { - if options.StartTime == nil { + if newStartTime.IsZero() { return nil, errors.New("start time is required for query trades if you take end time") } - if options.EndTime.Before(*options.StartTime) { - return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, *options.StartTime) + if options.EndTime.Before(newStartTime) { + return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, newStartTime) } - if options.EndTime.Sub(*options.StartTime) > queryMaxDuration { - return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", options.StartTime, options.EndTime) + if options.EndTime.Sub(newStartTime) > queryMaxDuration { + return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", newStartTime, options.EndTime) } req.EndTime(*options.EndTime) } diff --git a/pkg/types/strint.go b/pkg/types/strint.go index c5270d525a..b806dcea36 100644 --- a/pkg/types/strint.go +++ b/pkg/types/strint.go @@ -41,3 +41,10 @@ func (s *StrInt64) UnmarshalJSON(body []byte) error { return nil } + +func (s *StrInt64) String() string { + if s == nil { + return "" + } + return strconv.FormatInt(int64(*s), 10) +} From 93f8b79b69b2aecdc1f420b8a285ab46193afff6 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 16 Nov 2023 13:33:17 +0800 Subject: [PATCH 192/422] pkg/exchange: use GTC if time-in-force empty --- pkg/exchange/bitget/exchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index e8736cd653..be0030428c 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -289,7 +289,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr // 2. The query oepn/closed order does not including the `force` in SPOT. // If we support FOK/IOC, but you can't query them, that would be unreasonable. // The other case to consider is 'PostOnly', which is a trade-off because we want to support 'xmaker'. - if order.TimeInForce != types.TimeInForceGTC { + if order.TimeInForce != types.TimeInForceGTC && len(order.TimeInForce) != 0 { return nil, fmt.Errorf("time-in-force %s not supported", order.TimeInForce) } req.Force(v2.OrderForceGTC) From 5eb1ddb49a8f7b2c1452effa66653cd03e8d614b Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 16 Nov 2023 13:33:42 +0800 Subject: [PATCH 193/422] pkg/exchange: fix out-of-index --- pkg/exchange/bitget/exchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index be0030428c..e2b7a93471 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -123,7 +123,7 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke return nil, fmt.Errorf("unexpected length of query single symbol: %+v", resp) } - ticker := toGlobalTicker(resp[1]) + ticker := toGlobalTicker(resp[0]) return &ticker, nil } From a074f8c57acf82ffbd7137a759d105432f7000ff Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 16 Nov 2023 14:00:59 +0800 Subject: [PATCH 194/422] pkg/types: support bitget, bybit on exhcange unmrashal --- pkg/types/exchange.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/types/exchange.go b/pkg/types/exchange.go index 70e3c13798..06491224f6 100644 --- a/pkg/types/exchange.go +++ b/pkg/types/exchange.go @@ -26,13 +26,13 @@ func (n *ExchangeName) UnmarshalJSON(data []byte) error { } switch s { - case "max", "binance", "okex", "kucoin": + case "binance", "bitget", "bybit", "max", "okex", "kucoin": *n = ExchangeName(s) return nil } - return fmt.Errorf("unknown or unsupported exchange name: %s, valid names are: max, binance, okex, kucoin", s) + return fmt.Errorf("unknown or unsupported exchange name: %s, valid names are: binance, bitget, bybit, max, okex, kucoin", s) } func (n ExchangeName) String() string { From 34a844c03906a2aeb8db2703c88e694e4a495a18 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 16 Nov 2023 14:01:45 +0800 Subject: [PATCH 195/422] readme: update bitget, bybit to supported list --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 762f0b0820..bfddf9cb06 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,8 @@ the implementation. - OKEx Spot Exchange - Kucoin Spot Exchange - MAX Spot Exchange (located in Taiwan) -- Bitget Exchange (In Progress) -- Bybit Exchange (In Progress) +- Bitget Exchange +- Bybit Exchange ## Documentation and General Topics From 4f224c1c2a354dbab6020b9f11cea3afa0b742a8 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 17 Nov 2023 12:24:04 +0800 Subject: [PATCH 196/422] *: fix comments --- pkg/exchange/bitget/exchange.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index e2b7a93471..ad429acffc 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -21,10 +21,10 @@ const ( PlatformToken = "BGB" - queryLimit = 100 - defaultKLineLimit = 100 - maxOrderIdLen = 36 - queryMaxDuration = 90 * 24 * time.Hour + queryLimit = 100 + defaultKLineLimit = 100 + maxOrderIdLen = 36 + maxHistoricalDataQueryPeriod = 90 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -286,10 +286,10 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr // we support only GTC/PostOnly, this is because: // 1. We support only SPOT trading. - // 2. The query oepn/closed order does not including the `force` in SPOT. + // 2. The query open/closed order does not include the `force` in SPOT. // If we support FOK/IOC, but you can't query them, that would be unreasonable. // The other case to consider is 'PostOnly', which is a trade-off because we want to support 'xmaker'. - if order.TimeInForce != types.TimeInForceGTC && len(order.TimeInForce) != 0 { + if len(order.TimeInForce) != 0 && order.TimeInForce != types.TimeInForceGTC { return nil, fmt.Errorf("time-in-force %s not supported", order.TimeInForce) } req.Force(v2.OrderForceGTC) @@ -403,15 +403,17 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // ** Since from the last 90 days can be queried ** func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { newSince := since + now := time.Now() - if time.Since(newSince) > queryMaxDuration { - newSince = time.Now().Add(-queryMaxDuration) + if time.Since(newSince) > maxHistoricalDataQueryPeriod { + newSince = now.Add(-maxHistoricalDataQueryPeriod) log.Warnf("!!!BITGET EXCHANGE API NOTICE!!! The closed order API cannot query data beyond 90 days from the current date, update %s -> %s", since, newSince) } if until.Before(newSince) { - return nil, fmt.Errorf("end time %s before start %s", until, newSince) + log.Warnf("!!!BITGET EXCHANGE API NOTICE!!! The 'until' comes before 'since', update until to now(%s -> %s).", until, now) + until = now } - if until.Sub(newSince) > queryMaxDuration { + if until.Sub(newSince) > maxHistoricalDataQueryPeriod { return nil, fmt.Errorf("the start time %s and end time %s cannot exceed 90 days", newSince, until) } if lastOrderID != 0 { @@ -516,8 +518,8 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type var newStartTime time.Time if options.StartTime != nil { newStartTime = *options.StartTime - if time.Since(newStartTime) > queryMaxDuration { - newStartTime = time.Now().Add(-queryMaxDuration) + if time.Since(newStartTime) > maxHistoricalDataQueryPeriod { + newStartTime = time.Now().Add(-maxHistoricalDataQueryPeriod) log.Warnf("!!!BITGET EXCHANGE API NOTICE!!! The trade API cannot query data beyond 90 days from the current date, update %s -> %s", *options.StartTime, newStartTime) } req.StartTime(newStartTime) @@ -530,7 +532,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type if options.EndTime.Before(newStartTime) { return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, newStartTime) } - if options.EndTime.Sub(newStartTime) > queryMaxDuration { + if options.EndTime.Sub(newStartTime) > maxHistoricalDataQueryPeriod { return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", newStartTime, options.EndTime) } req.EndTime(*options.EndTime) From f46ca57bb2593c50808b1c52970cd63ace1ffe61 Mon Sep 17 00:00:00 2001 From: Edwin Date: Fri, 17 Nov 2023 16:01:15 +0800 Subject: [PATCH 197/422] pkg/types: refactor exchange name --- pkg/types/exchange.go | 64 +++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/pkg/types/exchange.go b/pkg/types/exchange.go index 06491224f6..b67a9b2b28 100644 --- a/pkg/types/exchange.go +++ b/pkg/types/exchange.go @@ -15,30 +15,6 @@ const DateFormat = "2006-01-02" type ExchangeName string -func (n *ExchangeName) Value() (driver.Value, error) { - return n.String(), nil -} - -func (n *ExchangeName) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - - switch s { - case "binance", "bitget", "bybit", "max", "okex", "kucoin": - *n = ExchangeName(s) - return nil - - } - - return fmt.Errorf("unknown or unsupported exchange name: %s, valid names are: binance, bitget, bybit, max, okex, kucoin", s) -} - -func (n ExchangeName) String() string { - return string(n) -} - const ( ExchangeMax ExchangeName = "max" ExchangeBinance ExchangeName = "binance" @@ -59,15 +35,43 @@ var SupportedExchanges = []ExchangeName{ // note: we are not using "backtest" } +func (n *ExchangeName) Value() (driver.Value, error) { + return n.String(), nil +} + +func (n *ExchangeName) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + *n = ExchangeName(s) + if !n.IsValid() { + return fmt.Errorf("%s is an invalid exchange name", s) + } + + return nil +} + +func (n ExchangeName) IsValid() bool { + switch n { + case ExchangeBinance, ExchangeBitget, ExchangeBybit, ExchangeMax, ExchangeOKEx, ExchangeKucoin: + return true + } + return false +} + +func (n ExchangeName) String() string { + return string(n) +} + func ValidExchangeName(a string) (ExchangeName, error) { - aa := strings.ToLower(a) - for _, n := range SupportedExchanges { - if string(n) == aa { - return n, nil - } + exName := ExchangeName(strings.ToLower(a)) + if !exName.IsValid() { + return "", fmt.Errorf("invalid exchange name: %s", a) } - return "", fmt.Errorf("invalid exchange name: %s", a) + return exName, nil } type ExchangeMinimal interface { From 592cdede66e22b8bdb5f627478fe90d0da14d0f3 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Fri, 17 Nov 2023 16:19:46 +0800 Subject: [PATCH 198/422] FEATURE: use new max v3 api to query closed orders by timestamp --- pkg/exchange/max/exchange.go | 45 ++++ .../v3/get_wallet_closed_orders_request.go | 27 ++ ...wallet_closed_orders_request_requestgen.go | 230 ++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go create mode 100644 pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 4c095a5c3c..d28aa09086 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -268,6 +268,10 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // lastOrderID is not supported on MAX func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) { log.Warn("!!!MAX EXCHANGE API NOTICE!!! the since/until conditions will not be effected on closed orders query, max exchange does not support time-range-based query") + if !since.IsZero() || !until.IsZero() { + return e.queryClosedOrdersByTime(ctx, symbol, since, until) + } + return e.queryClosedOrdersByLastOrderID(ctx, symbol, lastOrderID) } @@ -311,6 +315,47 @@ func (e *Exchange) queryClosedOrdersByLastOrderID(ctx context.Context, symbol st return types.SortOrdersAscending(orders), nil } +func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, since, until time.Time) (orders []types.Order, err error) { + if err := e.closedOrderQueryLimiter.Wait(ctx); err != nil { + return orders, err + } + + market := toLocalSymbol(symbol) + walletType := maxapi.WalletTypeSpot + if e.MarginSettings.IsMargin { + walletType = maxapi.WalletTypeMargin + } + + req := e.v3client.NewGetWalletClosedOrdersRequest(walletType).Market(market).Limit(1000) + + if !until.IsZero() { + req.Timestamp(until) + } + + maxOrders, err := req.Do(ctx) + if err != nil { + return orders, err + } + + for _, maxOrder := range maxOrders { + if maxOrder.CreatedAt.Time().Before(since) { + break + } + order, err2 := toGlobalOrder(maxOrder) + if err2 != nil { + err = multierr.Append(err, err2) + continue + } + + orders = append(orders, *order) + } + + if err != nil { + return nil, err + } + return types.SortOrdersAscending(orders), nil +} + func (e *Exchange) CancelAllOrders(ctx context.Context) ([]types.Order, error) { walletType := maxapi.WalletTypeSpot if e.MarginSettings.IsMargin { diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go new file mode 100644 index 0000000000..36cd947cae --- /dev/null +++ b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go @@ -0,0 +1,27 @@ +package v3 + +import ( + "time" + + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET +//go:generate -command PostRequest requestgen -method POST +//go:generate -command DeleteRequest requestgen -method DELETE + +func (s *Client) NewGetWalletClosedOrdersRequest(walletType WalletType) *GetWalletClosedOrdersRequest { + return &GetWalletClosedOrdersRequest{client: s.Client, walletType: walletType} +} + +//go:generate GetRequest -url "/api/v3/wallet/:walletType/orders/closed" -type GetWalletClosedOrdersRequest -responseType []Order +type GetWalletClosedOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + walletType WalletType `param:"walletType,slug,required"` + + market string `param:"market,required"` + timestamp *time.Time `param:"timestamp,milliseconds,omitempty"` + orderBy *string `param:"order_by,omitempty" validValues:"asc,desc,asc_updated_at,desc_updated_at"` + limit *uint `param:"limit,omitempty"` +} diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go new file mode 100644 index 0000000000..d1d9ec0b1c --- /dev/null +++ b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go @@ -0,0 +1,230 @@ +// Code generated by "requestgen --method GET -url /api/v3/wallet/:walletType/orders/closed -type GetWalletClosedOrdersRequest -responseType []Order"; DO NOT EDIT. + +package v3 + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/max/maxapi" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +func (g *GetWalletClosedOrdersRequest) Market(market string) *GetWalletClosedOrdersRequest { + g.market = market + return g +} + +func (g *GetWalletClosedOrdersRequest) Timestamp(timestamp time.Time) *GetWalletClosedOrdersRequest { + g.timestamp = ×tamp + return g +} + +func (g *GetWalletClosedOrdersRequest) OrderBy(orderBy string) *GetWalletClosedOrdersRequest { + g.orderBy = &orderBy + return g +} + +func (g *GetWalletClosedOrdersRequest) Limit(limit uint) *GetWalletClosedOrdersRequest { + g.limit = &limit + return g +} + +func (g *GetWalletClosedOrdersRequest) WalletType(walletType max.WalletType) *GetWalletClosedOrdersRequest { + g.walletType = walletType + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetWalletClosedOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetWalletClosedOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check market field -> json key market + market := g.market + + // TEMPLATE check-required + if len(market) == 0 { + return nil, fmt.Errorf("market is required, empty string given") + } + // END TEMPLATE check-required + + // assign parameter of market + params["market"] = market + // check timestamp field -> json key timestamp + if g.timestamp != nil { + timestamp := *g.timestamp + + // assign parameter of timestamp + // convert time.Time to milliseconds time stamp + params["timestamp"] = strconv.FormatInt(timestamp.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check orderBy field -> json key order_by + if g.orderBy != nil { + orderBy := *g.orderBy + + // TEMPLATE check-valid-values + switch orderBy { + case "asc", "desc": + params["order_by"] = orderBy + + default: + return nil, fmt.Errorf("order_by value %v is invalid", orderBy) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderBy + params["order_by"] = orderBy + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetWalletClosedOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetWalletClosedOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetWalletClosedOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check walletType field -> json key walletType + walletType := g.walletType + + // TEMPLATE check-required + if len(walletType) == 0 { + return nil, fmt.Errorf("walletType is required, empty string given") + } + // END TEMPLATE check-required + + // assign parameter of walletType + params["walletType"] = walletType + + return params, nil +} + +func (g *GetWalletClosedOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetWalletClosedOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetWalletClosedOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetWalletClosedOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +func (g *GetWalletClosedOrdersRequest) Do(ctx context.Context) ([]max.Order, error) { + + // empty params for GET operation + var params interface{} + query, err := g.GetParametersQuery() + if err != nil { + return nil, err + } + + apiURL := "/api/v3/wallet/:walletType/orders/closed" + slugs, err := g.GetSlugsMap() + if err != nil { + return nil, err + } + + apiURL = g.applySlugsToUrl(apiURL, slugs) + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse []max.Order + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + return apiResponse, nil +} From f2237032476d414f29cb230a574d0fe47c99e47f Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 14 Nov 2023 15:47:39 +0800 Subject: [PATCH 199/422] max: force type check on max.Exchange --- pkg/exchange/max/exchange.go | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 4c095a5c3c..b8c947b66d 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -22,6 +22,10 @@ import ( var log = logrus.WithField("exchange", "max") +func init() { + _ = types.ExchangeTradeHistoryService(&Exchange{}) +} + type Exchange struct { types.MarginSettings @@ -266,12 +270,16 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ } // lastOrderID is not supported on MAX -func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) { +func (e *Exchange) QueryClosedOrders( + ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, +) ([]types.Order, error) { log.Warn("!!!MAX EXCHANGE API NOTICE!!! the since/until conditions will not be effected on closed orders query, max exchange does not support time-range-based query") return e.queryClosedOrdersByLastOrderID(ctx, symbol, lastOrderID) } -func (e *Exchange) queryClosedOrdersByLastOrderID(ctx context.Context, symbol string, lastOrderID uint64) (orders []types.Order, err error) { +func (e *Exchange) queryClosedOrdersByLastOrderID( + ctx context.Context, symbol string, lastOrderID uint64, +) (orders []types.Order, err error) { if err := e.closedOrderQueryLimiter.Wait(ctx); err != nil { return orders, err } @@ -429,7 +437,9 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err return err2 } -func (e *Exchange) Withdraw(ctx context.Context, asset string, amount fixedpoint.Value, address string, options *types.WithdrawalOptions) error { +func (e *Exchange) Withdraw( + ctx context.Context, asset string, amount fixedpoint.Value, address string, options *types.WithdrawalOptions, +) error { asset = toLocalCurrency(asset) addresses, err := e.client.WithdrawalService.NewGetWithdrawalAddressesRequest(). @@ -673,7 +683,9 @@ func (e *Exchange) queryBalances(ctx context.Context, walletType maxapi.WalletTy return balances, nil } -func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []types.Withdraw, err error) { +func (e *Exchange) QueryWithdrawHistory( + ctx context.Context, asset string, since, until time.Time, +) (allWithdraws []types.Withdraw, err error) { startTime := since limit := 1000 txIDs := map[string]struct{}{} @@ -769,7 +781,9 @@ func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since return allWithdraws, nil } -func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []types.Deposit, err error) { +func (e *Exchange) QueryDepositHistory( + ctx context.Context, asset string, since, until time.Time, +) (allDeposits []types.Deposit, err error) { startTime := since limit := 1000 txIDs := map[string]struct{}{} @@ -846,7 +860,9 @@ func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, // For this QueryTrades spec (to be compatible with batch.TradeBatchQuery) // give LastTradeID -> ignore start_time (but still can filter the end_time) // without any parameters -> return trades within 24 hours -func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { +func (e *Exchange) QueryTrades( + ctx context.Context, symbol string, options *types.TradeQueryOptions, +) (trades []types.Trade, err error) { if err := e.queryTradeLimiter.Wait(ctx); err != nil { return nil, err } @@ -964,7 +980,9 @@ func (e *Exchange) QueryRewards(ctx context.Context, startTime time.Time) ([]typ // https://max-api.maicoin.com/api/v2/k?market=btctwd&limit=10&period=1×tamp=1620202440 // The above query will return a kline that starts with 1620202440 (unix timestamp) without endTime. // We need to calculate the endTime by ourself. -func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { +func (e *Exchange) QueryKLines( + ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions, +) ([]types.KLine, error) { if err := e.marketDataLimiter.Wait(ctx); err != nil { return nil, err } @@ -1041,7 +1059,9 @@ func (e *Exchange) BorrowMarginAsset(ctx context.Context, asset string, amount f return nil } -func (e *Exchange) QueryMarginAssetMaxBorrowable(ctx context.Context, asset string) (amount fixedpoint.Value, err error) { +func (e *Exchange) QueryMarginAssetMaxBorrowable( + ctx context.Context, asset string, +) (amount fixedpoint.Value, err error) { req := e.v3client.NewGetMarginBorrowingLimitsRequest() resp, err := req.Do(ctx) if err != nil { From d5fe13272e0ad4db5570114fc97f89f852d385dd Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 14 Nov 2023 16:09:29 +0800 Subject: [PATCH 200/422] service: log sync start time --- pkg/service/sync.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pkg/service/sync.go b/pkg/service/sync.go index 34010c5c5a..ae9f6adb73 100644 --- a/pkg/service/sync.go +++ b/pkg/service/sync.go @@ -25,30 +25,37 @@ type SyncService struct { } // SyncSessionSymbols syncs the trades from the given exchange session -func (s *SyncService) SyncSessionSymbols(ctx context.Context, exchange types.Exchange, startTime time.Time, symbols ...string) error { +func (s *SyncService) SyncSessionSymbols( + ctx context.Context, exchange types.Exchange, startTime time.Time, symbols ...string, +) error { markets, err := cache.LoadExchangeMarketsWithCache(ctx, exchange) if err != nil { return err } for _, symbol := range symbols { - if _, ok := markets[symbol]; ok { - log.Infof("syncing %s %s trades...", exchange.Name(), symbol) - if err := s.TradeService.Sync(ctx, exchange, symbol, startTime); err != nil { - return err - } - - log.Infof("syncing %s %s orders...", exchange.Name(), symbol) - if err := s.OrderService.Sync(ctx, exchange, symbol, startTime); err != nil { - return err - } + // skip symbols do not exist in the market info + if _, ok := markets[symbol]; !ok { + continue + } + + log.Infof("syncing %s %s trades from %s...", exchange.Name(), symbol, startTime) + if err := s.TradeService.Sync(ctx, exchange, symbol, startTime); err != nil { + return err + } + + log.Infof("syncing %s %s orders from %s...", exchange.Name(), symbol, startTime) + if err := s.OrderService.Sync(ctx, exchange, symbol, startTime); err != nil { + return err } } return nil } -func (s *SyncService) SyncMarginHistory(ctx context.Context, exchange types.Exchange, startTime time.Time, assets ...string) error { +func (s *SyncService) SyncMarginHistory( + ctx context.Context, exchange types.Exchange, startTime time.Time, assets ...string, +) error { if _, implemented := exchange.(types.MarginHistoryService); !implemented { log.Debugf("exchange %T does not support types.MarginHistoryService", exchange) return nil From 3cfc810f8d737192f9d954096fe05861c579f5e2 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 14 Nov 2023 16:09:47 +0800 Subject: [PATCH 201/422] max: group the request building statement --- pkg/exchange/max/exchange.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index b8c947b66d..d50279f4ba 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -24,6 +24,7 @@ var log = logrus.WithField("exchange", "max") func init() { _ = types.ExchangeTradeHistoryService(&Exchange{}) + } type Exchange struct { @@ -290,13 +291,14 @@ func (e *Exchange) queryClosedOrdersByLastOrderID( walletType = maxapi.WalletTypeMargin } - req := e.v3client.NewGetWalletOrderHistoryRequest(walletType).Market(market) if lastOrderID == 0 { lastOrderID = 1 } - req.FromID(lastOrderID) - req.Limit(1000) + req := e.v3client.NewGetWalletOrderHistoryRequest(walletType). + Market(market). + FromID(lastOrderID). + Limit(1000) maxOrders, err := req.Do(ctx) if err != nil { From fe9dc9a79d1a4ad7781604e867a9c0be365a21aa Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 14 Nov 2023 16:15:29 +0800 Subject: [PATCH 202/422] bbgo: change pending update log level to info --- pkg/bbgo/activeorderbook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 994b7329d7..5d8c0812d4 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -268,7 +268,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { b.mu.Lock() if !b.orders.Exists(order.OrderID) { - log.Infof("[ActiveOrderBook] order #%d %s does not exist, adding it to pending order update", order.OrderID, order.Status) + log.Debugf("[ActiveOrderBook] order #%d %s does not exist, adding it to pending order update", order.OrderID, order.Status) b.pendingOrderUpdates.Add(order) b.mu.Unlock() return From b307275e60ca3af4726be08cb66b526b0970168e Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 17 Nov 2023 16:18:55 +0800 Subject: [PATCH 203/422] types: add order.originalStatus --- pkg/types/order.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/types/order.go b/pkg/types/order.go index ae4dc8425e..f8bd13cb6e 100644 --- a/pkg/types/order.go +++ b/pkg/types/order.go @@ -258,11 +258,22 @@ type Order struct { OrderID uint64 `json:"orderID" db:"order_id"` // order id UUID string `json:"uuid,omitempty"` - Status OrderStatus `json:"status" db:"status"` + Status OrderStatus `json:"status" db:"status"` + + // OriginalStatus stores the original order status from the specific exchange + OriginalStatus string `json:"originalStatus,omitempty" db:"-"` + + // ExecutedQuantity is how much quantity has been executed ExecutedQuantity fixedpoint.Value `json:"executedQuantity" db:"executed_quantity"` - IsWorking bool `json:"isWorking" db:"is_working"` - CreationTime Time `json:"creationTime" db:"created_at"` - UpdateTime Time `json:"updateTime" db:"updated_at"` + + // IsWorking means if the order is still on the order book (active order) + IsWorking bool `json:"isWorking" db:"is_working"` + + // CreationTime is the time when this order is created + CreationTime Time `json:"creationTime" db:"created_at"` + + // UpdateTime is the latest time when this order gets updated + UpdateTime Time `json:"updateTime" db:"updated_at"` IsFutures bool `json:"isFutures,omitempty" db:"is_futures"` IsMargin bool `json:"isMargin,omitempty" db:"is_margin"` From 5795a71111297045cb0c2c0ee95395cb6a0236a1 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 17 Nov 2023 16:21:29 +0800 Subject: [PATCH 204/422] binance,max: store original order status into the order struct --- pkg/exchange/binance/convert.go | 3 ++- pkg/exchange/max/convert.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/binance/convert.go b/pkg/exchange/binance/convert.go index 378c647e08..4fdc4a5f14 100644 --- a/pkg/exchange/binance/convert.go +++ b/pkg/exchange/binance/convert.go @@ -177,6 +177,7 @@ func toGlobalOrder(binanceOrder *binance.Order, isMargin bool) (*types.Order, er IsWorking: binanceOrder.IsWorking, OrderID: uint64(binanceOrder.OrderID), Status: toGlobalOrderStatus(binanceOrder.Status), + OriginalStatus: string(binanceOrder.Status), ExecutedQuantity: fixedpoint.MustNewFromString(binanceOrder.ExecutedQuantity), CreationTime: types.Time(millisecondTime(binanceOrder.Time)), UpdateTime: types.Time(millisecondTime(binanceOrder.UpdateTime)), @@ -228,7 +229,7 @@ func toGlobalTrade(t binance.TradeV3, isMargin bool) (*types.Trade, error) { OrderID: uint64(t.OrderID), Price: price, Symbol: t.Symbol, - Exchange: "binance", + Exchange: types.ExchangeBinance, Quantity: quantity, QuoteQuantity: quoteQuantity, Side: side, diff --git a/pkg/exchange/max/convert.go b/pkg/exchange/max/convert.go index b616e75c94..56df7959b3 100644 --- a/pkg/exchange/max/convert.go +++ b/pkg/exchange/max/convert.go @@ -188,6 +188,7 @@ func toGlobalOrder(maxOrder max.Order) (*types.Order, error) { IsWorking: maxOrder.State == max.OrderStateWait, OrderID: maxOrder.ID, Status: toGlobalOrderStatus(maxOrder.State, executedVolume, remainingVolume), + OriginalStatus: string(maxOrder.State), ExecutedQuantity: executedVolume, CreationTime: types.Time(maxOrder.CreatedAt.Time()), UpdateTime: types.Time(maxOrder.UpdatedAt.Time()), From e5033c093a3886307c81a347fff072b4f951241e Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 17 Nov 2023 16:46:16 +0800 Subject: [PATCH 205/422] grid2: check order's original status for updating --- pkg/strategy/grid2/active_order_recover.go | 16 +++++++++---- pkg/strategy/grid2/recover.go | 28 +++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index 9e407b7265..d06f316fc6 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -4,14 +4,16 @@ import ( "context" "time" - "github.com/c9s/bbgo/pkg/bbgo" - "github.com/c9s/bbgo/pkg/exchange/retry" - "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "go.uber.org/multierr" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/exchange/max" + "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" ) type SyncActiveOrdersOpts struct { @@ -89,6 +91,11 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { } } +func isMaxExchange(ex interface{}) bool { + _, yes := ex.(*max.Exchange) + return yes +} + func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders") @@ -115,6 +122,7 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { var errs error // update active orders not in open orders for _, activeOrder := range activeOrders { + if _, exist := openOrdersMap[activeOrder.OrderID]; exist { // no need to sync active order already in active orderbook, because we only need to know if it filled or not. delete(openOrdersMap, activeOrder.OrderID) diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 9418dd5a8a..cae47b3037 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -6,11 +6,13 @@ import ( "strconv" "time" + "github.com/pkg/errors" + "github.com/c9s/bbgo/pkg/bbgo" + maxapi "github.com/c9s/bbgo/pkg/exchange/max/maxapi" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" - "github.com/pkg/errors" ) var syncWindow = -3 * time.Minute @@ -267,24 +269,34 @@ func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error return book, nil } -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64, syncBefore time.Time) (bool, error) { +func syncActiveOrder( + ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, + orderID uint64, syncBefore time.Time, +) (isOrderUpdated bool, err error) { + isMax := isMaxExchange(orderQueryService) + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ Symbol: activeOrderBook.Symbol, OrderID: strconv.FormatUint(orderID, 10), }) - isActiveOrderBookUpdated := false - if err != nil { - return isActiveOrderBookUpdated, err + return isOrderUpdated, err + } + + // maxapi.OrderStateFinalizing does not mean the fee is calculated + // we should only consider order state done for MAX + if isMax && updatedOrder.OriginalStatus != string(maxapi.OrderStateDone) { + return isOrderUpdated, nil } - isActiveOrderBookUpdated = updatedOrder.UpdateTime.Before(syncBefore) - if isActiveOrderBookUpdated { + // should only trigger order update when the updated time is old enough + isOrderUpdated = updatedOrder.UpdateTime.Before(syncBefore) + if isOrderUpdated { activeOrderBook.Update(*updatedOrder) } - return isActiveOrderBookUpdated, nil + return isOrderUpdated, nil } func queryTradesToUpdateTwinOrderBook( From c248b2a3236b849244cdcb5e8ea9558f28274b13 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 17 Nov 2023 17:05:44 +0800 Subject: [PATCH 206/422] bbgo: remove local trade snapshot from db --- pkg/bbgo/session.go | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 617abef2f2..aacb86b691 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" @@ -20,7 +19,6 @@ import ( exchange2 "github.com/c9s/bbgo/pkg/exchange" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/service" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -398,30 +396,7 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ return fmt.Errorf("market %s is not defined", symbol) } - var err error - var trades []types.Trade - if environ.SyncService != nil && environ.BacktestService == nil { - tradingFeeCurrency := session.Exchange.PlatformFeeCurrency() - if strings.HasPrefix(symbol, tradingFeeCurrency) { - trades, err = environ.TradeService.QueryForTradingFeeCurrency(session.Exchange.Name(), symbol, tradingFeeCurrency) - } else { - trades, err = environ.TradeService.Query(service.QueryTradesOptions{ - Exchange: session.Exchange.Name(), - Symbol: symbol, - Ordering: "DESC", - Limit: 100, - }) - } - - if err != nil { - return err - } - - trades = types.SortTradesAscending(trades) - log.Infof("symbol %s: %d trades loaded", symbol, len(trades)) - } - - session.Trades[symbol] = &types.TradeSlice{Trades: trades} + session.Trades[symbol] = &types.TradeSlice{Trades: nil} session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { if trade.Symbol != symbol { return @@ -430,12 +405,12 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ session.Trades[symbol].Append(trade) }) + // session wide position position := &types.Position{ Symbol: symbol, BaseCurrency: market.BaseCurrency, QuoteCurrency: market.QuoteCurrency, } - position.AddTrades(trades) position.BindStream(session.UserDataStream) session.positions[symbol] = position From eac0195815cd281e4fadcfdfeb9dbb3396e7c0ea Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 17 Nov 2023 17:11:22 +0800 Subject: [PATCH 207/422] bbgo: truncate trade buffer if it gets too large --- pkg/bbgo/config.go | 2 ++ pkg/bbgo/session.go | 26 +++++++++++++++++++------- pkg/types/trade.go | 10 ++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index 591176484b..1fac9e314d 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -329,6 +329,8 @@ type ServiceConfig struct { type EnvironmentConfig struct { DisableDefaultKLineSubscription bool `json:"disableDefaultKLineSubscription"` DisableHistoryKLinePreload bool `json:"disableHistoryKLinePreload"` + DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"` + MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` } type Config struct { diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index aacb86b691..75012fb038 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -396,14 +396,27 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ return fmt.Errorf("market %s is not defined", symbol) } + disableSessionTradeBuffer := environ.environmentConfig != nil && environ.environmentConfig.DisableSessionTradeBuffer + maxSessionTradeBufferSize := 0 + if environ.environmentConfig != nil && environ.environmentConfig.MaxSessionTradeBufferSize > 0 { + maxSessionTradeBufferSize = environ.environmentConfig.MaxSessionTradeBufferSize + } + session.Trades[symbol] = &types.TradeSlice{Trades: nil} - session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { - if trade.Symbol != symbol { - return - } - session.Trades[symbol].Append(trade) - }) + if !disableSessionTradeBuffer { + session.UserDataStream.OnTradeUpdate(func(trade types.Trade) { + if trade.Symbol != symbol { + return + } + + session.Trades[symbol].Append(trade) + + if maxSessionTradeBufferSize > 0 { + session.Trades[symbol].Truncate(maxSessionTradeBufferSize) + } + }) + } // session wide position position := &types.Position{ @@ -416,7 +429,6 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ orderStore := core.NewOrderStore(symbol) orderStore.AddOrderUpdate = true - orderStore.BindStream(session.UserDataStream) session.orderStores[symbol] = orderStore diff --git a/pkg/types/trade.go b/pkg/types/trade.go index f80b864a27..5a7eb0a7a0 100644 --- a/pkg/types/trade.go +++ b/pkg/types/trade.go @@ -47,6 +47,16 @@ func (s *TradeSlice) Append(t Trade) { s.mu.Unlock() } +func (s *TradeSlice) Truncate(size int) { + s.mu.Lock() + + if len(s.Trades) > size { + s.Trades = s.Trades[len(s.Trades)-1-size:] + } + + s.mu.Unlock() +} + type Trade struct { // GID is the global ID GID int64 `json:"gid" db:"gid"` From ce76ad3c0380650801555b33f2688dbafedb5418 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 20 Nov 2023 15:32:04 +0800 Subject: [PATCH 208/422] use OrderByType --- pkg/exchange/max/exchange.go | 2 +- pkg/exchange/max/maxapi/order.go | 9 +++++++++ .../maxapi/v3/get_wallet_closed_orders_request.go | 8 ++++---- .../get_wallet_closed_orders_request_requestgen.go | 13 +------------ pkg/exchange/max/maxapi/v3/order.go | 1 + 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index d28aa09086..583494b078 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -339,7 +339,7 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s for _, maxOrder := range maxOrders { if maxOrder.CreatedAt.Time().Before(since) { - break + continue } order, err2 := toGlobalOrder(maxOrder) if err2 != nil { diff --git a/pkg/exchange/max/maxapi/order.go b/pkg/exchange/max/maxapi/order.go index be3fdb1fe9..6f783c33a9 100644 --- a/pkg/exchange/max/maxapi/order.go +++ b/pkg/exchange/max/maxapi/order.go @@ -17,6 +17,15 @@ const ( WalletTypeMargin WalletType = "m" ) +type OrderByType string + +const ( + OrderByAsc OrderByType = "asc" + OrderByDesc OrderByType = "desc" + OrderByAscUpdatedAt OrderByType = "asc_updated_at" + OrderByDescUpdatedAt OrderByType = "desc_updated_at" +) + type OrderStateToQuery int const ( diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go index 36cd947cae..9ac43eadb1 100644 --- a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go +++ b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request.go @@ -20,8 +20,8 @@ type GetWalletClosedOrdersRequest struct { walletType WalletType `param:"walletType,slug,required"` - market string `param:"market,required"` - timestamp *time.Time `param:"timestamp,milliseconds,omitempty"` - orderBy *string `param:"order_by,omitempty" validValues:"asc,desc,asc_updated_at,desc_updated_at"` - limit *uint `param:"limit,omitempty"` + market string `param:"market,required"` + timestamp *time.Time `param:"timestamp,milliseconds,omitempty"` + orderBy *OrderByType `param:"order_by,omitempty"` + limit *uint `param:"limit,omitempty"` } diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go index d1d9ec0b1c..551f7ee9d8 100644 --- a/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go +++ b/pkg/exchange/max/maxapi/v3/get_wallet_closed_orders_request_requestgen.go @@ -24,7 +24,7 @@ func (g *GetWalletClosedOrdersRequest) Timestamp(timestamp time.Time) *GetWallet return g } -func (g *GetWalletClosedOrdersRequest) OrderBy(orderBy string) *GetWalletClosedOrdersRequest { +func (g *GetWalletClosedOrdersRequest) OrderBy(orderBy max.OrderByType) *GetWalletClosedOrdersRequest { g.orderBy = &orderBy return g } @@ -78,17 +78,6 @@ func (g *GetWalletClosedOrdersRequest) GetParameters() (map[string]interface{}, if g.orderBy != nil { orderBy := *g.orderBy - // TEMPLATE check-valid-values - switch orderBy { - case "asc", "desc": - params["order_by"] = orderBy - - default: - return nil, fmt.Errorf("order_by value %v is invalid", orderBy) - - } - // END TEMPLATE check-valid-values - // assign parameter of orderBy params["order_by"] = orderBy } else { diff --git a/pkg/exchange/max/maxapi/v3/order.go b/pkg/exchange/max/maxapi/v3/order.go index 4244857b6f..dc3e316a0a 100644 --- a/pkg/exchange/max/maxapi/v3/order.go +++ b/pkg/exchange/max/maxapi/v3/order.go @@ -8,6 +8,7 @@ import ( // create type alias type WalletType = maxapi.WalletType +type OrderByType = maxapi.OrderByType type OrderType = maxapi.OrderType type Order = maxapi.Order From 3ea333fd527070de11171d7a1dcc25b42607d093 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 16:14:09 +0800 Subject: [PATCH 209/422] bbgo: add DisableStartupBalanceQuery option --- pkg/bbgo/config.go | 9 +++++++-- pkg/bbgo/session.go | 23 +++++++++++++++-------- pkg/types/account.go | 20 +++++++++++++++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index 1fac9e314d..f892f2138f 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -329,8 +329,13 @@ type ServiceConfig struct { type EnvironmentConfig struct { DisableDefaultKLineSubscription bool `json:"disableDefaultKLineSubscription"` DisableHistoryKLinePreload bool `json:"disableHistoryKLinePreload"` - DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"` - MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` + + // DisableStartUpBalanceQuery disables the balance query in the startup process + // which initializes the session.Account with the QueryAccount method. + DisableStartupBalanceQuery bool `json:"disableStartupBalanceQuery"` + + DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"` + MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` } type Config struct { diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 75012fb038..ba10b1e1e4 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -256,15 +256,22 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) } } - logger.Infof("querying account balances...") - account, err := session.Exchange.QueryAccount(ctx) - if err != nil { - return err - } + disableStartupBalanceQuery := environ.environmentConfig != nil && environ.environmentConfig.DisableStartupBalanceQuery + if disableStartupBalanceQuery { + session.accountMutex.Lock() + session.Account = types.NewAccount() + session.accountMutex.Unlock() + } else { + logger.Infof("querying account balances...") + account, err := session.Exchange.QueryAccount(ctx) + if err != nil { + return err + } - session.accountMutex.Lock() - session.Account = account - session.accountMutex.Unlock() + session.accountMutex.Lock() + session.Account = account + session.accountMutex.Unlock() + } logger.Infof("account %s balances:\n%s", session.Name, account.Balances().String()) diff --git a/pkg/types/account.go b/pkg/types/account.go index cd590f5300..5cb4ca92de 100644 --- a/pkg/types/account.go +++ b/pkg/types/account.go @@ -103,8 +103,26 @@ type IsolatedMarginAccountInfo struct { func NewAccount() *Account { return &Account{ - balances: make(BalanceMap), + AccountType: "spot", + FuturesInfo: nil, + MarginInfo: nil, + IsolatedMarginInfo: nil, + MarginLevel: fixedpoint.Zero, + MarginTolerance: fixedpoint.Zero, + BorrowEnabled: false, + TransferEnabled: false, + MarginRatio: fixedpoint.Zero, + LiquidationPrice: fixedpoint.Zero, + LiquidationRate: fixedpoint.Zero, + MakerFeeRate: fixedpoint.Zero, + TakerFeeRate: fixedpoint.Zero, + TotalAccountValue: fixedpoint.Zero, + CanDeposit: false, + CanTrade: false, + CanWithdraw: false, + balances: make(BalanceMap), } + } // Balances lock the balances and returned the copied balances From 7c59e3ddc4633b55afa78f40c997ebe9c8721a19 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 16:15:33 +0800 Subject: [PATCH 210/422] bbgo: add setAccount for account mutex protection --- pkg/bbgo/session.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index ba10b1e1e4..bac3e0cf1a 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -177,10 +177,14 @@ func (session *ExchangeSession) UpdateAccount(ctx context.Context) (*types.Accou return nil, err } + session.setAccount(account) + return account, nil +} + +func (session *ExchangeSession) setAccount(a *types.Account) { session.accountMutex.Lock() - session.Account = account + session.Account = a session.accountMutex.Unlock() - return account, nil } // Init initializes the basic data structure and market information by its exchange. @@ -268,13 +272,10 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) return err } - session.accountMutex.Lock() - session.Account = account - session.accountMutex.Unlock() + session.setAccount(account) + logger.Infof("account %s balances:\n%s", session.Name, account.Balances().String()) } - logger.Infof("account %s balances:\n%s", session.Name, account.Balances().String()) - // forward trade updates and order updates to the order executor session.UserDataStream.OnTradeUpdate(session.OrderExecutor.EmitTradeUpdate) session.UserDataStream.OnOrderUpdate(session.OrderExecutor.EmitOrderUpdate) From c360c6045cd69a3484f3b6b17da7f06a72014f8a Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 16:20:39 +0800 Subject: [PATCH 211/422] bbgo: call retry.QueryAccountUntilSuccessful in the startup time --- pkg/bbgo/session.go | 5 +++-- pkg/exchange/retry/order.go | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index bac3e0cf1a..6778ce35b3 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -15,6 +15,7 @@ import ( "github.com/c9s/bbgo/pkg/cache" "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/util/templateutil" exchange2 "github.com/c9s/bbgo/pkg/exchange" @@ -267,12 +268,13 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) session.accountMutex.Unlock() } else { logger.Infof("querying account balances...") - account, err := session.Exchange.QueryAccount(ctx) + account, err := retry.QueryAccountUntilSuccessful(ctx, session.Exchange) if err != nil { return err } session.setAccount(account) + session.metricsBalancesUpdater(account.Balances()) logger.Infof("account %s balances:\n%s", session.Name, account.Balances().String()) } @@ -296,7 +298,6 @@ func (session *ExchangeSession) Init(ctx context.Context, environ *Environment) // if metrics mode is enabled, we bind the callbacks to update metrics if viper.GetBool("metrics") { - session.metricsBalancesUpdater(account.Balances()) session.bindUserDataStreamMetrics(session.UserDataStream) } } diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index df6ef6b59c..66a690a542 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -16,7 +16,9 @@ type advancedOrderCancelService interface { CancelOrdersByGroupID(ctx context.Context, groupID uint32) ([]types.Order, error) } -func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64) (o *types.Order, err error) { +func QueryOrderUntilFilled( + ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64, +) (o *types.Order, err error) { var op = func() (err2 error) { o, err2 = queryOrderService.QueryOrder(ctx, types.OrderQuery{ Symbol: symbol, @@ -56,7 +58,9 @@ func GeneralLiteBackoff(ctx context.Context, op backoff.Operation) (err error) { return err } -func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { +func QueryOpenOrdersUntilSuccessful( + ctx context.Context, ex types.Exchange, symbol string, +) (openOrders []types.Order, err error) { var op = func() (err2 error) { openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) return err2 @@ -66,7 +70,9 @@ func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symb return openOrders, err } -func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { +func QueryOpenOrdersUntilSuccessfulLite( + ctx context.Context, ex types.Exchange, symbol string, +) (openOrders []types.Order, err error) { var op = func() (err2 error) { openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) return err2 @@ -76,7 +82,21 @@ func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, return openOrders, err } -func QueryOrderUntilSuccessful(ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery) (order *types.Order, err error) { +func QueryAccountUntilSuccessful( + ctx context.Context, ex types.ExchangeAccountService, +) (account *types.Account, err error) { + var op = func() (err2 error) { + account, err2 = ex.QueryAccount(ctx) + return err2 + } + + err = GeneralBackoff(ctx, op) + return account, err +} + +func QueryOrderUntilSuccessful( + ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery, +) (order *types.Order, err error) { var op = func() (err2 error) { order, err2 = query.QueryOrder(ctx, opts) return err2 From 59ba04fa1d00e6ecd8b92a257291ea24ac62939e Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 17:32:19 +0800 Subject: [PATCH 212/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index 605f188bc8..f1b9d11eb7 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index ddd67acf42..e61e5722d2 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index 4c89aa619e..4ab2095b8f 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 014c4d5a87..1d5e392faf 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 8d679a4543..781473e7f7 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 956e07afb6..b3131ea436 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index 42754dcf11..bd1a9b7b98 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 05d2851889..c565b09a93 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index 5fe462deb8..5cd7a3e059 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index 947b9f0791..efa33d4e98 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index eefb045b20..1631ba30f2 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index 9032b60974..283b7bca61 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 6ea3daa8bd..ee1cb005c1 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index 645c604aa5..f1a84f9921 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index e336f2531d..431a0fddf1 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index b6f9b7aec7..73e03cf776 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index aaf0435e6c..d8332b8002 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 5dc938ed70..01897691bf 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index fc712c940c..069fe058ea 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index bbd40cef67..f38771ed93 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index 7c0a5122b1..e16aaea684 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 667b8c8c86..994636657d 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 4aeca4c37a..9889f099ed 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 7e0df13624..55699bda45 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index f2c0c9ac11..f15732d310 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index 02a156a824..5537cc2ba5 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index dd86d5bd82..502ae9f909 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index 0b83b12cb6..cc7d14dd1d 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index e27f0988fc..7755ad0291 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 9-Nov-2023 +###### Auto generated by spf13/cobra on 20-Nov-2023 From ae3f3e1f70c5e32eb45756965c46e6070e0f1415 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 17:32:20 +0800 Subject: [PATCH 213/422] bump version to v1.54.0 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index 5b92569c2f..491a8bbf2c 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.53.0-4c701676-dev" +const Version = "v1.54.0-da3150e2-dev" -const VersionGitRef = "4c701676" +const VersionGitRef = "da3150e2" diff --git a/pkg/version/version.go b/pkg/version/version.go index cdefa39ba3..c93b147b6c 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.53.0-4c701676" +const Version = "v1.54.0-da3150e2" -const VersionGitRef = "4c701676" +const VersionGitRef = "da3150e2" From 1a0db2cd396dfea3b60b7f6a39c090420fd80548 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 20 Nov 2023 17:32:20 +0800 Subject: [PATCH 214/422] add v1.54.0 release note --- doc/release/v1.54.0.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/release/v1.54.0.md diff --git a/doc/release/v1.54.0.md b/doc/release/v1.54.0.md new file mode 100644 index 0000000000..fd77793369 --- /dev/null +++ b/doc/release/v1.54.0.md @@ -0,0 +1,18 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.53.0...main) + + - [#1424](https://github.com/c9s/bbgo/pull/1424): IMPROVE: improve startup balance query process + - [#1408](https://github.com/c9s/bbgo/pull/1408): FEATURE: [datasource] add wise rate API + - [#1423](https://github.com/c9s/bbgo/pull/1423): IMPROVE: improve trade buffer memory size usage + - [#1419](https://github.com/c9s/bbgo/pull/1419): FIX: [bitget] use the now - 90 days instead of return err if since is 90 days earlier + - [#1412](https://github.com/c9s/bbgo/pull/1412): FEATURE: [bitget] implement order, trade user stream + - [#1417](https://github.com/c9s/bbgo/pull/1417): REFACTOR: [stream] skip pong event on emitting raw message + - [#1415](https://github.com/c9s/bbgo/pull/1415): FEATURE: [bitget] use v2 tickers + - [#1416](https://github.com/c9s/bbgo/pull/1416): FEATURE: [bitget] add response validator + - [#1413](https://github.com/c9s/bbgo/pull/1413): FEATURE: [bitget] use v2 symbols + - [#1410](https://github.com/c9s/bbgo/pull/1410): FEATURE: [bitget] Add query kline + - [#1406](https://github.com/c9s/bbgo/pull/1406): FEATURE: [bitget]add balance event + - [#1409](https://github.com/c9s/bbgo/pull/1409): CHORE: [bybit] print fee rate log + - [#1407](https://github.com/c9s/bbgo/pull/1407): FEATURE: add environment config for disabling some klines defaults + - [#1404](https://github.com/c9s/bbgo/pull/1404): FEATURE: [bitget] support cancel order + - [#1400](https://github.com/c9s/bbgo/pull/1400): FEATURE: [bitget] support query trades + - [#1399](https://github.com/c9s/bbgo/pull/1399): FEATURE: [bitget] support submit order From 102eb61188eb17f0a62873300bffc9b132876081 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 21 Nov 2023 17:06:20 +0800 Subject: [PATCH 215/422] remove unused log --- pkg/strategy/grid2/strategy.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index cc08dc22f1..f3c79803cf 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -1387,13 +1387,11 @@ func (s *Strategy) generateGridOrders(totalQuote, totalBase, lastPrice fixedpoin roundUpQuoteQuantity := quoteQuantity.Round(s.Market.PricePrecision, fixedpoint.Up) if usedQuote.Add(roundUpQuoteQuantity).Compare(totalQuote) > 0 { if i > 0 { - s.logger.Errorf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) return nil, fmt.Errorf("used quote %f > total quote %f, this should not happen", usedQuote.Add(quoteQuantity).Float64(), totalQuote.Float64()) } else { restQuote := totalQuote.Sub(usedQuote) quantity = restQuote.Div(price).Round(s.Market.VolumePrecision, fixedpoint.Down) if s.Market.MinQuantity.Compare(quantity) > 0 { - s.logger.Errorf("the round down quantity (%s) is less than min quantity (%s), we cannot place this order", quantity, s.Market.MinQuantity) return nil, fmt.Errorf("the round down quantity (%s) is less than min quantity (%s), we cannot place this order", quantity, s.Market.MinQuantity) } } From 7cb8da08cddf1eb231d9b8a154257c8f9d52b8bf Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 21 Nov 2023 17:14:33 +0800 Subject: [PATCH 216/422] use asc as order by to query closed orders --- pkg/exchange/max/exchange.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 583494b078..f46acecb0d 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -326,11 +326,11 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s walletType = maxapi.WalletTypeMargin } - req := e.v3client.NewGetWalletClosedOrdersRequest(walletType).Market(market).Limit(1000) - - if !until.IsZero() { - req.Timestamp(until) - } + req := e.v3client.NewGetWalletClosedOrdersRequest(walletType). + Market(market). + Timestamp(since). + Limit(1000). + OrderBy(maxapi.OrderByAsc) maxOrders, err := req.Do(ctx) if err != nil { @@ -338,7 +338,7 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s } for _, maxOrder := range maxOrders { - if maxOrder.CreatedAt.Time().Before(since) { + if maxOrder.CreatedAt.Time().After(until) { continue } order, err2 := toGlobalOrder(maxOrder) @@ -353,7 +353,7 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s if err != nil { return nil, err } - return types.SortOrdersAscending(orders), nil + return orders, nil } func (e *Exchange) CancelAllOrders(ctx context.Context) ([]types.Order, error) { From 51718b6eb2b32946fdb401901b97b97fc6358333 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 21 Nov 2023 17:36:42 +0800 Subject: [PATCH 217/422] pkg/exchnage: add log rate limiter to stream event --- pkg/exchange/bitget/stream.go | 22 +++++++++++++++---- pkg/exchange/bybit/stream.go | 40 ++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index bdab89774c..864b5ba244 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gorilla/websocket" + "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" @@ -19,6 +20,11 @@ import ( var ( pingBytes = []byte("ping") pongBytes = []byte("pong") + + marketTradeLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + tradeLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + orderLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + kLineLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) ) //go:generate callbackgen -type Stream @@ -361,7 +367,9 @@ func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { for _, trade := range m.Events { globalTrade, err := trade.ToGlobal(m.instId) if err != nil { - log.WithError(err).Error("failed to convert to market trade") + if marketTradeLogLimiter.Allow() { + log.WithError(err).Error("failed to convert to market trade") + } return } @@ -377,7 +385,9 @@ func (s *Stream) handleKLineEvent(k KLineEvent) { interval, found := toGlobalInterval[string(k.channel)] if !found { - log.Errorf("unexpected interval %s on KLine subscription", k.channel) + if kLineLogLimiter.Allow() { + log.Errorf("unexpected interval %s on KLine subscription", k.channel) + } return } @@ -415,7 +425,9 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { for _, order := range m.Orders { globalOrder, err := order.toGlobalOrder() if err != nil { - log.Errorf("failed to convert order to global: %s", err) + if orderLogLimiter.Allow() { + log.Errorf("failed to convert order to global: %s", err) + } continue } // The bitget support only snapshot on orders channel, so we use snapshot as update to emit data. @@ -427,7 +439,9 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { if globalOrder.Status == types.OrderStatusPartiallyFilled { trade, err := order.toGlobalTrade() if err != nil { - log.Errorf("failed to convert trade to global: %s", err) + if tradeLogLimiter.Allow() { + log.Errorf("failed to convert trade to global: %s", err) + } continue } s.StandardStream.EmitTradeUpdate(trade) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 47581f2aed..475d4bbd99 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gorilla/websocket" + "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" "github.com/c9s/bbgo/pkg/fixedpoint" @@ -28,6 +29,11 @@ var ( // https://www.bybit.com/en-US/help-center/article/Trading-Fee-Structure defaultTakerFee = fixedpoint.NewFromFloat(0.001) defaultMakerFee = fixedpoint.NewFromFloat(0.001) + + marketTradeLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + tradeLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + orderLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) + kLineLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) ) // MarketInfoProvider calculates trade fees since trading fees are not supported by streaming. @@ -376,7 +382,9 @@ func (s *Stream) handleMarketTradeEvent(events []MarketTradeEvent) { for _, event := range events { trade, err := event.toGlobalTrade() if err != nil { - log.WithError(err).Error("failed to convert to market trade") + if marketTradeLogLimiter.Allow() { + log.WithError(err).Error("failed to convert to market trade") + } continue } @@ -396,7 +404,9 @@ func (s *Stream) handleOrderEvent(events []OrderEvent) { gOrder, err := toGlobalOrder(event.Order) if err != nil { - log.WithError(err).Error("failed to convert to global order") + if orderLogLimiter.Allow() { + log.WithError(err).Error("failed to convert to global order") + } continue } s.StandardStream.EmitOrderUpdate(*gOrder) @@ -411,7 +421,9 @@ func (s *Stream) handleKLineEvent(klineEvent KLineEvent) { for _, event := range klineEvent.KLines { kline, err := event.toGlobalKLine(klineEvent.Symbol) if err != nil { - log.WithError(err).Error("failed to convert to global k line") + if kLineLogLimiter.Allow() { + log.WithError(err).Error("failed to convert to global k line") + } continue } @@ -442,19 +454,23 @@ func (s *Stream) handleTradeEvent(events []TradeEvent) { feeRate.QuoteCoin = market.QuoteCurrency } - // The error log level was utilized due to a detected discrepancy in the fee calculations. - log.Errorf("failed to get %s fee rate, use default taker fee %f, maker fee %f, base coin: %s, quote coin: %s", - event.Symbol, - feeRate.TakerFeeRate.Float64(), - feeRate.MakerFeeRate.Float64(), - feeRate.BaseCoin, - feeRate.QuoteCoin, - ) + if tradeLogLimiter.Allow() { + // The error log level was utilized due to a detected discrepancy in the fee calculations. + log.Errorf("failed to get %s fee rate, use default taker fee %f, maker fee %f, base coin: %s, quote coin: %s", + event.Symbol, + feeRate.TakerFeeRate.Float64(), + feeRate.MakerFeeRate.Float64(), + feeRate.BaseCoin, + feeRate.QuoteCoin, + ) + } } gTrade, err := event.toGlobalTrade(feeRate) if err != nil { - log.WithError(err).Errorf("unable to convert: %+v", event) + if tradeLogLimiter.Allow() { + log.WithError(err).Errorf("unable to convert: %+v", event) + } continue } s.StandardStream.EmitTradeUpdate(*gTrade) From 87d763598f582781364b1662ad1af702f4fb6e39 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 21 Nov 2023 17:59:25 +0800 Subject: [PATCH 218/422] pkg/exchange: use backoff retry --- pkg/exchange/bybit/stream.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 475d4bbd99..3b61661943 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -11,9 +11,9 @@ import ( "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bybit/bybitapi" + "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" ) const ( @@ -351,11 +351,9 @@ func (s *Stream) handleAuthEvent() { var balnacesMap types.BalanceMap var err error - err = util.Retry(ctx, 10, 300*time.Millisecond, func() error { + err = retry.GeneralBackoff(ctx, func() error { balnacesMap, err = s.streamDataProvider.QueryAccountBalances(ctx) return err - }, func(err error) { - log.WithError(err).Error("failed to call query account balances") }) if err != nil { log.WithError(err).Error("no more attempts to retrieve balances") From dbac45aa76735a73e9afde27127e89b8dc994588 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 21 Nov 2023 18:00:49 +0800 Subject: [PATCH 219/422] pkg/util: rm retry --- pkg/util/retry.go | 69 --------------------------- pkg/util/retry_test.go | 106 ----------------------------------------- 2 files changed, 175 deletions(-) delete mode 100644 pkg/util/retry.go delete mode 100644 pkg/util/retry_test.go diff --git a/pkg/util/retry.go b/pkg/util/retry.go deleted file mode 100644 index 9753d78797..0000000000 --- a/pkg/util/retry.go +++ /dev/null @@ -1,69 +0,0 @@ -package util - -import ( - "context" - "time" - - "github.com/pkg/errors" -) - -const ( - InfiniteRetry = 0 -) - -type RetryPredicator func(e error) bool - -// Retry retrys the passed function for "attempts" times, if passed function return error. Setting attempts to zero means keep retrying. -func Retry(ctx context.Context, attempts int, duration time.Duration, fnToRetry func() error, errHandler func(error), predicators ...RetryPredicator) (err error) { - infinite := false - if attempts == InfiniteRetry { - infinite = true - } - - for attempts > 0 || infinite { - select { - case <-ctx.Done(): - errMsg := "return for context done" - if err != nil { - return errors.Wrap(err, errMsg) - } else { - return errors.New(errMsg) - } - default: - if err = fnToRetry(); err == nil { - return nil - } - - if !needRetry(err, predicators) { - return err - } - - err = errors.Wrapf(err, "failed in retry: countdown: %v", attempts) - - if errHandler != nil { - errHandler(err) - } - - if !infinite { - attempts-- - } - - time.Sleep(duration) - } - } - - return err -} - -func needRetry(err error, predicators []RetryPredicator) bool { - if err == nil { - return false - } - - // If no predicators specified, we will retry for all errors - if len(predicators) == 0 { - return true - } - - return predicators[0](err) -} diff --git a/pkg/util/retry_test.go b/pkg/util/retry_test.go deleted file mode 100644 index ff604e1ada..0000000000 --- a/pkg/util/retry_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package util - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" -) - -func addAndCheck(a *int, target int) error { - if *a++; *a == target { - return nil - } else { - return fmt.Errorf("a is not %v. It is %v\n", target, *a) - } -} - -func TestRetry(t *testing.T) { - type test struct { - input int - targetNum int - ans int - ansErr error - } - tests := []test{ - {input: 0, targetNum: 3, ans: 3, ansErr: nil}, - {input: 0, targetNum: 10, ans: 3, ansErr: errors.New("failed in retry")}, - } - - for _, tc := range tests { - errHandled := false - - err := Retry(context.Background(), 3, 1*time.Second, func() error { - return addAndCheck(&tc.input, tc.targetNum) - }, func(e error) { errHandled = true }) - - assert.Equal(t, true, errHandled) - if tc.ansErr == nil { - assert.NoError(t, err) - } else { - assert.Contains(t, err.Error(), tc.ansErr.Error()) - } - assert.Equal(t, tc.ans, tc.input) - } -} - -func TestRetryWithPredicator(t *testing.T) { - type test struct { - count int - f func() error - errHandler func(error) - predicator RetryPredicator - ansCount int - ansErr error - } - knownErr := errors.New("Duplicate entry '1-389837488-1' for key 'UNI_Trade'") - unknownErr := errors.New("Some Error") - tests := []test{ - { - predicator: func(err error) bool { - return !strings.Contains(err.Error(), "Duplicate entry") - }, - f: func() error { return knownErr }, - ansCount: 1, - ansErr: knownErr, - }, - { - predicator: func(err error) bool { - return !strings.Contains(err.Error(), "Duplicate entry") - }, - f: func() error { return unknownErr }, - ansCount: 3, - ansErr: unknownErr, - }, - } - attempts := 3 - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, tc := range tests { - err := Retry(ctx, attempts, 100*time.Millisecond, func() error { - tc.count++ - return tc.f() - }, tc.errHandler, tc.predicator) - - assert.Equal(t, tc.ansCount, tc.count) - assert.EqualError(t, errors.Cause(err), tc.ansErr.Error(), "should be equal") - } -} - -func TestRetryCtxCancel(t *testing.T) { - result := int(0) - target := int(3) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - err := Retry(ctx, 5, 1*time.Second, func() error { return addAndCheck(&result, target) }, func(error) {}) - assert.Error(t, err) - fmt.Println("Error:", err.Error()) - assert.Equal(t, int(0), result) -} From c30dd245506eee3262a3b889842462a318bda848 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Thu, 23 Nov 2023 14:53:22 +0800 Subject: [PATCH 220/422] fix order status length --- ...20231123125402_fix_order_status_length.sql | 11 ++++++ ...20231123125402_fix_order_status_length.sql | 12 +++++++ .../20231123125402_fix_order_status_length.go | 34 +++++++++++++++++++ .../20231123125402_fix_order_status_length.go | 34 +++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 migrations/mysql/20231123125402_fix_order_status_length.sql create mode 100644 migrations/sqlite3/20231123125402_fix_order_status_length.sql create mode 100644 pkg/migrations/mysql/20231123125402_fix_order_status_length.go create mode 100644 pkg/migrations/sqlite3/20231123125402_fix_order_status_length.go diff --git a/migrations/mysql/20231123125402_fix_order_status_length.sql b/migrations/mysql/20231123125402_fix_order_status_length.sql new file mode 100644 index 0000000000..2ed1d35694 --- /dev/null +++ b/migrations/mysql/20231123125402_fix_order_status_length.sql @@ -0,0 +1,11 @@ +-- +up +-- +begin +ALTER TABLE `orders` + CHANGE `status` `status` varchar(20) NOT NULL; +-- +end + +-- +down + +-- +begin +SELECT 1; +-- +end diff --git a/migrations/sqlite3/20231123125402_fix_order_status_length.sql b/migrations/sqlite3/20231123125402_fix_order_status_length.sql new file mode 100644 index 0000000000..583c4051e3 --- /dev/null +++ b/migrations/sqlite3/20231123125402_fix_order_status_length.sql @@ -0,0 +1,12 @@ +-- +up +-- +begin +-- We can not change column type in sqlite +-- However, SQLite does not enforce the length of a VARCHAR, i.e VARCHAR(8) == VARCHAR(20) == TEXT +SELECT 1; +-- +end + +-- +down + +-- +begin +SELECT 1; +-- +end diff --git a/pkg/migrations/mysql/20231123125402_fix_order_status_length.go b/pkg/migrations/mysql/20231123125402_fix_order_status_length.go new file mode 100644 index 0000000000..df7e3f62cf --- /dev/null +++ b/pkg/migrations/mysql/20231123125402_fix_order_status_length.go @@ -0,0 +1,34 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upFixOrderStatusLength, downFixOrderStatusLength) + +} + +func upFixOrderStatusLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "ALTER TABLE `orders`\n CHANGE `status` `status` varchar(20) NOT NULL;") + if err != nil { + return err + } + + return err +} + +func downFixOrderStatusLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} diff --git a/pkg/migrations/sqlite3/20231123125402_fix_order_status_length.go b/pkg/migrations/sqlite3/20231123125402_fix_order_status_length.go new file mode 100644 index 0000000000..2ef6408794 --- /dev/null +++ b/pkg/migrations/sqlite3/20231123125402_fix_order_status_length.go @@ -0,0 +1,34 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upFixOrderStatusLength, downFixOrderStatusLength) + +} + +func upFixOrderStatusLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} + +func downFixOrderStatusLength(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "SELECT 1;") + if err != nil { + return err + } + + return err +} From aea3abae0729ba4eb3da373a51de7eef042263ab Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 13 Nov 2023 16:20:25 +0800 Subject: [PATCH 221/422] FEATURE: new strategy dca2 perparation --- pkg/strategy/dca2/callbacks.go | 1 + pkg/strategy/dca2/strategy.go | 185 +++++++++++++++++++++++++++++ pkg/strategy/dca2/strategy_test.go | 141 ++++++++++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 pkg/strategy/dca2/callbacks.go create mode 100644 pkg/strategy/dca2/strategy.go create mode 100644 pkg/strategy/dca2/strategy_test.go diff --git a/pkg/strategy/dca2/callbacks.go b/pkg/strategy/dca2/callbacks.go new file mode 100644 index 0000000000..718d76e887 --- /dev/null +++ b/pkg/strategy/dca2/callbacks.go @@ -0,0 +1 @@ +package dca2 diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go new file mode 100644 index 0000000000..23c852480d --- /dev/null +++ b/pkg/strategy/dca2/strategy.go @@ -0,0 +1,185 @@ +package dca2 + +import ( + "context" + "fmt" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/sirupsen/logrus" +) + +const ID = "dca2" + +const orderTag = "dca2" + +var log = logrus.WithField("strategy", ID) + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + + // setting + Budget fixedpoint.Value `json:"budget"` + OrderNum int64 `json:"orderNum"` + Margin fixedpoint.Value `json:"margin"` + TakeProfitSpread fixedpoint.Value `json:"takeProfitSpread"` + RoundInterval types.Duration `json:"roundInterval"` + + // OrderGroupID is the group ID used for the strategy instance for canceling orders + OrderGroupID uint32 `json:"orderGroupID"` + + // log + logger *logrus.Entry + LogFields logrus.Fields `json:"logFields"` + + // persistence fields: position and profit + Position *types.Position `persistence:"position"` + ProfitStats *types.ProfitStats `persistence:"profit_stats"` + + // private field + session *bbgo.ExchangeSession + orderExecutor *bbgo.GeneralOrderExecutor + book *types.StreamOrderBook +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) Validate() error { + if s.OrderNum < 1 { + return fmt.Errorf("maxOrderNum can not be < 1") + } + + if s.TakeProfitSpread.Compare(fixedpoint.Zero) <= 0 { + return fmt.Errorf("takeProfitSpread can not be <= 0") + } + + if s.Margin.Compare(fixedpoint.Zero) <= 0 { + return fmt.Errorf("margin can not be <= 0") + } + + // TODO: validate balance is enough + return nil +} + +func (s *Strategy) Defaults() error { + if s.LogFields == nil { + s.LogFields = logrus.Fields{} + } + + s.LogFields["symbol"] = s.Symbol + s.LogFields["strategy"] = ID + return nil +} + +func (s *Strategy) Initialize() error { + s.logger = log.WithFields(s.LogFields) + return nil +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s-%s", ID, s.Symbol) +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{Depth: types.DepthLevel1}) +} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + if s.Position == nil { + s.Position = types.NewPositionFromMarket(s.Market) + } + + if s.ProfitStats == nil { + s.ProfitStats = types.NewProfitStats(s.Market) + } + + instanceID := s.InstanceID() + s.session = session + s.orderExecutor = bbgo.NewGeneralOrderExecutor(session, s.Symbol, ID, instanceID, s.Position) + s.orderExecutor.BindEnvironment(s.Environment) + s.orderExecutor.BindProfitStats(s.ProfitStats) + s.orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + bbgo.Sync(ctx, s) + }) + s.orderExecutor.Bind() + s.book = types.NewStreamBook(s.Symbol) + s.book.BindStream(s.session.MarketDataStream) + + balances := session.GetAccount().Balances() + balance := balances[s.Market.QuoteCurrency] + if balance.Available.Compare(s.Budget) < 0 { + return fmt.Errorf("the available balance of %s is %s which is less than budget setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.Budget) + } + + session.MarketDataStream.OnBookUpdate(func(book types.SliceOrderBook) { + bid, ok := book.BestBid() + if !ok { + return + } + + takeProfitPrice := s.Market.TruncatePrice(s.Position.AverageCost.Mul(fixedpoint.One.Add(s.TakeProfitSpread))) + if bid.Price.Compare(takeProfitPrice) >= 0 { + } + }) + + return nil +} + +func (s *Strategy) generateMakerOrder(budget, askPrice, margin fixedpoint.Value, orderNum int64) ([]types.SubmitOrder, error) { + marginPrice := askPrice.Mul(margin) + price := askPrice + var prices []fixedpoint.Value + var total fixedpoint.Value + for i := 0; i < int(orderNum); i++ { + price = price.Sub(marginPrice) + truncatePrice := s.Market.TruncatePrice(price) + prices = append(prices, truncatePrice) + total = total.Add(truncatePrice) + } + + quantity := budget.Div(total) + quantity = s.Market.TruncateQuantity(quantity) + + var submitOrders []types.SubmitOrder + + for _, price := range prices { + submitOrders = append(submitOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Market: s.Market, + Type: types.OrderTypeLimit, + Price: price, + Side: types.SideTypeBuy, + TimeInForce: types.TimeInForceGTC, + Quantity: quantity, + Tag: orderTag, + GroupID: s.OrderGroupID, + }) + } + + return submitOrders, nil +} + +func (s *Strategy) generateTakeProfitOrder(position *types.Position, takeProfitSpread fixedpoint.Value) types.SubmitOrder { + takeProfitPrice := s.Market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitSpread))) + return types.SubmitOrder{ + Symbol: s.Symbol, + Market: s.Market, + Type: types.OrderTypeLimit, + Price: takeProfitPrice, + Side: types.SideTypeSell, + TimeInForce: types.TimeInForceGTC, + Quantity: position.GetBase().Abs(), + Tag: orderTag, + GroupID: s.OrderGroupID, + } +} diff --git a/pkg/strategy/dca2/strategy_test.go b/pkg/strategy/dca2/strategy_test.go new file mode 100644 index 0000000000..5278819fe1 --- /dev/null +++ b/pkg/strategy/dca2/strategy_test.go @@ -0,0 +1,141 @@ +package dca2 + +import ( + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func number(a interface{}) fixedpoint.Value { + switch v := a.(type) { + case string: + return fixedpoint.MustNewFromString(v) + case int: + return fixedpoint.NewFromInt(int64(v)) + case int64: + return fixedpoint.NewFromInt(int64(v)) + case float64: + return fixedpoint.NewFromFloat(v) + } + + return fixedpoint.Zero +} + +func newTestMarket(symbol string) types.Market { + switch symbol { + case "BTCUSDT": + return types.Market{ + BaseCurrency: "BTC", + QuoteCurrency: "USDT", + TickSize: number(0.01), + StepSize: number(0.000001), + PricePrecision: 2, + VolumePrecision: 8, + MinNotional: number(8.0), + MinQuantity: number(0.0003), + } + case "ETHUSDT": + return types.Market{ + BaseCurrency: "ETH", + QuoteCurrency: "USDT", + TickSize: number(0.01), + StepSize: number(0.00001), + PricePrecision: 2, + VolumePrecision: 6, + MinNotional: number(8.000), + MinQuantity: number(0.0046), + } + } + + // default + return types.Market{ + BaseCurrency: "BTC", + QuoteCurrency: "USDT", + TickSize: number(0.01), + StepSize: number(0.00001), + PricePrecision: 2, + VolumePrecision: 8, + MinNotional: number(10.0), + MinQuantity: number(0.001), + } +} + +func newTestStrategy(va ...string) *Strategy { + symbol := "BTCUSDT" + + if len(va) > 0 { + symbol = va[0] + } + + market := newTestMarket(symbol) + s := &Strategy{ + logger: logrus.NewEntry(logrus.New()), + Symbol: symbol, + Market: market, + } + return s +} + +func TestGenerateMakerOrder(t *testing.T) { + assert := assert.New(t) + + strategy := newTestStrategy() + + budget := number("105000") + askPrice := number("30000") + margin := number("0.05") + submitOrders, err := strategy.generateMakerOrder(budget, askPrice, margin, 4) + if !assert.NoError(err) { + return + } + + assert.Len(submitOrders, 4) + assert.Equal(submitOrders[0].Price, number("28500")) + assert.Equal(submitOrders[0].Quantity, number("1")) + assert.Equal(submitOrders[1].Price, number("27000")) + assert.Equal(submitOrders[1].Quantity, number("1")) + assert.Equal(submitOrders[2].Price, number("25500")) + assert.Equal(submitOrders[2].Quantity, number("1")) + assert.Equal(submitOrders[3].Price, number("24000")) + assert.Equal(submitOrders[3].Quantity, number("1")) +} + +func TestGenerateTakeProfitOrder(t *testing.T) { + assert := assert.New(t) + + strategy := newTestStrategy() + + position := types.NewPositionFromMarket(strategy.Market) + position.AddTrade(types.Trade{ + Side: types.SideTypeBuy, + Price: number("28500"), + Quantity: number("1"), + QuoteQuantity: number("28500"), + Fee: number("0.0015"), + FeeCurrency: strategy.Market.BaseCurrency, + }) + + o := strategy.generateTakeProfitOrder(position, number("10%")) + assert.Equal(number("31397.09"), o.Price) + assert.Equal(number("0.9985"), o.Quantity) + assert.Equal(types.SideTypeSell, o.Side) + assert.Equal(strategy.Symbol, o.Symbol) + + position.AddTrade(types.Trade{ + Side: types.SideTypeBuy, + Price: number("27000"), + Quantity: number("0.5"), + QuoteQuantity: number("13500"), + Fee: number("0.00075"), + FeeCurrency: strategy.Market.BaseCurrency, + }) + o = strategy.generateTakeProfitOrder(position, number("10%")) + assert.Equal(number("30846.26"), o.Price) + assert.Equal(number("1.49775"), o.Quantity) + assert.Equal(types.SideTypeSell, o.Side) + assert.Equal(strategy.Symbol, o.Symbol) + +} From 800148b271909bb794e4f939d4d98120480f59bb Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Thu, 23 Nov 2023 16:45:28 +0800 Subject: [PATCH 222/422] remain only template part --- config/dca2.yaml | 30 ++++++ pkg/exchange/max/stream.go | 3 + pkg/strategy/common/strategy.go | 8 +- pkg/strategy/dca2/callbacks.go | 1 - pkg/strategy/dca2/debug.go | 19 ++++ pkg/strategy/dca2/strategy.go | 139 +++++++++++----------------- pkg/strategy/dca2/strategy_test.go | 141 ----------------------------- 7 files changed, 108 insertions(+), 233 deletions(-) create mode 100644 config/dca2.yaml delete mode 100644 pkg/strategy/dca2/callbacks.go create mode 100644 pkg/strategy/dca2/debug.go delete mode 100644 pkg/strategy/dca2/strategy_test.go diff --git a/config/dca2.yaml b/config/dca2.yaml new file mode 100644 index 0000000000..8c10453ca4 --- /dev/null +++ b/config/dca2.yaml @@ -0,0 +1,30 @@ +--- +backtest: + startTime: "2023-06-01" + endTime: "2023-07-01" + sessions: + - max + symbols: + - ETHUSDT + accounts: + binance: + balances: + USDT: 20_000.0 + +persistence: + redis: + host: 127.0.0.1 + port: 6379 + db: 0 + +exchangeStrategies: + +- on: max + dca2: + symbol: ETHUSDT + short: false + budget: 5000 + maxOrderNum: 10 + priceDeviation: 1% + takeProfitRatio: 1% + coolDownInterval: 5m diff --git a/pkg/exchange/max/stream.go b/pkg/exchange/max/stream.go index 72bece012d..12a9ae32f7 100644 --- a/pkg/exchange/max/stream.go +++ b/pkg/exchange/max/stream.go @@ -96,6 +96,9 @@ func (s *Stream) handleConnect() { case types.DepthLevelMedium: depth = 20 + case types.DepthLevel1: + depth = 1 + case types.DepthLevel5: depth = 5 diff --git a/pkg/strategy/common/strategy.go b/pkg/strategy/common/strategy.go index 991b6b8c58..cb040be2cf 100644 --- a/pkg/strategy/common/strategy.go +++ b/pkg/strategy/common/strategy.go @@ -69,9 +69,11 @@ func (s *Strategy) Initialize(ctx context.Context, environ *bbgo.Environment, se s.OrderExecutor.BindEnvironment(environ) s.OrderExecutor.BindProfitStats(s.ProfitStats) s.OrderExecutor.Bind() - s.OrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - // bbgo.Sync(ctx, s) - }) + /* + s.OrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + bbgo.Sync(ctx, s) + }) + */ if !s.PositionHardLimit.IsZero() && !s.MaxPositionQuantity.IsZero() { log.Infof("positionHardLimit and maxPositionQuantity are configured, setting up PositionRiskControl...") diff --git a/pkg/strategy/dca2/callbacks.go b/pkg/strategy/dca2/callbacks.go deleted file mode 100644 index 718d76e887..0000000000 --- a/pkg/strategy/dca2/callbacks.go +++ /dev/null @@ -1 +0,0 @@ -package dca2 diff --git a/pkg/strategy/dca2/debug.go b/pkg/strategy/dca2/debug.go new file mode 100644 index 0000000000..8e44bf916c --- /dev/null +++ b/pkg/strategy/dca2/debug.go @@ -0,0 +1,19 @@ +package dca2 + +import ( + "fmt" + "strings" + + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) debugOrders(submitOrders []types.Order) { + var sb strings.Builder + sb.WriteString("DCA ORDERS[\n") + for i, order := range submitOrders { + sb.WriteString(fmt.Sprintf("%3d) ", i+1) + order.String() + "\n") + } + sb.WriteString("] END OF DCA ORDERS") + + s.logger.Info(sb.String()) +} diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 23c852480d..55e13433bc 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -3,10 +3,15 @@ package dca2 import ( "context" "fmt" + "math" + "sync" + "time" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" "github.com/sirupsen/logrus" ) @@ -21,17 +26,20 @@ func init() { } type Strategy struct { + *common.Strategy + Environment *bbgo.Environment Market types.Market Symbol string `json:"symbol"` // setting + Short bool `json:"short"` Budget fixedpoint.Value `json:"budget"` - OrderNum int64 `json:"orderNum"` - Margin fixedpoint.Value `json:"margin"` - TakeProfitSpread fixedpoint.Value `json:"takeProfitSpread"` - RoundInterval types.Duration `json:"roundInterval"` + MaxOrderNum int64 `json:"maxOrderNum"` + PriceDeviation fixedpoint.Value `json:"priceDeviation"` + TakeProfitRatio fixedpoint.Value `json:"takeProfitRatio"` + CoolDownInterval types.Duration `json:"coolDownInterval"` // OrderGroupID is the group ID used for the strategy instance for canceling orders OrderGroupID uint32 `json:"orderGroupID"` @@ -40,14 +48,12 @@ type Strategy struct { logger *logrus.Entry LogFields logrus.Fields `json:"logFields"` - // persistence fields: position and profit - Position *types.Position `persistence:"position"` - ProfitStats *types.ProfitStats `persistence:"profit_stats"` - // private field - session *bbgo.ExchangeSession - orderExecutor *bbgo.GeneralOrderExecutor - book *types.StreamOrderBook + mu sync.Mutex + makerSide types.SideType + takeProfitSide types.SideType + takeProfitPrice fixedpoint.Value + startTimeOfNextRound time.Time } func (s *Strategy) ID() string { @@ -55,15 +61,15 @@ func (s *Strategy) ID() string { } func (s *Strategy) Validate() error { - if s.OrderNum < 1 { + if s.MaxOrderNum < 1 { return fmt.Errorf("maxOrderNum can not be < 1") } - if s.TakeProfitSpread.Compare(fixedpoint.Zero) <= 0 { + if s.TakeProfitRatio.Sign() <= 0 { return fmt.Errorf("takeProfitSpread can not be <= 0") } - if s.Margin.Compare(fixedpoint.Zero) <= 0 { + if s.PriceDeviation.Sign() <= 0 { return fmt.Errorf("margin can not be <= 0") } @@ -91,95 +97,52 @@ func (s *Strategy) InstanceID() string { } func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - session.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{Depth: types.DepthLevel1}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - if s.Position == nil { - s.Position = types.NewPositionFromMarket(s.Market) + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + instanceID := s.InstanceID() + + if s.Short { + s.makerSide = types.SideTypeSell + s.takeProfitSide = types.SideTypeBuy + } else { + s.makerSide = types.SideTypeBuy + s.takeProfitSide = types.SideTypeSell } - if s.ProfitStats == nil { - s.ProfitStats = types.NewProfitStats(s.Market) + if s.OrderGroupID == 0 { + s.OrderGroupID = util.FNV32(instanceID) % math.MaxInt32 } - instanceID := s.InstanceID() - s.session = session - s.orderExecutor = bbgo.NewGeneralOrderExecutor(session, s.Symbol, ID, instanceID, s.Position) - s.orderExecutor.BindEnvironment(s.Environment) - s.orderExecutor.BindProfitStats(s.ProfitStats) - s.orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + // order executor + s.OrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + s.logger.Infof("position: %s", s.Position.String()) bbgo.Sync(ctx, s) - }) - s.orderExecutor.Bind() - s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(s.session.MarketDataStream) - - balances := session.GetAccount().Balances() - balance := balances[s.Market.QuoteCurrency] - if balance.Available.Compare(s.Budget) < 0 { - return fmt.Errorf("the available balance of %s is %s which is less than budget setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.Budget) - } - session.MarketDataStream.OnBookUpdate(func(book types.SliceOrderBook) { - bid, ok := book.BestBid() - if !ok { - return - } + // update take profit price here + }) - takeProfitPrice := s.Market.TruncatePrice(s.Position.AverageCost.Mul(fixedpoint.One.Add(s.TakeProfitSpread))) - if bid.Price.Compare(takeProfitPrice) >= 0 { - } + session.MarketDataStream.OnKLine(func(kline types.KLine) { + // check price here }) - return nil -} + session.UserDataStream.OnAuth(func() { + s.logger.Info("user data stream authenticated, start the process") + // decide state here + }) -func (s *Strategy) generateMakerOrder(budget, askPrice, margin fixedpoint.Value, orderNum int64) ([]types.SubmitOrder, error) { - marginPrice := askPrice.Mul(margin) - price := askPrice - var prices []fixedpoint.Value - var total fixedpoint.Value - for i := 0; i < int(orderNum); i++ { - price = price.Sub(marginPrice) - truncatePrice := s.Market.TruncatePrice(price) - prices = append(prices, truncatePrice) - total = total.Add(truncatePrice) + balances, err := session.Exchange.QueryAccountBalances(ctx) + if err != nil { + return err } - quantity := budget.Div(total) - quantity = s.Market.TruncateQuantity(quantity) - - var submitOrders []types.SubmitOrder - - for _, price := range prices { - submitOrders = append(submitOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Market: s.Market, - Type: types.OrderTypeLimit, - Price: price, - Side: types.SideTypeBuy, - TimeInForce: types.TimeInForceGTC, - Quantity: quantity, - Tag: orderTag, - GroupID: s.OrderGroupID, - }) + balance := balances[s.Market.QuoteCurrency] + if balance.Available.Compare(s.Budget) < 0 { + return fmt.Errorf("the available balance of %s is %s which is less than budget setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.Budget) } - return submitOrders, nil -} - -func (s *Strategy) generateTakeProfitOrder(position *types.Position, takeProfitSpread fixedpoint.Value) types.SubmitOrder { - takeProfitPrice := s.Market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitSpread))) - return types.SubmitOrder{ - Symbol: s.Symbol, - Market: s.Market, - Type: types.OrderTypeLimit, - Price: takeProfitPrice, - Side: types.SideTypeSell, - TimeInForce: types.TimeInForceGTC, - Quantity: position.GetBase().Abs(), - Tag: orderTag, - GroupID: s.OrderGroupID, - } + return nil } diff --git a/pkg/strategy/dca2/strategy_test.go b/pkg/strategy/dca2/strategy_test.go deleted file mode 100644 index 5278819fe1..0000000000 --- a/pkg/strategy/dca2/strategy_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package dca2 - -import ( - "testing" - - "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/types" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -func number(a interface{}) fixedpoint.Value { - switch v := a.(type) { - case string: - return fixedpoint.MustNewFromString(v) - case int: - return fixedpoint.NewFromInt(int64(v)) - case int64: - return fixedpoint.NewFromInt(int64(v)) - case float64: - return fixedpoint.NewFromFloat(v) - } - - return fixedpoint.Zero -} - -func newTestMarket(symbol string) types.Market { - switch symbol { - case "BTCUSDT": - return types.Market{ - BaseCurrency: "BTC", - QuoteCurrency: "USDT", - TickSize: number(0.01), - StepSize: number(0.000001), - PricePrecision: 2, - VolumePrecision: 8, - MinNotional: number(8.0), - MinQuantity: number(0.0003), - } - case "ETHUSDT": - return types.Market{ - BaseCurrency: "ETH", - QuoteCurrency: "USDT", - TickSize: number(0.01), - StepSize: number(0.00001), - PricePrecision: 2, - VolumePrecision: 6, - MinNotional: number(8.000), - MinQuantity: number(0.0046), - } - } - - // default - return types.Market{ - BaseCurrency: "BTC", - QuoteCurrency: "USDT", - TickSize: number(0.01), - StepSize: number(0.00001), - PricePrecision: 2, - VolumePrecision: 8, - MinNotional: number(10.0), - MinQuantity: number(0.001), - } -} - -func newTestStrategy(va ...string) *Strategy { - symbol := "BTCUSDT" - - if len(va) > 0 { - symbol = va[0] - } - - market := newTestMarket(symbol) - s := &Strategy{ - logger: logrus.NewEntry(logrus.New()), - Symbol: symbol, - Market: market, - } - return s -} - -func TestGenerateMakerOrder(t *testing.T) { - assert := assert.New(t) - - strategy := newTestStrategy() - - budget := number("105000") - askPrice := number("30000") - margin := number("0.05") - submitOrders, err := strategy.generateMakerOrder(budget, askPrice, margin, 4) - if !assert.NoError(err) { - return - } - - assert.Len(submitOrders, 4) - assert.Equal(submitOrders[0].Price, number("28500")) - assert.Equal(submitOrders[0].Quantity, number("1")) - assert.Equal(submitOrders[1].Price, number("27000")) - assert.Equal(submitOrders[1].Quantity, number("1")) - assert.Equal(submitOrders[2].Price, number("25500")) - assert.Equal(submitOrders[2].Quantity, number("1")) - assert.Equal(submitOrders[3].Price, number("24000")) - assert.Equal(submitOrders[3].Quantity, number("1")) -} - -func TestGenerateTakeProfitOrder(t *testing.T) { - assert := assert.New(t) - - strategy := newTestStrategy() - - position := types.NewPositionFromMarket(strategy.Market) - position.AddTrade(types.Trade{ - Side: types.SideTypeBuy, - Price: number("28500"), - Quantity: number("1"), - QuoteQuantity: number("28500"), - Fee: number("0.0015"), - FeeCurrency: strategy.Market.BaseCurrency, - }) - - o := strategy.generateTakeProfitOrder(position, number("10%")) - assert.Equal(number("31397.09"), o.Price) - assert.Equal(number("0.9985"), o.Quantity) - assert.Equal(types.SideTypeSell, o.Side) - assert.Equal(strategy.Symbol, o.Symbol) - - position.AddTrade(types.Trade{ - Side: types.SideTypeBuy, - Price: number("27000"), - Quantity: number("0.5"), - QuoteQuantity: number("13500"), - Fee: number("0.00075"), - FeeCurrency: strategy.Market.BaseCurrency, - }) - o = strategy.generateTakeProfitOrder(position, number("10%")) - assert.Equal(number("30846.26"), o.Price) - assert.Equal(number("1.49775"), o.Quantity) - assert.Equal(types.SideTypeSell, o.Side) - assert.Equal(strategy.Symbol, o.Symbol) - -} From 19be49fca88ed75e22a4f91dae72db6b8996541c Mon Sep 17 00:00:00 2001 From: chiahung Date: Fri, 24 Nov 2023 14:17:19 +0800 Subject: [PATCH 223/422] FIX: use original status for recover --- pkg/strategy/grid2/grid_recover.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/strategy/grid2/grid_recover.go b/pkg/strategy/grid2/grid_recover.go index 945f2c0380..70ae575dd3 100644 --- a/pkg/strategy/grid2/grid_recover.go +++ b/pkg/strategy/grid2/grid_recover.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/c9s/bbgo/pkg/bbgo" + maxapi "github.com/c9s/bbgo/pkg/exchange/max/maxapi" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -18,6 +19,8 @@ func (s *Strategy) recoverByScanningTrades(ctx context.Context, session *bbgo.Ex defer func() { s.updateGridNumOfOrdersMetricsWithLock() }() + isMax := isMaxExchange(session.Exchange) + s.logger.Infof("isMax: %t", isMax) historyService, implemented := session.Exchange.(types.ExchangeTradeHistoryService) // if the exchange doesn't support ExchangeTradeHistoryService, do not run recover @@ -71,6 +74,10 @@ func (s *Strategy) recoverByScanningTrades(ctx context.Context, session *bbgo.Ex // emit the filled orders activeOrderBook := s.orderExecutor.ActiveMakerOrders() for _, filledOrder := range filledOrders { + if isMax && filledOrder.OriginalStatus != string(maxapi.OrderStateDone) { + activeOrderBook.Add(filledOrder) + continue + } activeOrderBook.EmitFilled(filledOrder) } From 8f5f5dfeed0913c5c71476a1df447ef037bfcaf3 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 22 Nov 2023 17:34:26 +0800 Subject: [PATCH 224/422] bbgo: add executed quantity check when order status is OrderStatusPartiallyFilled --- pkg/bbgo/activeorderbook.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 5d8c0812d4..2f80716d71 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -349,6 +349,12 @@ func isNewerUpdate(a, b types.Order) bool { switch b.Status { case types.OrderStatusNew: return true + case types.OrderStatusPartiallyFilled: + // unknown for equal + if a.ExecutedQuantity.Compare(b.ExecutedQuantity) > 0 { + return true + } + } case types.OrderStatusFilled: From 8afd3c9ee1cb9ef8fc82534fc4e33b0bcbe713e1 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 22 Nov 2023 17:37:10 +0800 Subject: [PATCH 225/422] bbgo: add test Test_isNewerUpdate --- pkg/bbgo/activeorderbook_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/bbgo/activeorderbook_test.go b/pkg/bbgo/activeorderbook_test.go index 65cd6aa57a..65a53f26f2 100644 --- a/pkg/bbgo/activeorderbook_test.go +++ b/pkg/bbgo/activeorderbook_test.go @@ -64,3 +64,16 @@ func TestActiveOrderBook_pendingOrders(t *testing.T) { assert.True(t, filled, "filled event should be fired") } + +func Test_isNewerUpdate(t *testing.T) { + a := types.Order{ + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: number(0.2), + } + b := types.Order{ + Status: types.OrderStatusPartiallyFilled, + ExecutedQuantity: number(0.1), + } + ret := isNewerUpdate(a, b) + assert.True(t, ret) +} From 9e663916ed35979e46853e5935d680aaa459fc13 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 22 Nov 2023 17:43:38 +0800 Subject: [PATCH 226/422] bbgo: add test case for isNewerUpdateTime --- pkg/bbgo/activeorderbook.go | 4 ++++ pkg/bbgo/activeorderbook_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 2f80716d71..4855cc89d2 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -364,6 +364,10 @@ func isNewerUpdate(a, b types.Order) bool { } } + return isNewerUpdateTime(a, b) +} + +func isNewerUpdateTime(a, b types.Order) bool { au := time.Time(a.UpdateTime) bu := time.Time(b.UpdateTime) diff --git a/pkg/bbgo/activeorderbook_test.go b/pkg/bbgo/activeorderbook_test.go index 65a53f26f2..c09a5d8434 100644 --- a/pkg/bbgo/activeorderbook_test.go +++ b/pkg/bbgo/activeorderbook_test.go @@ -77,3 +77,14 @@ func Test_isNewerUpdate(t *testing.T) { ret := isNewerUpdate(a, b) assert.True(t, ret) } + +func Test_isNewerUpdateTime(t *testing.T) { + a := types.Order{ + UpdateTime: types.NewTimeFromUnix(200, 0), + } + b := types.Order{ + UpdateTime: types.NewTimeFromUnix(100, 0), + } + ret := isNewerUpdateTime(a, b) + assert.True(t, ret) +} From 6b27722b03a85d89dea34c6e4888470bf8645f4a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 22 Nov 2023 17:44:41 +0800 Subject: [PATCH 227/422] bbgo: rename func isNewerOrderUpdate --- pkg/bbgo/activeorderbook.go | 8 ++++---- pkg/bbgo/activeorderbook_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 4855cc89d2..547ba46e7b 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -335,7 +335,7 @@ func (b *ActiveOrderBook) Add(orders ...types.Order) { } } -func isNewerUpdate(a, b types.Order) bool { +func isNewerOrderUpdate(a, b types.Order) bool { // compare state first switch a.Status { @@ -364,10 +364,10 @@ func isNewerUpdate(a, b types.Order) bool { } } - return isNewerUpdateTime(a, b) + return isNewerOrderUpdateTime(a, b) } -func isNewerUpdateTime(a, b types.Order) bool { +func isNewerOrderUpdateTime(a, b types.Order) bool { au := time.Time(a.UpdateTime) bu := time.Time(b.UpdateTime) @@ -388,7 +388,7 @@ func (b *ActiveOrderBook) add(order types.Order) { // if the pending order update time is newer than the adding order // we should use the pending order rather than the adding order. // if pending order is older, than we should add the new one, and drop the pending order - if isNewerUpdate(pendingOrder, order) { + if isNewerOrderUpdate(pendingOrder, order) { order = pendingOrder } diff --git a/pkg/bbgo/activeorderbook_test.go b/pkg/bbgo/activeorderbook_test.go index c09a5d8434..eb0a28e354 100644 --- a/pkg/bbgo/activeorderbook_test.go +++ b/pkg/bbgo/activeorderbook_test.go @@ -74,7 +74,7 @@ func Test_isNewerUpdate(t *testing.T) { Status: types.OrderStatusPartiallyFilled, ExecutedQuantity: number(0.1), } - ret := isNewerUpdate(a, b) + ret := isNewerOrderUpdate(a, b) assert.True(t, ret) } @@ -85,6 +85,6 @@ func Test_isNewerUpdateTime(t *testing.T) { b := types.Order{ UpdateTime: types.NewTimeFromUnix(100, 0), } - ret := isNewerUpdateTime(a, b) + ret := isNewerOrderUpdateTime(a, b) assert.True(t, ret) } From 326a0c61287cf97dd07c637764d738cb05c9d6da Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 22 Nov 2023 17:56:13 +0800 Subject: [PATCH 228/422] bbgo: replace update time check with isNewerOrderUpdate func call --- pkg/bbgo/activeorderbook.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 547ba46e7b..85a054def4 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -276,8 +276,7 @@ func (b *ActiveOrderBook) Update(order types.Order) { // if order update time is too old, skip it if previousOrder, ok := b.orders.Get(order.OrderID); ok { - previousUpdateTime := previousOrder.UpdateTime.Time() - if !previousUpdateTime.IsZero() && order.UpdateTime.Before(previousUpdateTime) { + if isNewerOrderUpdate(previousOrder, order) { log.Infof("[ActiveOrderBook] order #%d updateTime %s is out of date, skip it", order.OrderID, order.UpdateTime) b.mu.Unlock() return From 55cbe806d99ecc5e1158673b3bf70fb78bd3d2d4 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 25 Nov 2023 13:22:03 +0800 Subject: [PATCH 229/422] bbgo: fix isNewerOrderUpdate check and tests --- pkg/bbgo/activeorderbook.go | 7 +++- pkg/bbgo/activeorderbook_test.go | 59 +++++++++++++++++++------------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 85a054def4..ee1cdf0567 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -276,7 +276,10 @@ func (b *ActiveOrderBook) Update(order types.Order) { // if order update time is too old, skip it if previousOrder, ok := b.orders.Get(order.OrderID); ok { - if isNewerOrderUpdate(previousOrder, order) { + // the arguments ordering is important here + // if we can't detect which is newer, isNewerOrderUpdate returns false + // if you pass two same objects to isNewerOrderUpdate, it returns false + if !isNewerOrderUpdate(order, previousOrder) { log.Infof("[ActiveOrderBook] order #%d updateTime %s is out of date, skip it", order.OrderID, order.UpdateTime) b.mu.Unlock() return @@ -387,7 +390,9 @@ func (b *ActiveOrderBook) add(order types.Order) { // if the pending order update time is newer than the adding order // we should use the pending order rather than the adding order. // if pending order is older, than we should add the new one, and drop the pending order + log.Infof("found pending order update") if isNewerOrderUpdate(pendingOrder, order) { + log.Infof("pending order update is newer") order = pendingOrder } diff --git a/pkg/bbgo/activeorderbook_test.go b/pkg/bbgo/activeorderbook_test.go index eb0a28e354..f9863156e0 100644 --- a/pkg/bbgo/activeorderbook_test.go +++ b/pkg/bbgo/activeorderbook_test.go @@ -7,61 +7,72 @@ import ( "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/fixedpoint" + . "github.com/c9s/bbgo/pkg/testing/testhelper" "github.com/c9s/bbgo/pkg/types" ) func TestActiveOrderBook_pendingOrders(t *testing.T) { now := time.Now() + t1 := now + t2 := now.Add(time.Millisecond) - ob := NewActiveOrderBook("") + ob := NewActiveOrderBook("BTCUSDT") filled := false ob.OnFilled(func(o types.Order) { filled = true }) - // if we received filled order first - // should be added to pending orders - ob.Update(types.Order{ + quantity := Number("0.01") + orderUpdate1 := types.Order{ OrderID: 99, SubmitOrder: types.SubmitOrder{ Symbol: "BTCUSDT", Side: types.SideTypeBuy, Type: types.OrderTypeLimit, - Quantity: number(0.01), - Price: number(19000.0), + Quantity: quantity, + Price: Number(19000.0), AveragePrice: fixedpoint.Zero, StopPrice: fixedpoint.Zero, }, - Status: types.OrderStatusFilled, - CreationTime: types.Time(now), - UpdateTime: types.Time(now), - }) - - assert.Len(t, ob.pendingOrderUpdates.Orders(), 1) - - o99, ok := ob.pendingOrderUpdates.Get(99) - if assert.True(t, ok) { - assert.Equal(t, types.OrderStatusFilled, o99.Status) + ExecutedQuantity: Number(0.0), + Status: types.OrderStatusNew, + CreationTime: types.Time(t1), + UpdateTime: types.Time(t1), } - // should be added to pending orders - ob.Add(types.Order{ + orderUpdate2 := types.Order{ OrderID: 99, SubmitOrder: types.SubmitOrder{ Symbol: "BTCUSDT", Side: types.SideTypeBuy, Type: types.OrderTypeLimit, - Quantity: number(0.01), - Price: number(19000.0), + Quantity: quantity, + Price: Number(19000.0), AveragePrice: fixedpoint.Zero, StopPrice: fixedpoint.Zero, }, - Status: types.OrderStatusNew, - CreationTime: types.Time(now), - UpdateTime: types.Time(now), - }) + ExecutedQuantity: quantity, + Status: types.OrderStatusFilled, + CreationTime: types.Time(t1), + UpdateTime: types.Time(t2), + } + + assert.True(t, isNewerOrderUpdate(orderUpdate2, orderUpdate1), "orderUpdate2 should be newer than orderUpdate1") + + // if we received filled order first + // should be added to pending orders + ob.Update(orderUpdate2) + assert.Len(t, ob.pendingOrderUpdates.Orders(), 1) + + o99, ok := ob.pendingOrderUpdates.Get(99) + if assert.True(t, ok) { + assert.Equal(t, types.OrderStatusFilled, o99.Status) + } + // when adding the older order update to the book, + // it should trigger the filled event once the order is registered to the active order book + ob.Add(orderUpdate1) assert.True(t, filled, "filled event should be fired") } From a4ccad9463130897ed46167e6b0a82e02e1e1ce4 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 29 Nov 2023 18:26:01 +0800 Subject: [PATCH 230/422] FIX: deactivate exit when position in closing --- pkg/bbgo/exit_cumulated_volume_take_profit.go | 3 +-- pkg/bbgo/exit_hh_ll_stop.go | 2 +- pkg/bbgo/exit_lower_shadow_take_profit.go | 3 +-- pkg/bbgo/exit_roi_stop_loss.go | 2 +- pkg/bbgo/exit_roi_take_profit.go | 2 +- pkg/bbgo/exit_trailing_stop.go | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/bbgo/exit_cumulated_volume_take_profit.go b/pkg/bbgo/exit_cumulated_volume_take_profit.go index 440dedf492..332a09b2ac 100644 --- a/pkg/bbgo/exit_cumulated_volume_take_profit.go +++ b/pkg/bbgo/exit_cumulated_volume_take_profit.go @@ -15,7 +15,6 @@ import ( // To query the historical quote volume, use the following query: // // > SELECT start_time, `interval`, quote_volume, open, close FROM binance_klines WHERE symbol = 'ETHUSDT' AND `interval` = '5m' ORDER BY quote_volume DESC LIMIT 20; -// type CumulatedVolumeTakeProfit struct { Symbol string `json:"symbol"` @@ -39,7 +38,7 @@ func (s *CumulatedVolumeTakeProfit) Bind(session *ExchangeSession, orderExecutor session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, s.Interval, func(kline types.KLine) { closePrice := kline.Close openPrice := kline.Open - if position.IsClosed() || position.IsDust(closePrice) { + if position.IsClosed() || position.IsDust(closePrice) || position.IsClosing() { return } diff --git a/pkg/bbgo/exit_hh_ll_stop.go b/pkg/bbgo/exit_hh_ll_stop.go index 6d6847af54..771a9692b4 100644 --- a/pkg/bbgo/exit_hh_ll_stop.go +++ b/pkg/bbgo/exit_hh_ll_stop.go @@ -62,7 +62,7 @@ func (s *HigherHighLowerLowStop) Subscribe(session *ExchangeSession) { // determine whether this stop should be activated func (s *HigherHighLowerLowStop) updateActivated(position *types.Position, closePrice fixedpoint.Value) { // deactivate when no position - if position.IsClosed() || position.IsDust(closePrice) { + if position.IsClosed() || position.IsDust(closePrice) || position.IsClosing() { s.activated = false return diff --git a/pkg/bbgo/exit_lower_shadow_take_profit.go b/pkg/bbgo/exit_lower_shadow_take_profit.go index 32ac9928e4..954a05b8d6 100644 --- a/pkg/bbgo/exit_lower_shadow_take_profit.go +++ b/pkg/bbgo/exit_lower_shadow_take_profit.go @@ -30,11 +30,10 @@ func (s *LowerShadowTakeProfit) Bind(session *ExchangeSession, orderExecutor *Ge stdIndicatorSet := session.StandardIndicatorSet(s.Symbol) ewma := stdIndicatorSet.EWMA(s.IntervalWindow) - position := orderExecutor.Position() session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, s.Interval, func(kline types.KLine) { closePrice := kline.Close - if position.IsClosed() || position.IsDust(closePrice) { + if position.IsClosed() || position.IsDust(closePrice) || position.IsClosing() { return } diff --git a/pkg/bbgo/exit_roi_stop_loss.go b/pkg/bbgo/exit_roi_stop_loss.go index bdcf6e4809..b026ecfa0e 100644 --- a/pkg/bbgo/exit_roi_stop_loss.go +++ b/pkg/bbgo/exit_roi_stop_loss.go @@ -45,7 +45,7 @@ func (s *RoiStopLoss) Bind(session *ExchangeSession, orderExecutor *GeneralOrder } func (s *RoiStopLoss) checkStopPrice(closePrice fixedpoint.Value, position *types.Position) { - if position.IsClosed() || position.IsDust(closePrice) { + if position.IsClosed() || position.IsDust(closePrice) || position.IsClosing() { return } diff --git a/pkg/bbgo/exit_roi_take_profit.go b/pkg/bbgo/exit_roi_take_profit.go index d0d4e8e72a..c5853a0e51 100644 --- a/pkg/bbgo/exit_roi_take_profit.go +++ b/pkg/bbgo/exit_roi_take_profit.go @@ -29,7 +29,7 @@ func (s *RoiTakeProfit) Bind(session *ExchangeSession, orderExecutor *GeneralOrd position := orderExecutor.Position() session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, types.Interval1m, func(kline types.KLine) { closePrice := kline.Close - if position.IsClosed() || position.IsDust(closePrice) { + if position.IsClosed() || position.IsDust(closePrice) || position.IsClosing() { return } diff --git a/pkg/bbgo/exit_trailing_stop.go b/pkg/bbgo/exit_trailing_stop.go index f0db8892b5..adfc28ae48 100644 --- a/pkg/bbgo/exit_trailing_stop.go +++ b/pkg/bbgo/exit_trailing_stop.go @@ -93,7 +93,7 @@ func (s *TrailingStop2) getRatio(price fixedpoint.Value, position *types.Positio } func (s *TrailingStop2) checkStopPrice(price fixedpoint.Value, position *types.Position) error { - if position.IsClosed() || position.IsDust(price) { + if position.IsClosed() || position.IsDust(price) || position.IsClosing() { return nil } From cdeb0bc908a636b0a71239d12fcddaa7e9464eed Mon Sep 17 00:00:00 2001 From: root Date: Wed, 29 Nov 2023 18:37:28 +0800 Subject: [PATCH 231/422] FIX: format minimal profit to percent --- pkg/bbgo/exit_trailing_stop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bbgo/exit_trailing_stop.go b/pkg/bbgo/exit_trailing_stop.go index adfc28ae48..a46024c34c 100644 --- a/pkg/bbgo/exit_trailing_stop.go +++ b/pkg/bbgo/exit_trailing_stop.go @@ -101,7 +101,7 @@ func (s *TrailingStop2) checkStopPrice(price fixedpoint.Value, position *types.P // check if we have the minimal profit roi := position.ROI(price) if roi.Compare(s.MinProfit) >= 0 { - Notify("[trailingStop] activated: %s ROI %s > minimal profit ratio %f", s.Symbol, roi.Percentage(), s.MinProfit.Float64()) + Notify("[trailingStop] activated: %s ROI %s > minimal profit ratio %s", s.Symbol, roi.Percentage(), s.MinProfit.Percentage()) s.activated = true } } else if !s.ActivationRatio.IsZero() { From 21c037a877d8832e088263bf66e90f8d662371ab Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 4 Dec 2023 20:01:54 +0800 Subject: [PATCH 232/422] FIX: fix list closed orders api limit --- pkg/exchange/max/exchange.go | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 369d4f5a30..dc1f29237b 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -270,18 +270,34 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return orders, err } -// lastOrderID is not supported on MAX func (e *Exchange) QueryClosedOrders( ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, ) ([]types.Order, error) { - log.Warn("!!!MAX EXCHANGE API NOTICE!!! the since/until conditions will not be effected on closed orders query, max exchange does not support time-range-based query") if !since.IsZero() || !until.IsZero() { - return e.queryClosedOrdersByTime(ctx, symbol, since, until) + return e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByAsc, 1000) } return e.queryClosedOrdersByLastOrderID(ctx, symbol, lastOrderID) } +func (e *Exchange) QueryClosedOrdersDesc( + ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, +) ([]types.Order, error) { + closedOrders, err := e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByDesc, 1000) + if lastOrderID == 0 { + return closedOrders, err + } + + var filterClosedOrders []types.Order + for _, closedOrder := range filterClosedOrders { + if closedOrder.OrderID > lastOrderID { + filterClosedOrders = append(filterClosedOrders, closedOrder) + } + } + + return filterClosedOrders, err +} + func (e *Exchange) queryClosedOrdersByLastOrderID( ctx context.Context, symbol string, lastOrderID uint64, ) (orders []types.Order, err error) { @@ -325,11 +341,21 @@ func (e *Exchange) queryClosedOrdersByLastOrderID( return types.SortOrdersAscending(orders), nil } -func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, since, until time.Time) (orders []types.Order, err error) { +func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, since, until time.Time, orderByType maxapi.OrderByType, limit uint) (orders []types.Order, err error) { if err := e.closedOrderQueryLimiter.Wait(ctx); err != nil { return orders, err } + // there is since limit for closed orders API + sinceLimit := time.Date(2018, time.January, 1, 0, 0, 0, 0, time.Local) + if since.Before(sinceLimit) { + since = sinceLimit + } + + if until.IsZero() { + until = time.Now() + } + market := toLocalSymbol(symbol) walletType := maxapi.WalletTypeSpot if e.MarginSettings.IsMargin { @@ -338,9 +364,23 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s req := e.v3client.NewGetWalletClosedOrdersRequest(walletType). Market(market). - Timestamp(since). - Limit(1000). - OrderBy(maxapi.OrderByAsc) + Limit(limit). + OrderBy(orderByType) + + switch orderByType { + case maxapi.OrderByAsc: + req.Timestamp(since) + case maxapi.OrderByDesc: + req.Timestamp(until) + case maxapi.OrderByAscUpdatedAt: + // not implement yet + return nil, fmt.Errorf("unsupported order by type: %s", orderByType) + case maxapi.OrderByDescUpdatedAt: + // not implement yet + return nil, fmt.Errorf("unsupported order by type: %s", orderByType) + default: + return nil, fmt.Errorf("unsupported order by type: %s", orderByType) + } maxOrders, err := req.Do(ctx) if err != nil { @@ -348,9 +388,11 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s } for _, maxOrder := range maxOrders { - if maxOrder.CreatedAt.Time().After(until) { + createdAt := maxOrder.CreatedAt.Time() + if createdAt.Before(since) || createdAt.After(until) { continue } + order, err2 := toGlobalOrder(maxOrder) if err2 != nil { err = multierr.Append(err, err2) From 9fab37a284175a5184644ddf0bb6abafc120e766 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 5 Dec 2023 15:34:31 +0800 Subject: [PATCH 233/422] use getLaunchDate --- pkg/exchange/max/exchange.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index dc1f29237b..42f1d5268a 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -346,8 +346,11 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s return orders, err } - // there is since limit for closed orders API - sinceLimit := time.Date(2018, time.January, 1, 0, 0, 0, 0, time.Local) + // there is since limit for closed orders API. If the since is before launch date, it will respond error + sinceLimit, err := e.getLaunchDate() + if err != nil { + return orders, err + } if since.Before(sinceLimit) { since = sinceLimit } From 165e788c3d87fd6c535d4aa1f614e0fbb72e610e Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 5 Dec 2023 16:59:26 +0800 Subject: [PATCH 234/422] fix --- pkg/exchange/max/exchange.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 42f1d5268a..9813a264c8 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -274,7 +274,7 @@ func (e *Exchange) QueryClosedOrders( ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, ) ([]types.Order, error) { if !since.IsZero() || !until.IsZero() { - return e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByAsc, 1000) + return e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByAsc) } return e.queryClosedOrdersByLastOrderID(ctx, symbol, lastOrderID) @@ -283,13 +283,13 @@ func (e *Exchange) QueryClosedOrders( func (e *Exchange) QueryClosedOrdersDesc( ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, ) ([]types.Order, error) { - closedOrders, err := e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByDesc, 1000) + closedOrders, err := e.queryClosedOrdersByTime(ctx, symbol, since, until, maxapi.OrderByDesc) if lastOrderID == 0 { return closedOrders, err } var filterClosedOrders []types.Order - for _, closedOrder := range filterClosedOrders { + for _, closedOrder := range closedOrders { if closedOrder.OrderID > lastOrderID { filterClosedOrders = append(filterClosedOrders, closedOrder) } @@ -341,7 +341,7 @@ func (e *Exchange) queryClosedOrdersByLastOrderID( return types.SortOrdersAscending(orders), nil } -func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, since, until time.Time, orderByType maxapi.OrderByType, limit uint) (orders []types.Order, err error) { +func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, since, until time.Time, orderByType maxapi.OrderByType) (orders []types.Order, err error) { if err := e.closedOrderQueryLimiter.Wait(ctx); err != nil { return orders, err } @@ -367,7 +367,7 @@ func (e *Exchange) queryClosedOrdersByTime(ctx context.Context, symbol string, s req := e.v3client.NewGetWalletClosedOrdersRequest(walletType). Market(market). - Limit(limit). + Limit(1000). OrderBy(orderByType) switch orderByType { From a1d98e25c6edc14471a02d1bfe0fe77a5bdf3345 Mon Sep 17 00:00:00 2001 From: chiahung Date: Tue, 21 Nov 2023 18:43:27 +0800 Subject: [PATCH 235/422] FEATURE: use max v3 new open orders api --- pkg/exchange/max/exchange.go | 36 ++++++++++---- .../v3/get_wallet_open_orders_request.go | 15 ++++-- ...t_wallet_open_orders_request_requestgen.go | 47 ++++++++++++++++++- 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 9813a264c8..bf69174549 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -253,18 +253,36 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ walletType = maxapi.WalletTypeMargin } - maxOrders, err := e.v3client.NewGetWalletOpenOrdersRequest(walletType).Market(market).Do(ctx) - if err != nil { - return orders, err - } - - for _, maxOrder := range maxOrders { - order, err := toGlobalOrder(maxOrder) + // timestamp can't be negative, so we need to use time which epochtime is > 0 + var since time.Time = time.Date(2018, time.January, 1, 0, 0, 0, 0, time.Local) + var limit uint = 1000 + for { + req := e.v3client.NewGetWalletOpenOrdersRequest(walletType).Market(market).Timestamp(since).OrderBy(maxapi.OrderByAsc).Limit(limit) + maxOrders, err := req.Do(ctx) if err != nil { - return orders, err + return nil, err } - orders = append(orders, *order) + for _, maxOrder := range maxOrders { + createdAt := maxOrder.CreatedAt.Time() + if createdAt.After(since) { + since = createdAt + } + + order, err := toGlobalOrder(maxOrder) + if err != nil { + return orders, err + } + + orders = append(orders, *order) + } + + if len(maxOrders) < int(limit) { + break + } + + // open orders api will get the open orders which created_at >= since, so we need to add 1 ms to avoid duplicated + since = since.Add(1 * time.Millisecond) } return orders, err diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request.go b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request.go index b909f04917..15afdb002a 100644 --- a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request.go +++ b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request.go @@ -1,6 +1,10 @@ package v3 -import "github.com/c9s/requestgen" +import ( + "time" + + "github.com/c9s/requestgen" +) //go:generate -command GetRequest requestgen -method GET //go:generate -command PostRequest requestgen -method POST @@ -10,10 +14,13 @@ func (s *Client) NewGetWalletOpenOrdersRequest(walletType WalletType) *GetWallet return &GetWalletOpenOrdersRequest{client: s.Client, walletType: walletType} } -//go:generate GetRequest -url "/api/v3/wallet/:walletType/orders/open" -type GetWalletOpenOrdersRequest -responseType []Order +//go:generate GetRequest -url "/api/v3/wallet/:walletType/orders/new/open" -type GetWalletOpenOrdersRequest -responseType []Order type GetWalletOpenOrdersRequest struct { client requestgen.AuthenticatedAPIClient - walletType WalletType `param:"walletType,slug,required"` - market string `param:"market,required"` + walletType WalletType `param:"walletType,slug,required"` + market string `param:"market,required"` + timestamp *time.Time `param:"timestamp,milliseconds,omitempty"` + orderBy *OrderByType `param:"order_by,omitempty"` + limit *uint `param:"limit,omitempty"` } diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go index 8121085b25..51416ccda7 100644 --- a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go +++ b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go @@ -1,4 +1,4 @@ -// Code generated by "requestgen -method GET -url /api/v3/wallet/:walletType/orders/open -type GetWalletOpenOrdersRequest -responseType []Order"; DO NOT EDIT. +// Code generated by "requestgen -method GET -url /api/v3/wallet/:walletType/orders/new/open -type GetWalletOpenOrdersRequest -responseType []Order"; DO NOT EDIT. package v3 @@ -10,6 +10,8 @@ import ( "net/url" "reflect" "regexp" + "strconv" + "time" ) func (g *GetWalletOpenOrdersRequest) Market(market string) *GetWalletOpenOrdersRequest { @@ -17,6 +19,21 @@ func (g *GetWalletOpenOrdersRequest) Market(market string) *GetWalletOpenOrdersR return g } +func (g *GetWalletOpenOrdersRequest) Timestamp(timestamp time.Time) *GetWalletOpenOrdersRequest { + g.timestamp = ×tamp + return g +} + +func (g *GetWalletOpenOrdersRequest) OrderBy(orderBy max.OrderByType) *GetWalletOpenOrdersRequest { + g.orderBy = &orderBy + return g +} + +func (g *GetWalletOpenOrdersRequest) Limit(limit uint) *GetWalletOpenOrdersRequest { + g.limit = &limit + return g +} + func (g *GetWalletOpenOrdersRequest) WalletType(walletType max.WalletType) *GetWalletOpenOrdersRequest { g.walletType = walletType return g @@ -48,6 +65,32 @@ func (g *GetWalletOpenOrdersRequest) GetParameters() (map[string]interface{}, er // assign parameter of market params["market"] = market + // check timestamp field -> json key timestamp + if g.timestamp != nil { + timestamp := *g.timestamp + + // assign parameter of timestamp + // convert time.Time to milliseconds time stamp + params["timestamp"] = strconv.FormatInt(timestamp.UnixNano()/int64(time.Millisecond), 10) + fmt.Println(params["timestamp"], timestamp) + } else { + } + // check orderBy field -> json key order_by + if g.orderBy != nil { + orderBy := *g.orderBy + + // assign parameter of orderBy + params["order_by"] = orderBy + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } return params, nil } @@ -151,7 +194,7 @@ func (g *GetWalletOpenOrdersRequest) Do(ctx context.Context) ([]max.Order, error return nil, err } - apiURL := "/api/v3/wallet/:walletType/orders/open" + apiURL := "/api/v3/wallet/:walletType/orders/new/open" slugs, err := g.GetSlugsMap() if err != nil { return nil, err From d54b7365dd653d19d573c40196ed70288f71eb4d Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 5 Dec 2023 20:10:37 +0800 Subject: [PATCH 236/422] FEATURE: use types.OrderMap to avoid missing and duplicated orders --- pkg/exchange/max/exchange.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index bf69174549..9e661e7545 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -246,7 +246,7 @@ func (e *Exchange) QueryOrder(ctx context.Context, q types.OrderQuery) (*types.O return toGlobalOrder(*maxOrder) } -func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { +func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) ([]types.Order, error) { market := toLocalSymbol(symbol) walletType := maxapi.WalletTypeSpot if e.MarginSettings.IsMargin { @@ -254,7 +254,15 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ } // timestamp can't be negative, so we need to use time which epochtime is > 0 - var since time.Time = time.Date(2018, time.January, 1, 0, 0, 0, 0, time.Local) + since, err := e.getLaunchDate() + if err != nil { + return nil, err + } + + // use types.OrderMap because the timestamp params is inclusive. We will get the duplicated order if we use the last order as new since. + // If we use since = since + 1ms, we may miss some orders with the same created_at. + // As a result, we use OrderMap to avoid duplicated or missing order. + var orderMap types.OrderMap = make(types.OrderMap) var limit uint = 1000 for { req := e.v3client.NewGetWalletOpenOrdersRequest(walletType).Market(market).Timestamp(since).OrderBy(maxapi.OrderByAsc).Limit(limit) @@ -263,6 +271,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ return nil, err } + noDuplicatedCnt := 0 for _, maxOrder := range maxOrders { createdAt := maxOrder.CreatedAt.Time() if createdAt.After(since) { @@ -271,21 +280,25 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ order, err := toGlobalOrder(maxOrder) if err != nil { - return orders, err + return nil, err } - orders = append(orders, *order) + if _, exist := orderMap[order.OrderID]; !exist { + orderMap[order.OrderID] = *order + noDuplicatedCnt++ + } } if len(maxOrders) < int(limit) { break } - // open orders api will get the open orders which created_at >= since, so we need to add 1 ms to avoid duplicated - since = since.Add(1 * time.Millisecond) + if noDuplicatedCnt == 0 { + break + } } - return orders, err + return orderMap.Orders(), err } func (e *Exchange) QueryClosedOrders( From c906d6a74d2977a4970b8ddeaab29f79e1020a03 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 6 Dec 2023 11:27:06 +0800 Subject: [PATCH 237/422] rename variable --- pkg/exchange/max/exchange.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 9e661e7545..98a6bbbdaf 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -271,7 +271,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) ([]types. return nil, err } - noDuplicatedCnt := 0 + numUniqueOrders := 0 for _, maxOrder := range maxOrders { createdAt := maxOrder.CreatedAt.Time() if createdAt.After(since) { @@ -285,7 +285,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) ([]types. if _, exist := orderMap[order.OrderID]; !exist { orderMap[order.OrderID] = *order - noDuplicatedCnt++ + numUniqueOrders++ } } @@ -293,7 +293,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) ([]types. break } - if noDuplicatedCnt == 0 { + if numUniqueOrders == 0 { break } } From 445f0f1c4cfcce08db1f64d76602892faf545c70 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Fri, 24 Nov 2023 14:46:26 +0800 Subject: [PATCH 238/422] FEATURE: prepare open maker orders function --- pkg/strategy/dca2/maker.go | 107 ++++++++++++++++++++++++++++++++ pkg/strategy/dca2/maker_test.go | 102 ++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 pkg/strategy/dca2/maker.go create mode 100644 pkg/strategy/dca2/maker_test.go diff --git a/pkg/strategy/dca2/maker.go b/pkg/strategy/dca2/maker.go new file mode 100644 index 0000000000..92ad261b0f --- /dev/null +++ b/pkg/strategy/dca2/maker.go @@ -0,0 +1,107 @@ +package dca2 + +import ( + "context" + "fmt" + "time" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) openMakerOrders(ctx context.Context) error { + s.logger.Infof("[DCA] open maker orders") + price, err := s.retryGetBestPrice(ctx, s.Short) + if err != nil { + return err + } + + orders, err := s.generateMakerOrder(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) + if err != nil { + return err + } + + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orders...) + if err != nil { + return err + } + + s.debugOrders(createdOrders) + + return nil +} + +func (s *Strategy) retryGetBestPrice(ctx context.Context, short bool) (fixedpoint.Value, error) { + var err error + var ticker *types.Ticker + for try := 1; try <= 100; try++ { + ticker, err = s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if err == nil && ticker != nil { + s.logger.Infof("ticker: %s", ticker.String()) + if short { + return ticker.Buy, nil + } + return ticker.Sell, nil + } + + time.Sleep(1 * time.Second) + } + + return fixedpoint.Zero, err +} + +func (s *Strategy) generateMakerOrder(short bool, budget, price, margin fixedpoint.Value, orderNum int64) ([]types.SubmitOrder, error) { + marginPrice := price.Mul(margin) + if !short { + marginPrice = marginPrice.Neg() + } + + var prices []fixedpoint.Value + var total fixedpoint.Value + for i := 0; i < int(orderNum); i++ { + price = price.Add(marginPrice) + truncatePrice := s.Market.TruncatePrice(price) + + // need to avoid the price is below 0 + if truncatePrice.Compare(fixedpoint.Zero) <= 0 { + break + } + + prices = append(prices, truncatePrice) + total = total.Add(truncatePrice) + } + + quantity := fixedpoint.Zero + l := len(prices) - 1 + for ; l >= 0; l-- { + if total.IsZero() { + return nil, fmt.Errorf("total is zero, please check it") + } + + quantity = budget.Div(total) + quantity = s.Market.TruncateQuantity(quantity) + if prices[l].Mul(quantity).Compare(s.Market.MinNotional) > 0 { + break + } + + total = total.Sub(prices[l]) + } + + var submitOrders []types.SubmitOrder + + for i := 0; i <= l; i++ { + submitOrders = append(submitOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Market: s.Market, + Type: types.OrderTypeLimit, + Price: prices[i], + Side: s.makerSide, + TimeInForce: types.TimeInForceGTC, + Quantity: quantity, + Tag: orderTag, + GroupID: s.OrderGroupID, + }) + } + + return submitOrders, nil +} diff --git a/pkg/strategy/dca2/maker_test.go b/pkg/strategy/dca2/maker_test.go new file mode 100644 index 0000000000..68e17fa8f5 --- /dev/null +++ b/pkg/strategy/dca2/maker_test.go @@ -0,0 +1,102 @@ +package dca2 + +import ( + "testing" + + "github.com/sirupsen/logrus" + + . "github.com/c9s/bbgo/pkg/testing/testhelper" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func newTestMarket() types.Market { + return types.Market{ + BaseCurrency: "BTC", + QuoteCurrency: "USDT", + TickSize: Number(0.01), + StepSize: Number(0.000001), + PricePrecision: 2, + VolumePrecision: 8, + MinNotional: Number(8.0), + MinQuantity: Number(0.0003), + } +} + +func newTestStrategy(va ...string) *Strategy { + symbol := "BTCUSDT" + + if len(va) > 0 { + symbol = va[0] + } + + market := newTestMarket() + s := &Strategy{ + logger: logrus.NewEntry(logrus.New()), + Symbol: symbol, + Market: market, + } + return s +} + +func TestGenerateMakerOrder(t *testing.T) { + assert := assert.New(t) + + strategy := newTestStrategy() + + t.Run("case 1: all config is valid and we can place enough orders", func(t *testing.T) { + budget := Number("105000") + askPrice := Number("30000") + margin := Number("0.05") + submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 4) + if !assert.NoError(err) { + return + } + + assert.Len(submitOrders, 4) + assert.Equal(submitOrders[0].Price, Number("28500")) + assert.Equal(submitOrders[1].Price, Number("27000")) + assert.Equal(submitOrders[2].Price, Number("25500")) + assert.Equal(submitOrders[3].Price, Number("24000")) + assert.Equal(submitOrders[0].Quantity, Number("1")) + assert.Equal(submitOrders[1].Quantity, Number("1")) + assert.Equal(submitOrders[2].Quantity, Number("1")) + assert.Equal(submitOrders[3].Quantity, Number("1")) + }) + + t.Run("case 2: some orders' price will below 0 and we should not create such order", func(t *testing.T) { + budget := Number("100000") + askPrice := Number("30000") + margin := Number("0.2") + submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 5) + if !assert.NoError(err) { + return + } + + assert.Len(submitOrders, 4) + assert.Equal(submitOrders[0].Price, Number("24000")) + assert.Equal(submitOrders[1].Price, Number("18000")) + assert.Equal(submitOrders[2].Price, Number("12000")) + assert.Equal(submitOrders[3].Price, Number("6000")) + assert.Equal(submitOrders[0].Quantity, Number("1.666666")) + assert.Equal(submitOrders[1].Quantity, Number("1.666666")) + assert.Equal(submitOrders[2].Quantity, Number("1.666666")) + assert.Equal(submitOrders[3].Quantity, Number("1.666666")) + }) + + t.Run("case 3: some orders' notional is too small and we should not create such order", func(t *testing.T) { + budget := Number("30") + askPrice := Number("30000") + margin := Number("0.2") + submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 5) + if !assert.NoError(err) { + return + } + + assert.Len(submitOrders, 2) + assert.Equal(submitOrders[0].Price, Number("24000")) + assert.Equal(submitOrders[1].Price, Number("18000")) + assert.Equal(submitOrders[0].Quantity, Number("0.000714")) + assert.Equal(submitOrders[1].Quantity, Number("0.000714")) + }) +} From 60003fc4729b7340f8a86a4ce5340b424a4ebe5f Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Thu, 30 Nov 2023 16:36:21 +0800 Subject: [PATCH 239/422] rename somme part --- pkg/strategy/dca2/maker.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/strategy/dca2/maker.go b/pkg/strategy/dca2/maker.go index 92ad261b0f..fa92917283 100644 --- a/pkg/strategy/dca2/maker.go +++ b/pkg/strategy/dca2/maker.go @@ -9,9 +9,9 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func (s *Strategy) openMakerOrders(ctx context.Context) error { - s.logger.Infof("[DCA] open maker orders") - price, err := s.retryGetBestPrice(ctx, s.Short) +func (s *Strategy) placeMakerOrders(ctx context.Context) error { + s.logger.Infof("[DCA] start placing maker orders") + price, err := s.getBestPriceUntilSuccess(ctx, s.Short) if err != nil { return err } @@ -31,7 +31,7 @@ func (s *Strategy) openMakerOrders(ctx context.Context) error { return nil } -func (s *Strategy) retryGetBestPrice(ctx context.Context, short bool) (fixedpoint.Value, error) { +func (s *Strategy) getBestPriceUntilSuccess(ctx context.Context, short bool) (fixedpoint.Value, error) { var err error var ticker *types.Ticker for try := 1; try <= 100; try++ { @@ -56,6 +56,7 @@ func (s *Strategy) generateMakerOrder(short bool, budget, price, margin fixedpoi marginPrice = marginPrice.Neg() } + // TODO: not implement short part yet var prices []fixedpoint.Value var total fixedpoint.Value for i := 0; i < int(orderNum); i++ { From 4aa6ea3a467d032961062b1e5e973f0b2b2f84bb Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 6 Dec 2023 11:22:56 +0800 Subject: [PATCH 240/422] FEATURE: use notional based to crease dca maker orders --- pkg/strategy/dca2/maker.go | 76 +++++++++++++++++++-------------- pkg/strategy/dca2/maker_test.go | 55 ++++++------------------ 2 files changed, 58 insertions(+), 73 deletions(-) diff --git a/pkg/strategy/dca2/maker.go b/pkg/strategy/dca2/maker.go index fa92917283..f138b11f43 100644 --- a/pkg/strategy/dca2/maker.go +++ b/pkg/strategy/dca2/maker.go @@ -9,14 +9,14 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func (s *Strategy) placeMakerOrders(ctx context.Context) error { +func (s *Strategy) placeDCAOrders(ctx context.Context) error { s.logger.Infof("[DCA] start placing maker orders") price, err := s.getBestPriceUntilSuccess(ctx, s.Short) if err != nil { return err } - orders, err := s.generateMakerOrder(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) + orders, err := s.generateDCAOrders(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) if err != nil { return err } @@ -50,47 +50,35 @@ func (s *Strategy) getBestPriceUntilSuccess(ctx context.Context, short bool) (fi return fixedpoint.Zero, err } -func (s *Strategy) generateMakerOrder(short bool, budget, price, margin fixedpoint.Value, orderNum int64) ([]types.SubmitOrder, error) { - marginPrice := price.Mul(margin) - if !short { - marginPrice = marginPrice.Neg() +func (s *Strategy) generateDCAOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { + // TODO: not implement short part yet + factor := fixedpoint.One.Sub(priceDeviation) + if short { + factor = fixedpoint.One.Add(priceDeviation) } - // TODO: not implement short part yet + // calculate all valid prices var prices []fixedpoint.Value - var total fixedpoint.Value - for i := 0; i < int(orderNum); i++ { - price = price.Add(marginPrice) - truncatePrice := s.Market.TruncatePrice(price) - - // need to avoid the price is below 0 - if truncatePrice.Compare(fixedpoint.Zero) <= 0 { + for i := 0; i < int(maxOrderNum); i++ { + if i > 0 { + price = price.Mul(factor) + } + price = s.Market.TruncatePrice(price) + if price.Compare(s.Market.MinPrice) < 0 { break } - prices = append(prices, truncatePrice) - total = total.Add(truncatePrice) + prices = append(prices, price) } - quantity := fixedpoint.Zero - l := len(prices) - 1 - for ; l >= 0; l-- { - if total.IsZero() { - return nil, fmt.Errorf("total is zero, please check it") - } - - quantity = budget.Div(total) - quantity = s.Market.TruncateQuantity(quantity) - if prices[l].Mul(quantity).Compare(s.Market.MinNotional) > 0 { - break - } - - total = total.Sub(prices[l]) + notional, orderNum := calculateDCAMakerOrderNotionalAndNum(s.Market, short, budget, prices) + if orderNum == 0 { + return nil, fmt.Errorf("failed to calculate DCA maker order notional and num, price: %s, budget: %s", price, budget) } var submitOrders []types.SubmitOrder - - for i := 0; i <= l; i++ { + for i := 0; i < orderNum; i++ { + quantity := s.Market.TruncateQuantity(notional.Div(prices[i])) submitOrders = append(submitOrders, types.SubmitOrder{ Symbol: s.Symbol, Market: s.Market, @@ -106,3 +94,27 @@ func (s *Strategy) generateMakerOrder(short bool, budget, price, margin fixedpoi return submitOrders, nil } + +// calculateDCAMakerNotionalAndNum will calculate the notional and num of DCA orders +// DCA2 is notional-based, every order has the same notional +func calculateDCAMakerOrderNotionalAndNum(market types.Market, short bool, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { + for num := len(prices); num > 0; num-- { + notional := budget.Div(fixedpoint.NewFromInt(int64(num))) + if notional.Compare(market.MinNotional) < 0 { + continue + } + + maxPriceIdx := 0 + if short { + maxPriceIdx = num - 1 + } + quantity := market.TruncateQuantity(notional.Div(prices[maxPriceIdx])) + if quantity.Compare(market.MinQuantity) < 0 { + continue + } + + return notional, num + } + + return fixedpoint.Zero, 0 +} diff --git a/pkg/strategy/dca2/maker_test.go b/pkg/strategy/dca2/maker_test.go index 68e17fa8f5..97e8bc1fa1 100644 --- a/pkg/strategy/dca2/maker_test.go +++ b/pkg/strategy/dca2/maker_test.go @@ -45,58 +45,31 @@ func TestGenerateMakerOrder(t *testing.T) { strategy := newTestStrategy() t.Run("case 1: all config is valid and we can place enough orders", func(t *testing.T) { - budget := Number("105000") + budget := Number("10500") askPrice := Number("30000") margin := Number("0.05") - submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 4) + submitOrders, err := strategy.generateDCAOrders(false, budget, askPrice, margin, 4) if !assert.NoError(err) { return } assert.Len(submitOrders, 4) - assert.Equal(submitOrders[0].Price, Number("28500")) - assert.Equal(submitOrders[1].Price, Number("27000")) - assert.Equal(submitOrders[2].Price, Number("25500")) - assert.Equal(submitOrders[3].Price, Number("24000")) - assert.Equal(submitOrders[0].Quantity, Number("1")) - assert.Equal(submitOrders[1].Quantity, Number("1")) - assert.Equal(submitOrders[2].Quantity, Number("1")) - assert.Equal(submitOrders[3].Quantity, Number("1")) + assert.Equal(Number("30000"), submitOrders[0].Price) + assert.Equal(Number("0.0875"), submitOrders[0].Quantity) + assert.Equal(Number("28500"), submitOrders[1].Price) + assert.Equal(Number("0.092105"), submitOrders[1].Quantity) + assert.Equal(Number("27075"), submitOrders[2].Price) + assert.Equal(Number("0.096952"), submitOrders[2].Quantity) + assert.Equal(Number("25721.25"), submitOrders[3].Price) + assert.Equal(Number("0.102055"), submitOrders[3].Quantity) }) - t.Run("case 2: some orders' price will below 0 and we should not create such order", func(t *testing.T) { - budget := Number("100000") - askPrice := Number("30000") - margin := Number("0.2") - submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 5) - if !assert.NoError(err) { - return - } - - assert.Len(submitOrders, 4) - assert.Equal(submitOrders[0].Price, Number("24000")) - assert.Equal(submitOrders[1].Price, Number("18000")) - assert.Equal(submitOrders[2].Price, Number("12000")) - assert.Equal(submitOrders[3].Price, Number("6000")) - assert.Equal(submitOrders[0].Quantity, Number("1.666666")) - assert.Equal(submitOrders[1].Quantity, Number("1.666666")) - assert.Equal(submitOrders[2].Quantity, Number("1.666666")) - assert.Equal(submitOrders[3].Quantity, Number("1.666666")) + t.Run("case 2: some orders' price will below 0, so we should not create such order", func(t *testing.T) { }) - t.Run("case 3: some orders' notional is too small and we should not create such order", func(t *testing.T) { - budget := Number("30") - askPrice := Number("30000") - margin := Number("0.2") - submitOrders, err := strategy.generateMakerOrder(false, budget, askPrice, margin, 5) - if !assert.NoError(err) { - return - } + t.Run("case 3: notional is too small, so we should decrease num of orders", func(t *testing.T) { + }) - assert.Len(submitOrders, 2) - assert.Equal(submitOrders[0].Price, Number("24000")) - assert.Equal(submitOrders[1].Price, Number("18000")) - assert.Equal(submitOrders[0].Quantity, Number("0.000714")) - assert.Equal(submitOrders[1].Quantity, Number("0.000714")) + t.Run("case 4: quantity is too small, so we should decrease num of orders", func(t *testing.T) { }) } From c67737a6d60c96782336bbdf5f4bb1f7f61349c7 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 6 Dec 2023 16:16:17 +0800 Subject: [PATCH 241/422] use retry package --- pkg/exchange/retry/ticker.go | 17 +++++++++++++++++ pkg/strategy/dca2/maker.go | 28 +++++++++++----------------- 2 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 pkg/exchange/retry/ticker.go diff --git a/pkg/exchange/retry/ticker.go b/pkg/exchange/retry/ticker.go new file mode 100644 index 0000000000..9fc1a4391d --- /dev/null +++ b/pkg/exchange/retry/ticker.go @@ -0,0 +1,17 @@ +package retry + +import ( + "context" + + "github.com/c9s/bbgo/pkg/types" +) + +func QueryTickerUntilSuccessful(ctx context.Context, ex types.Exchange, symbol string) (ticker *types.Ticker, err error) { + var op = func() (err2 error) { + ticker, err2 = ex.QueryTicker(ctx, symbol) + return err2 + } + + err = GeneralBackoff(ctx, op) + return ticker, err +} diff --git a/pkg/strategy/dca2/maker.go b/pkg/strategy/dca2/maker.go index f138b11f43..ec656a6795 100644 --- a/pkg/strategy/dca2/maker.go +++ b/pkg/strategy/dca2/maker.go @@ -3,15 +3,15 @@ package dca2 import ( "context" "fmt" - "time" + "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) func (s *Strategy) placeDCAOrders(ctx context.Context) error { s.logger.Infof("[DCA] start placing maker orders") - price, err := s.getBestPriceUntilSuccess(ctx, s.Short) + price, err := getBestPriceUntilSuccess(ctx, s.Session.Exchange, s.Symbol, s.Short) if err != nil { return err } @@ -31,23 +31,17 @@ func (s *Strategy) placeDCAOrders(ctx context.Context) error { return nil } -func (s *Strategy) getBestPriceUntilSuccess(ctx context.Context, short bool) (fixedpoint.Value, error) { - var err error - var ticker *types.Ticker - for try := 1; try <= 100; try++ { - ticker, err = s.Session.Exchange.QueryTicker(ctx, s.Symbol) - if err == nil && ticker != nil { - s.logger.Infof("ticker: %s", ticker.String()) - if short { - return ticker.Buy, nil - } - return ticker.Sell, nil - } - - time.Sleep(1 * time.Second) +func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol string, short bool) (fixedpoint.Value, error) { + ticker, err := retry.QueryTickerUntilSuccessful(ctx, ex, symbol) + if err != nil { + return fixedpoint.Zero, err } - return fixedpoint.Zero, err + if short { + return ticker.Buy, nil + } else { + return ticker.Sell, nil + } } func (s *Strategy) generateDCAOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { From 05d446cb545f1b2397d799b3e0aab0aba87f5e3c Mon Sep 17 00:00:00 2001 From: dydysy <200615@gmail.com> Date: Wed, 6 Dec 2023 18:42:10 +0800 Subject: [PATCH 242/422] FIX: [indicator] Possibly incorrect assignment --- pkg/types/indicator.go | 20 ++++++++++---------- pkg/types/indicator_test.go | 4 ++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/types/indicator.go b/pkg/types/indicator.go index 593f6bec82..839e9b3fc9 100644 --- a/pkg/types/indicator.go +++ b/pkg/types/indicator.go @@ -327,29 +327,29 @@ func Dot(a interface{}, b interface{}, limit ...int) float64 { aas = tp isaf = false default: - panic("input should be either Series or float64") + panic("input should be either *Series or numbers") } switch tp := b.(type) { case float64: bbf = tp isbf = true case int32: - aaf = float64(tp) - isaf = true + bbf = float64(tp) + isbf = true case int64: - aaf = float64(tp) - isaf = true + bbf = float64(tp) + isbf = true case float32: - aaf = float64(tp) - isaf = true + bbf = float64(tp) + isbf = true case int: - aaf = float64(tp) - isaf = true + bbf = float64(tp) + isbf = true case Series: bbs = tp isbf = false default: - panic("input should be either Series or float64") + panic("input should be either *Series or numbers") } l := 1 diff --git a/pkg/types/indicator_test.go b/pkg/types/indicator_test.go index da328c6661..d8c158f2a8 100644 --- a/pkg/types/indicator_test.go +++ b/pkg/types/indicator_test.go @@ -215,6 +215,10 @@ func TestDot(t *testing.T) { assert.InDelta(t, out2, 3., 0.001) out3 := Dot(3., &a, 2) assert.InDelta(t, out2, out3, 0.001) + out4 := Dot(&a, 3, 2) + assert.InDelta(t, out2, 3., 0.001) + out5 := Dot(3, &a, 2) + assert.InDelta(t, out4, out5, 0.001) } func TestClone(t *testing.T) { From 2982be1cbc1740d97094f4d6fac46be22314728a Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Thu, 7 Dec 2023 11:27:28 +0800 Subject: [PATCH 243/422] rename dca maker orders to open position orders --- pkg/strategy/dca2/{maker.go => openPosition.go} | 14 +++++++------- .../dca2/{maker_test.go => openPosition_test.go} | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename pkg/strategy/dca2/{maker.go => openPosition.go} (77%) rename pkg/strategy/dca2/{maker_test.go => openPosition_test.go} (95%) diff --git a/pkg/strategy/dca2/maker.go b/pkg/strategy/dca2/openPosition.go similarity index 77% rename from pkg/strategy/dca2/maker.go rename to pkg/strategy/dca2/openPosition.go index ec656a6795..10a09b4e1f 100644 --- a/pkg/strategy/dca2/maker.go +++ b/pkg/strategy/dca2/openPosition.go @@ -9,14 +9,14 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func (s *Strategy) placeDCAOrders(ctx context.Context) error { - s.logger.Infof("[DCA] start placing maker orders") +func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { + s.logger.Infof("[DCA] start placing open position orders") price, err := getBestPriceUntilSuccess(ctx, s.Session.Exchange, s.Symbol, s.Short) if err != nil { return err } - orders, err := s.generateDCAOrders(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) + orders, err := s.generateOpenPositionOrders(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) if err != nil { return err } @@ -44,7 +44,7 @@ func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol str } } -func (s *Strategy) generateDCAOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { +func (s *Strategy) generateOpenPositionOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { // TODO: not implement short part yet factor := fixedpoint.One.Sub(priceDeviation) if short { @@ -65,7 +65,7 @@ func (s *Strategy) generateDCAOrders(short bool, budget, price, priceDeviation f prices = append(prices, price) } - notional, orderNum := calculateDCAMakerOrderNotionalAndNum(s.Market, short, budget, prices) + notional, orderNum := calculateNotionalAndNum(s.Market, short, budget, prices) if orderNum == 0 { return nil, fmt.Errorf("failed to calculate DCA maker order notional and num, price: %s, budget: %s", price, budget) } @@ -89,9 +89,9 @@ func (s *Strategy) generateDCAOrders(short bool, budget, price, priceDeviation f return submitOrders, nil } -// calculateDCAMakerNotionalAndNum will calculate the notional and num of DCA orders +// calculateNotionalAndNum calculates the notional and num of open position orders // DCA2 is notional-based, every order has the same notional -func calculateDCAMakerOrderNotionalAndNum(market types.Market, short bool, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { +func calculateNotionalAndNum(market types.Market, short bool, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { for num := len(prices); num > 0; num-- { notional := budget.Div(fixedpoint.NewFromInt(int64(num))) if notional.Compare(market.MinNotional) < 0 { diff --git a/pkg/strategy/dca2/maker_test.go b/pkg/strategy/dca2/openPosition_test.go similarity index 95% rename from pkg/strategy/dca2/maker_test.go rename to pkg/strategy/dca2/openPosition_test.go index 97e8bc1fa1..4b7cc3cb9f 100644 --- a/pkg/strategy/dca2/maker_test.go +++ b/pkg/strategy/dca2/openPosition_test.go @@ -48,7 +48,7 @@ func TestGenerateMakerOrder(t *testing.T) { budget := Number("10500") askPrice := Number("30000") margin := Number("0.05") - submitOrders, err := strategy.generateDCAOrders(false, budget, askPrice, margin, 4) + submitOrders, err := strategy.generateOpenPositionOrders(false, budget, askPrice, margin, 4) if !assert.NoError(err) { return } From 68577342828a8e0721918d8ef4da5326d4a5895d Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Thu, 7 Dec 2023 11:29:42 +0800 Subject: [PATCH 244/422] rename --- pkg/strategy/dca2/openPosition.go | 3 +-- pkg/strategy/dca2/openPosition_test.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/dca2/openPosition.go b/pkg/strategy/dca2/openPosition.go index 10a09b4e1f..f79a0c5aee 100644 --- a/pkg/strategy/dca2/openPosition.go +++ b/pkg/strategy/dca2/openPosition.go @@ -45,7 +45,6 @@ func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol str } func (s *Strategy) generateOpenPositionOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { - // TODO: not implement short part yet factor := fixedpoint.One.Sub(priceDeviation) if short { factor = fixedpoint.One.Add(priceDeviation) @@ -67,7 +66,7 @@ func (s *Strategy) generateOpenPositionOrders(short bool, budget, price, priceDe notional, orderNum := calculateNotionalAndNum(s.Market, short, budget, prices) if orderNum == 0 { - return nil, fmt.Errorf("failed to calculate DCA maker order notional and num, price: %s, budget: %s", price, budget) + return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, budget: %s", price, budget) } var submitOrders []types.SubmitOrder diff --git a/pkg/strategy/dca2/openPosition_test.go b/pkg/strategy/dca2/openPosition_test.go index 4b7cc3cb9f..ede4c9f31b 100644 --- a/pkg/strategy/dca2/openPosition_test.go +++ b/pkg/strategy/dca2/openPosition_test.go @@ -39,7 +39,7 @@ func newTestStrategy(va ...string) *Strategy { return s } -func TestGenerateMakerOrder(t *testing.T) { +func TestGenerateOpenPositionOrders(t *testing.T) { assert := assert.New(t) strategy := newTestStrategy() From 53bf443b1d83ffdaa9e738afc934d5b3d8bc2d14 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 27 Nov 2023 17:45:12 +0800 Subject: [PATCH 245/422] xdepthmaker: first commit --- pkg/strategy/xdepthmaker/aggregate.go | 32 + pkg/strategy/xdepthmaker/state.go | 68 +++ pkg/strategy/xdepthmaker/strategy.go | 811 ++++++++++++++++++++++++++ 3 files changed, 911 insertions(+) create mode 100644 pkg/strategy/xdepthmaker/aggregate.go create mode 100644 pkg/strategy/xdepthmaker/state.go create mode 100644 pkg/strategy/xdepthmaker/strategy.go diff --git a/pkg/strategy/xdepthmaker/aggregate.go b/pkg/strategy/xdepthmaker/aggregate.go new file mode 100644 index 0000000000..a9ff5298f7 --- /dev/null +++ b/pkg/strategy/xdepthmaker/aggregate.go @@ -0,0 +1,32 @@ +package xdepthmaker + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func aggregatePrice(pvs types.PriceVolumeSlice, requiredQuantity fixedpoint.Value) (price fixedpoint.Value) { + q := requiredQuantity + totalAmount := fixedpoint.Zero + + if len(pvs) == 0 { + price = fixedpoint.Zero + return price + } else if pvs[0].Volume.Compare(requiredQuantity) >= 0 { + return pvs[0].Price + } + + for i := 0; i < len(pvs); i++ { + pv := pvs[i] + if pv.Volume.Compare(q) >= 0 { + totalAmount = totalAmount.Add(q.Mul(pv.Price)) + break + } + + q = q.Sub(pv.Volume) + totalAmount = totalAmount.Add(pv.Volume.Mul(pv.Price)) + } + + price = totalAmount.Div(requiredQuantity) + return price +} diff --git a/pkg/strategy/xdepthmaker/state.go b/pkg/strategy/xdepthmaker/state.go new file mode 100644 index 0000000000..cfdfe2ce4b --- /dev/null +++ b/pkg/strategy/xdepthmaker/state.go @@ -0,0 +1,68 @@ +package xdepthmaker + +import ( + "sync" + "time" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type State struct { + CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty"` + + // Deprecated: + Position *types.Position `json:"position,omitempty"` + + // Deprecated: + ProfitStats ProfitStats `json:"profitStats,omitempty"` +} + +type ProfitStats struct { + *types.ProfitStats + + lock sync.Mutex + + MakerExchange types.ExchangeName `json:"makerExchange"` + + AccumulatedMakerVolume fixedpoint.Value `json:"accumulatedMakerVolume,omitempty"` + AccumulatedMakerBidVolume fixedpoint.Value `json:"accumulatedMakerBidVolume,omitempty"` + AccumulatedMakerAskVolume fixedpoint.Value `json:"accumulatedMakerAskVolume,omitempty"` + + TodayMakerVolume fixedpoint.Value `json:"todayMakerVolume,omitempty"` + TodayMakerBidVolume fixedpoint.Value `json:"todayMakerBidVolume,omitempty"` + TodayMakerAskVolume fixedpoint.Value `json:"todayMakerAskVolume,omitempty"` +} + +func (s *ProfitStats) AddTrade(trade types.Trade) { + s.ProfitStats.AddTrade(trade) + + if trade.Exchange == s.MakerExchange { + s.lock.Lock() + s.AccumulatedMakerVolume = s.AccumulatedMakerVolume.Add(trade.Quantity) + s.TodayMakerVolume = s.TodayMakerVolume.Add(trade.Quantity) + + switch trade.Side { + + case types.SideTypeSell: + s.AccumulatedMakerAskVolume = s.AccumulatedMakerAskVolume.Add(trade.Quantity) + s.TodayMakerAskVolume = s.TodayMakerAskVolume.Add(trade.Quantity) + + case types.SideTypeBuy: + s.AccumulatedMakerBidVolume = s.AccumulatedMakerBidVolume.Add(trade.Quantity) + s.TodayMakerBidVolume = s.TodayMakerBidVolume.Add(trade.Quantity) + + } + s.lock.Unlock() + } +} + +func (s *ProfitStats) ResetToday() { + s.ProfitStats.ResetToday(time.Now()) + + s.lock.Lock() + s.TodayMakerVolume = fixedpoint.Zero + s.TodayMakerBidVolume = fixedpoint.Zero + s.TodayMakerAskVolume = fixedpoint.Zero + s.lock.Unlock() +} diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go new file mode 100644 index 0000000000..5d85be1c3d --- /dev/null +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -0,0 +1,811 @@ +package xdepthmaker + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/time/rate" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" +) + +var lastPriceModifier = fixedpoint.NewFromFloat(1.001) +var minGap = fixedpoint.NewFromFloat(1.02) +var defaultMargin = fixedpoint.NewFromFloat(0.003) + +var Two = fixedpoint.NewFromInt(2) + +const priceUpdateTimeout = 30 * time.Second + +const ID = "xdepthmaker" + +var log = logrus.WithField("strategy", ID) + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + Environment *bbgo.Environment + + Symbol string `json:"symbol"` + + // SourceExchange session name + SourceExchange string `json:"sourceExchange"` + + // MakerExchange session name + MakerExchange string `json:"makerExchange"` + + UpdateInterval types.Duration `json:"updateInterval"` + HedgeInterval types.Duration `json:"hedgeInterval"` + OrderCancelWaitTime types.Duration `json:"orderCancelWaitTime"` + + Margin fixedpoint.Value `json:"margin"` + BidMargin fixedpoint.Value `json:"bidMargin"` + AskMargin fixedpoint.Value `json:"askMargin"` + UseDepthPrice bool `json:"useDepthPrice"` + DepthQuantity fixedpoint.Value `json:"depthQuantity"` + + EnableBollBandMargin bool `json:"enableBollBandMargin"` + + StopHedgeQuoteBalance fixedpoint.Value `json:"stopHedgeQuoteBalance"` + StopHedgeBaseBalance fixedpoint.Value `json:"stopHedgeBaseBalance"` + + // Quantity is used for fixed quantity of the first layer + Quantity fixedpoint.Value `json:"quantity"` + + // QuantityMultiplier is the factor that multiplies the quantity of the previous layer + QuantityMultiplier fixedpoint.Value `json:"quantityMultiplier"` + + // QuantityScale helps user to define the quantity by layer scale + QuantityScale *bbgo.LayerScale `json:"quantityScale,omitempty"` + + // MaxExposurePosition defines the unhedged quantity of stop + MaxExposurePosition fixedpoint.Value `json:"maxExposurePosition"` + + DisableHedge bool `json:"disableHedge"` + + NotifyTrade bool `json:"notifyTrade"` + + // RecoverTrade tries to find the missing trades via the REStful API + RecoverTrade bool `json:"recoverTrade"` + + RecoverTradeScanPeriod types.Duration `json:"recoverTradeScanPeriod"` + + NumLayers int `json:"numLayers"` + + // Pips is the pips of the layer prices + Pips fixedpoint.Value `json:"pips"` + + // -------------------------------- + // private field + + makerSession, sourceSession *bbgo.ExchangeSession + + makerMarket, sourceMarket types.Market + + state *State + + // persistence fields + Position *types.Position `json:"position,omitempty" persistence:"position"` + ProfitStats *ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` + CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` + + book *types.StreamOrderBook + activeMakerOrders *bbgo.ActiveOrderBook + + hedgeErrorLimiter *rate.Limiter + hedgeErrorRateReservation *rate.Reservation + + orderStore *core.OrderStore + tradeCollector *core.TradeCollector + + askPriceHeartBeat, bidPriceHeartBeat types.PriceHeartBeat + + lastPrice fixedpoint.Value + groupID uint32 + + stopC chan struct{} +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { + sourceSession, ok := sessions[s.SourceExchange] + if !ok { + panic(fmt.Errorf("source session %s is not defined", s.SourceExchange)) + } + + sourceSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) + sourceSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) + + makerSession, ok := sessions[s.MakerExchange] + if !ok { + panic(fmt.Errorf("maker session %s is not defined", s.MakerExchange)) + } + + makerSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) +} + +func (s *Strategy) Validate() error { + if s.Quantity.IsZero() || s.QuantityScale == nil { + return errors.New("quantity or quantityScale can not be empty") + } + + if !s.QuantityMultiplier.IsZero() && s.QuantityMultiplier.Sign() < 0 { + return errors.New("quantityMultiplier can not be a negative number") + } + + if len(s.Symbol) == 0 { + return errors.New("symbol is required") + } + + return nil +} + +func (s *Strategy) Defaults() error { + if s.UpdateInterval == 0 { + s.UpdateInterval = types.Duration(time.Second) + } + + if s.HedgeInterval == 0 { + s.HedgeInterval = types.Duration(3 * time.Second) + } + + if s.NumLayers == 0 { + s.NumLayers = 1 + } + + if s.Margin.IsZero() { + s.Margin = defaultMargin + } + + if s.BidMargin.IsZero() { + if !s.Margin.IsZero() { + s.BidMargin = s.Margin + } else { + s.BidMargin = defaultMargin + } + } + + if s.AskMargin.IsZero() { + if !s.Margin.IsZero() { + s.AskMargin = s.Margin + } else { + s.AskMargin = defaultMargin + } + } + + s.hedgeErrorLimiter = rate.NewLimiter(rate.Every(1*time.Minute), 1) + return nil +} + +func (s *Strategy) Initialize() error { + return nil +} + +func (s *Strategy) CrossRun( + ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, +) error { + // configure sessions + sourceSession, ok := sessions[s.SourceExchange] + if !ok { + return fmt.Errorf("source exchange session %s is not defined", s.SourceExchange) + } + + s.sourceSession = sourceSession + + makerSession, ok := sessions[s.MakerExchange] + if !ok { + return fmt.Errorf("maker exchange session %s is not defined", s.MakerExchange) + } + + s.makerSession = makerSession + + s.sourceMarket, ok = s.sourceSession.Market(s.Symbol) + if !ok { + return fmt.Errorf("source session market %s is not defined", s.Symbol) + } + + s.makerMarket, ok = s.makerSession.Market(s.Symbol) + if !ok { + return fmt.Errorf("maker session market %s is not defined", s.Symbol) + } + + // restore state + instanceID := s.InstanceID() + s.groupID = util.FNV32(instanceID) + log.Infof("using group id %d from fnv(%s)", s.groupID, instanceID) + + if s.Position == nil { + s.Position = types.NewPositionFromMarket(s.makerMarket) + + // force update for legacy code + s.Position.Market = s.makerMarket + } + + bbgo.Notify("xdepthmaker: %s position is restored", s.Symbol, s.Position) + + if s.ProfitStats == nil { + s.ProfitStats = &ProfitStats{ + ProfitStats: types.NewProfitStats(s.makerMarket), + MakerExchange: s.makerSession.ExchangeName, + } + } + + if s.CoveredPosition.IsZero() { + if s.state != nil && !s.CoveredPosition.IsZero() { + s.CoveredPosition = s.state.CoveredPosition + } + } + + if s.makerSession.MakerFeeRate.Sign() > 0 || s.makerSession.TakerFeeRate.Sign() > 0 { + s.Position.SetExchangeFeeRate(types.ExchangeName(s.MakerExchange), types.ExchangeFee{ + MakerFeeRate: s.makerSession.MakerFeeRate, + TakerFeeRate: s.makerSession.TakerFeeRate, + }) + } + + if s.sourceSession.MakerFeeRate.Sign() > 0 || s.sourceSession.TakerFeeRate.Sign() > 0 { + s.Position.SetExchangeFeeRate(types.ExchangeName(s.SourceExchange), types.ExchangeFee{ + MakerFeeRate: s.sourceSession.MakerFeeRate, + TakerFeeRate: s.sourceSession.TakerFeeRate, + }) + } + + s.book = types.NewStreamBook(s.Symbol) + s.book.BindStream(s.sourceSession.MarketDataStream) + + s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol) + s.activeMakerOrders.BindStream(s.makerSession.UserDataStream) + + s.orderStore = core.NewOrderStore(s.Symbol) + s.orderStore.BindStream(s.sourceSession.UserDataStream) + s.orderStore.BindStream(s.makerSession.UserDataStream) + + s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore) + + if s.NotifyTrade { + s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { + bbgo.Notify(trade) + }) + } + + s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { + c := trade.PositionChange() + if trade.Exchange == s.sourceSession.ExchangeName { + s.CoveredPosition = s.CoveredPosition.Add(c) + } + + s.ProfitStats.AddTrade(trade) + + if profit.Compare(fixedpoint.Zero) == 0 { + s.Environment.RecordPosition(s.Position, trade, nil) + } else { + log.Infof("%s generated profit: %v", s.Symbol, profit) + + p := s.Position.NewProfit(trade, profit, netProfit) + p.Strategy = ID + p.StrategyInstanceID = instanceID + bbgo.Notify(&p) + s.ProfitStats.AddProfit(p) + + s.Environment.RecordPosition(s.Position, trade, &p) + } + }) + + s.tradeCollector.OnPositionUpdate(func(position *types.Position) { + bbgo.Notify(position) + }) + s.tradeCollector.OnRecover(func(trade types.Trade) { + bbgo.Notify("Recovered trade", trade) + }) + s.tradeCollector.BindStream(s.sourceSession.UserDataStream) + s.tradeCollector.BindStream(s.makerSession.UserDataStream) + + s.stopC = make(chan struct{}) + + if s.RecoverTrade { + go s.tradeRecover(ctx) + } + + go func() { + posTicker := time.NewTicker(util.MillisecondsJitter(s.HedgeInterval.Duration(), 200)) + defer posTicker.Stop() + + quoteTicker := time.NewTicker(util.MillisecondsJitter(s.UpdateInterval.Duration(), 200)) + defer quoteTicker.Stop() + + reportTicker := time.NewTicker(time.Hour) + defer reportTicker.Stop() + + defer func() { + if err := s.activeMakerOrders.GracefulCancel(context.Background(), s.makerSession.Exchange); err != nil { + log.WithError(err).Errorf("can not cancel %s orders", s.Symbol) + } + }() + + for { + select { + + case <-s.stopC: + log.Warnf("%s maker goroutine stopped, due to the stop signal", s.Symbol) + return + + case <-ctx.Done(): + log.Warnf("%s maker goroutine stopped, due to the cancelled context", s.Symbol) + return + + case <-quoteTicker.C: + s.updateQuote(ctx, orderExecutionRouter) + + case <-reportTicker.C: + bbgo.Notify(s.ProfitStats) + + case <-posTicker.C: + // For positive position and positive covered position: + // uncover position = +5 - +3 (covered position) = 2 + // + // For positive position and negative covered position: + // uncover position = +5 - (-3) (covered position) = 8 + // + // meaning we bought 5 on MAX and sent buy order with 3 on binance + // + // For negative position: + // uncover position = -5 - -3 (covered position) = -2 + s.tradeCollector.Process() + + position := s.Position.GetBase() + + uncoverPosition := position.Sub(s.CoveredPosition) + absPos := uncoverPosition.Abs() + if !s.DisableHedge && absPos.Compare(s.sourceMarket.MinQuantity) > 0 { + log.Infof("%s base position %v coveredPosition: %v uncoverPosition: %v", + s.Symbol, + position, + s.CoveredPosition, + uncoverPosition, + ) + + s.Hedge(ctx, uncoverPosition.Neg()) + } + } + } + }() + + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + + close(s.stopC) + + // wait for the quoter to stop + time.Sleep(s.UpdateInterval.Duration()) + + shutdownCtx, cancelShutdown := context.WithTimeout(context.TODO(), time.Minute) + defer cancelShutdown() + + if err := s.activeMakerOrders.GracefulCancel(shutdownCtx, s.makerSession.Exchange); err != nil { + log.WithError(err).Errorf("graceful cancel error") + } + + bbgo.Notify("%s: %s position", ID, s.Symbol, s.Position) + }) + + return nil +} + +func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { + side := types.SideTypeBuy + if pos.IsZero() { + return + } + + quantity := pos.Abs() + + if pos.Sign() < 0 { + side = types.SideTypeSell + } + + lastPrice := s.lastPrice + sourceBook := s.book.CopyDepth(1) + switch side { + + case types.SideTypeBuy: + if bestAsk, ok := sourceBook.BestAsk(); ok { + lastPrice = bestAsk.Price + } + + case types.SideTypeSell: + if bestBid, ok := sourceBook.BestBid(); ok { + lastPrice = bestBid.Price + } + } + + notional := quantity.Mul(lastPrice) + if notional.Compare(s.sourceMarket.MinNotional) <= 0 { + log.Warnf("%s %v less than min notional, skipping hedge", s.Symbol, notional) + return + } + + // adjust quantity according to the balances + account := s.sourceSession.GetAccount() + switch side { + + case types.SideTypeBuy: + // check quote quantity + if quote, ok := account.Balance(s.sourceMarket.QuoteCurrency); ok { + if quote.Available.Compare(notional) < 0 { + // adjust price to higher 0.1%, so that we can ensure that the order can be executed + quantity = bbgo.AdjustQuantityByMaxAmount(quantity, lastPrice.Mul(lastPriceModifier), quote.Available) + quantity = s.sourceMarket.TruncateQuantity(quantity) + } + } + + case types.SideTypeSell: + // check quote quantity + if base, ok := account.Balance(s.sourceMarket.BaseCurrency); ok { + if base.Available.Compare(quantity) < 0 { + quantity = base.Available + } + } + } + + // truncate quantity for the supported precision + quantity = s.sourceMarket.TruncateQuantity(quantity) + + if notional.Compare(s.sourceMarket.MinNotional.Mul(minGap)) <= 0 { + log.Warnf("the adjusted amount %v is less than minimal notional %v, skipping hedge", notional, s.sourceMarket.MinNotional) + return + } + + if quantity.Compare(s.sourceMarket.MinQuantity.Mul(minGap)) <= 0 { + log.Warnf("the adjusted quantity %v is less than minimal quantity %v, skipping hedge", quantity, s.sourceMarket.MinQuantity) + return + } + + if s.hedgeErrorRateReservation != nil { + if !s.hedgeErrorRateReservation.OK() { + return + } + bbgo.Notify("Hit hedge error rate limit, waiting...") + time.Sleep(s.hedgeErrorRateReservation.Delay()) + s.hedgeErrorRateReservation = nil + } + + log.Infof("submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) + bbgo.Notify("Submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) + orderExecutor := &bbgo.ExchangeOrderExecutor{Session: s.sourceSession} + returnOrders, err := orderExecutor.SubmitOrders(ctx, types.SubmitOrder{ + Market: s.sourceMarket, + Symbol: s.Symbol, + Type: types.OrderTypeMarket, + Side: side, + Quantity: quantity, + }) + + if err != nil { + s.hedgeErrorRateReservation = s.hedgeErrorLimiter.Reserve() + log.WithError(err).Errorf("market order submit error: %s", err.Error()) + return + } + + // if it's selling, than we should add positive position + if side == types.SideTypeSell { + s.CoveredPosition = s.CoveredPosition.Add(quantity) + } else { + s.CoveredPosition = s.CoveredPosition.Add(quantity.Neg()) + } + + s.orderStore.Add(returnOrders...) +} + +func (s *Strategy) tradeRecover(ctx context.Context) { + tradeScanInterval := s.RecoverTradeScanPeriod.Duration() + if tradeScanInterval == 0 { + tradeScanInterval = 30 * time.Minute + } + + tradeScanOverlapBufferPeriod := 5 * time.Minute + + tradeScanTicker := time.NewTicker(tradeScanInterval) + defer tradeScanTicker.Stop() + + for { + select { + case <-ctx.Done(): + return + + case <-tradeScanTicker.C: + log.Infof("scanning trades from %s ago...", tradeScanInterval) + + if s.RecoverTrade { + startTime := time.Now().Add(-tradeScanInterval).Add(-tradeScanOverlapBufferPeriod) + + if err := s.tradeCollector.Recover(ctx, s.sourceSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { + log.WithError(err).Errorf("query trades error") + } + + if err := s.tradeCollector.Recover(ctx, s.makerSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { + log.WithError(err).Errorf("query trades error") + } + } + } + } +} + +func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter) { + if err := s.activeMakerOrders.GracefulCancel(ctx, s.makerSession.Exchange); err != nil { + log.Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) + s.activeMakerOrders.Print() + return + } + + if s.activeMakerOrders.NumOfOrders() > 0 { + return + } + + bestBid, bestAsk, hasPrice := s.book.BestBidAndAsk() + if !hasPrice { + return + } + + // use mid-price for the last price + s.lastPrice = bestBid.Price.Add(bestAsk.Price).Div(Two) + + bookLastUpdateTime := s.book.LastUpdateTime() + + if _, err := s.bidPriceHeartBeat.Update(bestBid, priceUpdateTimeout); err != nil { + log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", + s.Symbol, + time.Since(bookLastUpdateTime)) + return + } + + if _, err := s.askPriceHeartBeat.Update(bestAsk, priceUpdateTimeout); err != nil { + log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", + s.Symbol, + time.Since(bookLastUpdateTime)) + return + } + + sourceBook := s.book.CopyDepth(10) + if valid, err := sourceBook.IsValid(); !valid { + log.WithError(err).Errorf("%s invalid copied order book, skip quoting: %v", s.Symbol, err) + return + } + + var disableMakerBid = false + var disableMakerAsk = false + + // check maker's balance quota + // we load the balances from the account while we're generating the orders, + // the balance may have a chance to be deducted by other strategies or manual orders submitted by the user + makerBalances := s.makerSession.GetAccount().Balances() + makerQuota := &bbgo.QuotaTransaction{} + if b, ok := makerBalances[s.makerMarket.BaseCurrency]; ok { + if b.Available.Compare(s.makerMarket.MinQuantity) > 0 { + makerQuota.BaseAsset.Add(b.Available) + } else { + disableMakerAsk = true + } + } + + if b, ok := makerBalances[s.makerMarket.QuoteCurrency]; ok { + if b.Available.Compare(s.makerMarket.MinNotional) > 0 { + makerQuota.QuoteAsset.Add(b.Available) + } else { + disableMakerBid = true + } + } + + hedgeBalances := s.sourceSession.GetAccount().Balances() + hedgeQuota := &bbgo.QuotaTransaction{} + if b, ok := hedgeBalances[s.sourceMarket.BaseCurrency]; ok { + // to make bid orders, we need enough base asset in the foreign exchange, + // if the base asset balance is not enough for selling + if s.StopHedgeBaseBalance.Sign() > 0 { + minAvailable := s.StopHedgeBaseBalance.Add(s.sourceMarket.MinQuantity) + if b.Available.Compare(minAvailable) > 0 { + hedgeQuota.BaseAsset.Add(b.Available.Sub(minAvailable)) + } else { + log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) + disableMakerBid = true + } + } else if b.Available.Compare(s.sourceMarket.MinQuantity) > 0 { + hedgeQuota.BaseAsset.Add(b.Available) + } else { + log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) + disableMakerBid = true + } + } + + if b, ok := hedgeBalances[s.sourceMarket.QuoteCurrency]; ok { + // to make ask orders, we need enough quote asset in the foreign exchange, + // if the quote asset balance is not enough for buying + if s.StopHedgeQuoteBalance.Sign() > 0 { + minAvailable := s.StopHedgeQuoteBalance.Add(s.sourceMarket.MinNotional) + if b.Available.Compare(minAvailable) > 0 { + hedgeQuota.QuoteAsset.Add(b.Available.Sub(minAvailable)) + } else { + log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) + disableMakerAsk = true + } + } else if b.Available.Compare(s.sourceMarket.MinNotional) > 0 { + hedgeQuota.QuoteAsset.Add(b.Available) + } else { + log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) + disableMakerAsk = true + } + } + + // if max exposure position is configured, we should not: + // 1. place bid orders when we already bought too much + // 2. place ask orders when we already sold too much + if s.MaxExposurePosition.Sign() > 0 { + pos := s.Position.GetBase() + + if pos.Compare(s.MaxExposurePosition.Neg()) > 0 { + // stop sell if we over-sell + disableMakerAsk = true + } else if pos.Compare(s.MaxExposurePosition) > 0 { + // stop buy if we over buy + disableMakerBid = true + } + } + + if disableMakerAsk && disableMakerBid { + log.Warnf("%s bid/ask maker is disabled due to insufficient balances", s.Symbol) + return + } + + bestBidPrice := bestBid.Price + bestAskPrice := bestAsk.Price + log.Infof("%s book ticker: best ask / best bid = %v / %v", s.Symbol, bestAskPrice, bestBidPrice) + + var submitOrders []types.SubmitOrder + var accumulativeBidQuantity, accumulativeAskQuantity fixedpoint.Value + var bidQuantity = s.Quantity + var askQuantity = s.Quantity + var bidMargin = s.BidMargin + var askMargin = s.AskMargin + var pips = s.Pips + + bidPrice := bestBidPrice + askPrice := bestAskPrice + for i := 0; i < s.NumLayers; i++ { + // for maker bid orders + if !disableMakerBid { + if s.QuantityScale != nil { + qf, err := s.QuantityScale.Scale(i + 1) + if err != nil { + log.WithError(err).Errorf("quantityScale error") + return + } + + log.Infof("%s scaling bid #%d quantity to %f", s.Symbol, i+1, qf) + + // override the default bid quantity + bidQuantity = fixedpoint.NewFromFloat(qf) + } + + accumulativeBidQuantity = accumulativeBidQuantity.Add(bidQuantity) + if s.UseDepthPrice { + if s.DepthQuantity.Sign() > 0 { + bidPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeBuy), s.DepthQuantity) + } else { + bidPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeBuy), accumulativeBidQuantity) + } + } + + bidPrice = bidPrice.Mul(fixedpoint.One.Sub(bidMargin)) + if i > 0 && pips.Sign() > 0 { + bidPrice = bidPrice.Sub(pips.Mul(fixedpoint.NewFromInt(int64(i)). + Mul(s.makerMarket.TickSize))) + } + + if makerQuota.QuoteAsset.Lock(bidQuantity.Mul(bidPrice)) && hedgeQuota.BaseAsset.Lock(bidQuantity) { + // if we bought, then we need to sell the base from the hedge session + submitOrders = append(submitOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimit, + Side: types.SideTypeBuy, + Price: bidPrice, + Quantity: bidQuantity, + TimeInForce: types.TimeInForceGTC, + GroupID: s.groupID, + }) + + makerQuota.Commit() + hedgeQuota.Commit() + } else { + makerQuota.Rollback() + hedgeQuota.Rollback() + } + + if s.QuantityMultiplier.Sign() > 0 { + bidQuantity = bidQuantity.Mul(s.QuantityMultiplier) + } + } + + // for maker ask orders + if !disableMakerAsk { + if s.QuantityScale != nil { + qf, err := s.QuantityScale.Scale(i + 1) + if err != nil { + log.WithError(err).Errorf("quantityScale error") + return + } + + log.Infof("%s scaling ask #%d quantity to %f", s.Symbol, i+1, qf) + + // override the default bid quantity + askQuantity = fixedpoint.NewFromFloat(qf) + } + accumulativeAskQuantity = accumulativeAskQuantity.Add(askQuantity) + + if s.UseDepthPrice { + if s.DepthQuantity.Sign() > 0 { + askPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeSell), s.DepthQuantity) + } else { + askPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeSell), accumulativeAskQuantity) + } + } + + askPrice = askPrice.Mul(fixedpoint.One.Add(askMargin)) + if i > 0 && pips.Sign() > 0 { + askPrice = askPrice.Add(pips.Mul(fixedpoint.NewFromInt(int64(i)).Mul(s.makerMarket.TickSize))) + } + + if makerQuota.BaseAsset.Lock(askQuantity) && hedgeQuota.QuoteAsset.Lock(askQuantity.Mul(askPrice)) { + // if we bought, then we need to sell the base from the hedge session + submitOrders = append(submitOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Market: s.makerMarket, + Type: types.OrderTypeLimit, + Side: types.SideTypeSell, + Price: askPrice, + Quantity: askQuantity, + TimeInForce: types.TimeInForceGTC, + GroupID: s.groupID, + }) + makerQuota.Commit() + hedgeQuota.Commit() + } else { + makerQuota.Rollback() + hedgeQuota.Rollback() + } + + if s.QuantityMultiplier.Sign() > 0 { + askQuantity = askQuantity.Mul(s.QuantityMultiplier) + } + } + } + + if len(submitOrders) == 0 { + log.Warnf("no orders generated") + return + } + + makerOrders, err := orderExecutionRouter.SubmitOrdersTo(ctx, s.MakerExchange, submitOrders...) + if err != nil { + log.WithError(err).Errorf("order error: %s", err.Error()) + return + } + + s.activeMakerOrders.Add(makerOrders...) + s.orderStore.Add(makerOrders...) +} From df2daf33a72d33700967140bf5d045a75bdd0e29 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 27 Nov 2023 17:55:46 +0800 Subject: [PATCH 246/422] types: add PeriodProfitStats --- pkg/strategy/xdepthmaker/strategy.go | 18 ++---------------- pkg/types/profit.go | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 5d85be1c3d..5c40cd2320 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -62,9 +62,6 @@ type Strategy struct { // Quantity is used for fixed quantity of the first layer Quantity fixedpoint.Value `json:"quantity"` - // QuantityMultiplier is the factor that multiplies the quantity of the previous layer - QuantityMultiplier fixedpoint.Value `json:"quantityMultiplier"` - // QuantityScale helps user to define the quantity by layer scale QuantityScale *bbgo.LayerScale `json:"quantityScale,omitempty"` @@ -86,8 +83,8 @@ type Strategy struct { Pips fixedpoint.Value `json:"pips"` // -------------------------------- - // private field - + // private fields + // -------------------------------- makerSession, sourceSession *bbgo.ExchangeSession makerMarket, sourceMarket types.Market @@ -146,10 +143,6 @@ func (s *Strategy) Validate() error { return errors.New("quantity or quantityScale can not be empty") } - if !s.QuantityMultiplier.IsZero() && s.QuantityMultiplier.Sign() < 0 { - return errors.New("quantityMultiplier can not be a negative number") - } - if len(s.Symbol) == 0 { return errors.New("symbol is required") } @@ -735,10 +728,6 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or makerQuota.Rollback() hedgeQuota.Rollback() } - - if s.QuantityMultiplier.Sign() > 0 { - bidQuantity = bidQuantity.Mul(s.QuantityMultiplier) - } } // for maker ask orders @@ -789,9 +778,6 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or hedgeQuota.Rollback() } - if s.QuantityMultiplier.Sign() > 0 { - askQuantity = askQuantity.Mul(s.QuantityMultiplier) - } } } diff --git a/pkg/types/profit.go b/pkg/types/profit.go index c7742e0b8f..eb4d998acf 100644 --- a/pkg/types/profit.go +++ b/pkg/types/profit.go @@ -147,6 +147,19 @@ func (p *Profit) PlainText() string { ) } +// PeriodProfitStats defined the profit stats for a period +// TODO: replace AccumulatedPnL and TodayPnL fields from the ProfitStats struct +type PeriodProfitStats struct { + PnL fixedpoint.Value `json:"pnl,omitempty"` + NetProfit fixedpoint.Value `json:"netProfit,omitempty"` + GrossProfit fixedpoint.Value `json:"grossProfit,omitempty"` + GrossLoss fixedpoint.Value `json:"grossLoss,omitempty"` + Volume fixedpoint.Value `json:"volume,omitempty"` + LastTradeTime time.Time `json:"lastTradeTime,omitempty"` + StartTime time.Time `json:"startTime,omitempty"` + EndTime time.Time `json:"endTime,omitempty"` +} + type ProfitStats struct { Symbol string `json:"symbol"` QuoteCurrency string `json:"quoteCurrency"` @@ -164,9 +177,6 @@ type ProfitStats struct { TodayGrossProfit fixedpoint.Value `json:"todayGrossProfit,omitempty"` TodayGrossLoss fixedpoint.Value `json:"todayGrossLoss,omitempty"` TodaySince int64 `json:"todaySince,omitempty"` - - //StartTime time.Time - //EndTime time.Time } func NewProfitStats(market Market) *ProfitStats { @@ -185,8 +195,8 @@ func NewProfitStats(market Market) *ProfitStats { TodayGrossProfit: fixedpoint.Zero, TodayGrossLoss: fixedpoint.Zero, TodaySince: 0, - //StartTime: time.Now().UTC(), - //EndTime: time.Now().UTC(), + // StartTime: time.Now().UTC(), + // EndTime: time.Now().UTC(), } } @@ -229,7 +239,7 @@ func (s *ProfitStats) AddProfit(profit Profit) { s.TodayGrossLoss = s.TodayGrossLoss.Add(profit.Profit) } - //s.EndTime = profit.TradedAt.UTC() + // s.EndTime = profit.TradedAt.UTC() } func (s *ProfitStats) AddTrade(trade Trade) { From e67fa1932370289f96ad897be7cbb0439c16b7a0 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 27 Nov 2023 17:56:57 +0800 Subject: [PATCH 247/422] types: extend PeriodProfitStats fields --- pkg/types/profit.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/types/profit.go b/pkg/types/profit.go index eb4d998acf..cca9884148 100644 --- a/pkg/types/profit.go +++ b/pkg/types/profit.go @@ -155,9 +155,14 @@ type PeriodProfitStats struct { GrossProfit fixedpoint.Value `json:"grossProfit,omitempty"` GrossLoss fixedpoint.Value `json:"grossLoss,omitempty"` Volume fixedpoint.Value `json:"volume,omitempty"` - LastTradeTime time.Time `json:"lastTradeTime,omitempty"` - StartTime time.Time `json:"startTime,omitempty"` - EndTime time.Time `json:"endTime,omitempty"` + VolumeInQuote fixedpoint.Value `json:"volumeInQuote,omitempty"` + MakerVolume fixedpoint.Value `json:"makerVolume,omitempty"` + TakerVolume fixedpoint.Value `json:"takerVolume,omitempty"` + + // time fields + LastTradeTime time.Time `json:"lastTradeTime,omitempty"` + StartTime time.Time `json:"startTime,omitempty"` + EndTime time.Time `json:"endTime,omitempty"` } type ProfitStats struct { From ed63b23e2a4ab52c4f38e076adafa17ab9807726 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 15:54:06 +0800 Subject: [PATCH 248/422] xdepthmaker: refactor CrossRun with CrossExchangeMarketMakingStrategy --- pkg/bbgo/order_executor_general.go | 6 +- pkg/strategy/xdepthmaker/strategy.go | 251 +++++++++++++++++---------- 2 files changed, 165 insertions(+), 92 deletions(-) diff --git a/pkg/bbgo/order_executor_general.go b/pkg/bbgo/order_executor_general.go index c836a29102..0268294668 100644 --- a/pkg/bbgo/order_executor_general.go +++ b/pkg/bbgo/order_executor_general.go @@ -68,8 +68,12 @@ type GeneralOrderExecutor struct { disableNotify bool } +// NewGeneralOrderExecutor allocates a GeneralOrderExecutor +// which has its own order store, trade collector func NewGeneralOrderExecutor( - session *ExchangeSession, symbol, strategy, strategyInstanceID string, position *types.Position, + session *ExchangeSession, + symbol, strategy, strategyInstanceID string, + position *types.Position, ) *GeneralOrderExecutor { // Always update the position fields position.Strategy = strategy diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 5c40cd2320..951a1a8896 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -33,13 +33,119 @@ func init() { bbgo.RegisterStrategy(ID, &Strategy{}) } +func notifyTrade(trade types.Trade, _, _ fixedpoint.Value) { + bbgo.Notify(trade) +} + +type CrossExchangeMarketMakingStrategy struct { + ctx, parent context.Context + cancel context.CancelFunc + + Environ *bbgo.Environment + + makerSession, hedgeSession *bbgo.ExchangeSession + makerMarket, hedgeMarket types.Market + + Position *types.Position `json:"position,omitempty" persistence:"position"` + ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` + + MakerOrderExecutor, HedgeOrderExecutor *bbgo.GeneralOrderExecutor + + // orderStore is a shared order store between the maker session and the hedge session + orderStore *core.OrderStore + + // tradeCollector is a shared trade collector between the maker session and the hedge session + tradeCollector *core.TradeCollector +} + +func (s *CrossExchangeMarketMakingStrategy) Initialize( + ctx context.Context, environ *bbgo.Environment, + makerSession, hedgeSession *bbgo.ExchangeSession, + symbol, strategyID, instanceID string, +) error { + s.parent = ctx + s.ctx, s.cancel = context.WithCancel(ctx) + + s.Environ = environ + + s.makerSession = makerSession + s.hedgeSession = hedgeSession + + var ok bool + s.hedgeMarket, ok = s.hedgeSession.Market(symbol) + if !ok { + return fmt.Errorf("source session market %s is not defined", symbol) + } + + s.makerMarket, ok = s.makerSession.Market(symbol) + if !ok { + return fmt.Errorf("maker session market %s is not defined", symbol) + } + + if s.ProfitStats == nil { + s.ProfitStats = types.NewProfitStats(s.makerMarket) + } + + if s.Position == nil { + s.Position = types.NewPositionFromMarket(s.makerMarket) + } + + // Always update the position fields + s.Position.Strategy = strategyID + s.Position.StrategyInstanceID = instanceID + + // if anyone of the fee rate is defined, this assumes that both are defined. + // so that zero maker fee could be applied + for _, ses := range []*bbgo.ExchangeSession{makerSession, hedgeSession} { + if ses.MakerFeeRate.Sign() > 0 || ses.TakerFeeRate.Sign() > 0 { + s.Position.SetExchangeFeeRate(ses.ExchangeName, types.ExchangeFee{ + MakerFeeRate: ses.MakerFeeRate, + TakerFeeRate: ses.TakerFeeRate, + }) + } + } + + s.MakerOrderExecutor = bbgo.NewGeneralOrderExecutor( + makerSession, + s.makerMarket.Symbol, + strategyID, instanceID, + s.Position) + s.MakerOrderExecutor.BindEnvironment(environ) + s.MakerOrderExecutor.BindProfitStats(s.ProfitStats) + s.MakerOrderExecutor.Bind() + s.MakerOrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + // bbgo.Sync(ctx, s) + }) + + s.HedgeOrderExecutor = bbgo.NewGeneralOrderExecutor( + hedgeSession, + s.hedgeMarket.Symbol, + strategyID, instanceID, + s.Position) + s.HedgeOrderExecutor.BindEnvironment(environ) + s.HedgeOrderExecutor.BindProfitStats(s.ProfitStats) + s.HedgeOrderExecutor.Bind() + s.HedgeOrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { + // bbgo.Sync(ctx, s) + }) + + s.orderStore = core.NewOrderStore(s.Position.Symbol) + s.orderStore.BindStream(hedgeSession.UserDataStream) + s.orderStore.BindStream(makerSession.UserDataStream) + s.tradeCollector = core.NewTradeCollector(s.Position.Symbol, s.Position, s.orderStore) + + return nil +} + type Strategy struct { + *CrossExchangeMarketMakingStrategy + Environment *bbgo.Environment Symbol string `json:"symbol"` - // SourceExchange session name - SourceExchange string `json:"sourceExchange"` + // HedgeExchange session name + HedgeExchange string `json:"hedgeExchange"` // MakerExchange session name MakerExchange string `json:"makerExchange"` @@ -54,8 +160,6 @@ type Strategy struct { UseDepthPrice bool `json:"useDepthPrice"` DepthQuantity fixedpoint.Value `json:"depthQuantity"` - EnableBollBandMargin bool `json:"enableBollBandMargin"` - StopHedgeQuoteBalance fixedpoint.Value `json:"stopHedgeQuoteBalance"` StopHedgeBaseBalance fixedpoint.Value `json:"stopHedgeBaseBalance"` @@ -85,15 +189,9 @@ type Strategy struct { // -------------------------------- // private fields // -------------------------------- - makerSession, sourceSession *bbgo.ExchangeSession - - makerMarket, sourceMarket types.Market - state *State // persistence fields - Position *types.Position `json:"position,omitempty" persistence:"position"` - ProfitStats *ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` book *types.StreamOrderBook @@ -102,13 +200,9 @@ type Strategy struct { hedgeErrorLimiter *rate.Limiter hedgeErrorRateReservation *rate.Reservation - orderStore *core.OrderStore - tradeCollector *core.TradeCollector - askPriceHeartBeat, bidPriceHeartBeat types.PriceHeartBeat lastPrice fixedpoint.Value - groupID uint32 stopC chan struct{} } @@ -122,9 +216,9 @@ func (s *Strategy) InstanceID() string { } func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { - sourceSession, ok := sessions[s.SourceExchange] + sourceSession, ok := sessions[s.HedgeExchange] if !ok { - panic(fmt.Errorf("source session %s is not defined", s.SourceExchange)) + panic(fmt.Errorf("source session %s is not defined", s.HedgeExchange)) } sourceSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) @@ -194,50 +288,16 @@ func (s *Strategy) Initialize() error { func (s *Strategy) CrossRun( ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, ) error { - // configure sessions - sourceSession, ok := sessions[s.SourceExchange] - if !ok { - return fmt.Errorf("source exchange session %s is not defined", s.SourceExchange) - } - - s.sourceSession = sourceSession - - makerSession, ok := sessions[s.MakerExchange] - if !ok { - return fmt.Errorf("maker exchange session %s is not defined", s.MakerExchange) - } - - s.makerSession = makerSession - - s.sourceMarket, ok = s.sourceSession.Market(s.Symbol) - if !ok { - return fmt.Errorf("source session market %s is not defined", s.Symbol) - } - - s.makerMarket, ok = s.makerSession.Market(s.Symbol) - if !ok { - return fmt.Errorf("maker session market %s is not defined", s.Symbol) - } - - // restore state instanceID := s.InstanceID() - s.groupID = util.FNV32(instanceID) - log.Infof("using group id %d from fnv(%s)", s.groupID, instanceID) - if s.Position == nil { - s.Position = types.NewPositionFromMarket(s.makerMarket) - - // force update for legacy code - s.Position.Market = s.makerMarket + makerSession, hedgeSession, err := selectSessions2(sessions, s.MakerExchange, s.HedgeExchange) + if err != nil { + return err } - bbgo.Notify("xdepthmaker: %s position is restored", s.Symbol, s.Position) - - if s.ProfitStats == nil { - s.ProfitStats = &ProfitStats{ - ProfitStats: types.NewProfitStats(s.makerMarket), - MakerExchange: s.makerSession.ExchangeName, - } + s.CrossExchangeMarketMakingStrategy = &CrossExchangeMarketMakingStrategy{} + if err := s.CrossExchangeMarketMakingStrategy.Initialize(ctx, s.Environment, makerSession, hedgeSession, s.Symbol, ID, s.InstanceID()); err != nil { + return err } if s.CoveredPosition.IsZero() { @@ -253,34 +313,31 @@ func (s *Strategy) CrossRun( }) } - if s.sourceSession.MakerFeeRate.Sign() > 0 || s.sourceSession.TakerFeeRate.Sign() > 0 { - s.Position.SetExchangeFeeRate(types.ExchangeName(s.SourceExchange), types.ExchangeFee{ - MakerFeeRate: s.sourceSession.MakerFeeRate, - TakerFeeRate: s.sourceSession.TakerFeeRate, + if s.hedgeSession.MakerFeeRate.Sign() > 0 || s.hedgeSession.TakerFeeRate.Sign() > 0 { + s.Position.SetExchangeFeeRate(types.ExchangeName(s.HedgeExchange), types.ExchangeFee{ + MakerFeeRate: s.hedgeSession.MakerFeeRate, + TakerFeeRate: s.hedgeSession.TakerFeeRate, }) } s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(s.sourceSession.MarketDataStream) + s.book.BindStream(s.hedgeSession.MarketDataStream) s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeMakerOrders.BindStream(s.makerSession.UserDataStream) s.orderStore = core.NewOrderStore(s.Symbol) - s.orderStore.BindStream(s.sourceSession.UserDataStream) + s.orderStore.BindStream(s.hedgeSession.UserDataStream) s.orderStore.BindStream(s.makerSession.UserDataStream) - s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore) if s.NotifyTrade { - s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { - bbgo.Notify(trade) - }) + s.tradeCollector.OnTrade(notifyTrade) } s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() - if trade.Exchange == s.sourceSession.ExchangeName { + if trade.Exchange == s.hedgeSession.ExchangeName { s.CoveredPosition = s.CoveredPosition.Add(c) } @@ -307,7 +364,7 @@ func (s *Strategy) CrossRun( s.tradeCollector.OnRecover(func(trade types.Trade) { bbgo.Notify("Recovered trade", trade) }) - s.tradeCollector.BindStream(s.sourceSession.UserDataStream) + s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) s.tradeCollector.BindStream(s.makerSession.UserDataStream) s.stopC = make(chan struct{}) @@ -366,7 +423,7 @@ func (s *Strategy) CrossRun( uncoverPosition := position.Sub(s.CoveredPosition) absPos := uncoverPosition.Abs() - if !s.DisableHedge && absPos.Compare(s.sourceMarket.MinQuantity) > 0 { + if !s.DisableHedge && absPos.Compare(s.hedgeMarket.MinQuantity) > 0 { log.Infof("%s base position %v coveredPosition: %v uncoverPosition: %v", s.Symbol, position, @@ -429,28 +486,28 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { } notional := quantity.Mul(lastPrice) - if notional.Compare(s.sourceMarket.MinNotional) <= 0 { + if notional.Compare(s.hedgeMarket.MinNotional) <= 0 { log.Warnf("%s %v less than min notional, skipping hedge", s.Symbol, notional) return } // adjust quantity according to the balances - account := s.sourceSession.GetAccount() + account := s.hedgeSession.GetAccount() switch side { case types.SideTypeBuy: // check quote quantity - if quote, ok := account.Balance(s.sourceMarket.QuoteCurrency); ok { + if quote, ok := account.Balance(s.hedgeMarket.QuoteCurrency); ok { if quote.Available.Compare(notional) < 0 { // adjust price to higher 0.1%, so that we can ensure that the order can be executed quantity = bbgo.AdjustQuantityByMaxAmount(quantity, lastPrice.Mul(lastPriceModifier), quote.Available) - quantity = s.sourceMarket.TruncateQuantity(quantity) + quantity = s.hedgeMarket.TruncateQuantity(quantity) } } case types.SideTypeSell: // check quote quantity - if base, ok := account.Balance(s.sourceMarket.BaseCurrency); ok { + if base, ok := account.Balance(s.hedgeMarket.BaseCurrency); ok { if base.Available.Compare(quantity) < 0 { quantity = base.Available } @@ -458,15 +515,15 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { } // truncate quantity for the supported precision - quantity = s.sourceMarket.TruncateQuantity(quantity) + quantity = s.hedgeMarket.TruncateQuantity(quantity) - if notional.Compare(s.sourceMarket.MinNotional.Mul(minGap)) <= 0 { - log.Warnf("the adjusted amount %v is less than minimal notional %v, skipping hedge", notional, s.sourceMarket.MinNotional) + if notional.Compare(s.hedgeMarket.MinNotional.Mul(minGap)) <= 0 { + log.Warnf("the adjusted amount %v is less than minimal notional %v, skipping hedge", notional, s.hedgeMarket.MinNotional) return } - if quantity.Compare(s.sourceMarket.MinQuantity.Mul(minGap)) <= 0 { - log.Warnf("the adjusted quantity %v is less than minimal quantity %v, skipping hedge", quantity, s.sourceMarket.MinQuantity) + if quantity.Compare(s.hedgeMarket.MinQuantity.Mul(minGap)) <= 0 { + log.Warnf("the adjusted quantity %v is less than minimal quantity %v, skipping hedge", quantity, s.hedgeMarket.MinQuantity) return } @@ -481,9 +538,9 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { log.Infof("submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) bbgo.Notify("Submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) - orderExecutor := &bbgo.ExchangeOrderExecutor{Session: s.sourceSession} + orderExecutor := &bbgo.ExchangeOrderExecutor{Session: s.hedgeSession} returnOrders, err := orderExecutor.SubmitOrders(ctx, types.SubmitOrder{ - Market: s.sourceMarket, + Market: s.hedgeMarket, Symbol: s.Symbol, Type: types.OrderTypeMarket, Side: side, @@ -528,7 +585,7 @@ func (s *Strategy) tradeRecover(ctx context.Context) { if s.RecoverTrade { startTime := time.Now().Add(-tradeScanInterval).Add(-tradeScanOverlapBufferPeriod) - if err := s.tradeCollector.Recover(ctx, s.sourceSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { + if err := s.tradeCollector.Recover(ctx, s.hedgeSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { log.WithError(err).Errorf("query trades error") } @@ -605,20 +662,20 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or } } - hedgeBalances := s.sourceSession.GetAccount().Balances() + hedgeBalances := s.hedgeSession.GetAccount().Balances() hedgeQuota := &bbgo.QuotaTransaction{} - if b, ok := hedgeBalances[s.sourceMarket.BaseCurrency]; ok { + if b, ok := hedgeBalances[s.hedgeMarket.BaseCurrency]; ok { // to make bid orders, we need enough base asset in the foreign exchange, // if the base asset balance is not enough for selling if s.StopHedgeBaseBalance.Sign() > 0 { - minAvailable := s.StopHedgeBaseBalance.Add(s.sourceMarket.MinQuantity) + minAvailable := s.StopHedgeBaseBalance.Add(s.hedgeMarket.MinQuantity) if b.Available.Compare(minAvailable) > 0 { hedgeQuota.BaseAsset.Add(b.Available.Sub(minAvailable)) } else { log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) disableMakerBid = true } - } else if b.Available.Compare(s.sourceMarket.MinQuantity) > 0 { + } else if b.Available.Compare(s.hedgeMarket.MinQuantity) > 0 { hedgeQuota.BaseAsset.Add(b.Available) } else { log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) @@ -626,18 +683,18 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or } } - if b, ok := hedgeBalances[s.sourceMarket.QuoteCurrency]; ok { + if b, ok := hedgeBalances[s.hedgeMarket.QuoteCurrency]; ok { // to make ask orders, we need enough quote asset in the foreign exchange, // if the quote asset balance is not enough for buying if s.StopHedgeQuoteBalance.Sign() > 0 { - minAvailable := s.StopHedgeQuoteBalance.Add(s.sourceMarket.MinNotional) + minAvailable := s.StopHedgeQuoteBalance.Add(s.hedgeMarket.MinNotional) if b.Available.Compare(minAvailable) > 0 { hedgeQuota.QuoteAsset.Add(b.Available.Sub(minAvailable)) } else { log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) disableMakerAsk = true } - } else if b.Available.Compare(s.sourceMarket.MinNotional) > 0 { + } else if b.Available.Compare(s.hedgeMarket.MinNotional) > 0 { hedgeQuota.QuoteAsset.Add(b.Available) } else { log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) @@ -719,7 +776,6 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or Price: bidPrice, Quantity: bidQuantity, TimeInForce: types.TimeInForceGTC, - GroupID: s.groupID, }) makerQuota.Commit() @@ -769,7 +825,6 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or Price: askPrice, Quantity: askQuantity, TimeInForce: types.TimeInForceGTC, - GroupID: s.groupID, }) makerQuota.Commit() hedgeQuota.Commit() @@ -795,3 +850,17 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or s.activeMakerOrders.Add(makerOrders...) s.orderStore.Add(makerOrders...) } + +func selectSessions2( + sessions map[string]*bbgo.ExchangeSession, n1, n2 string, +) (s1, s2 *bbgo.ExchangeSession, err error) { + for _, n := range []string{n1, n2} { + if _, ok := sessions[n]; !ok { + return nil, nil, fmt.Errorf("session %s is not defined", n) + } + } + + s1 = sessions[n1] + s2 = sessions[n2] + return s1, s2, nil +} From 6b289101391c4699ffd5727f2e3ab6c48760699d Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 15:56:39 +0800 Subject: [PATCH 249/422] xdepthmaker: refactor CrossSubscribe --- pkg/strategy/xdepthmaker/strategy.go | 32 ++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 951a1a8896..d6b1d612cd 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -194,7 +194,9 @@ type Strategy struct { // persistence fields CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` - book *types.StreamOrderBook + // pricingBook is the order book (depth) from the hedging session + pricingBook *types.StreamOrderBook + activeMakerOrders *bbgo.ActiveOrderBook hedgeErrorLimiter *rate.Limiter @@ -216,19 +218,13 @@ func (s *Strategy) InstanceID() string { } func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { - sourceSession, ok := sessions[s.HedgeExchange] - if !ok { - panic(fmt.Errorf("source session %s is not defined", s.HedgeExchange)) - } - - sourceSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) - sourceSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) - - makerSession, ok := sessions[s.MakerExchange] - if !ok { - panic(fmt.Errorf("maker session %s is not defined", s.MakerExchange)) + makerSession, hedgeSession, err := selectSessions2(sessions, s.MakerExchange, s.HedgeExchange) + if err != nil { + panic(err) } + hedgeSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) + hedgeSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) makerSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) } @@ -320,8 +316,8 @@ func (s *Strategy) CrossRun( }) } - s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(s.hedgeSession.MarketDataStream) + s.pricingBook = types.NewStreamBook(s.Symbol) + s.pricingBook.BindStream(s.hedgeSession.MarketDataStream) s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeMakerOrders.BindStream(s.makerSession.UserDataStream) @@ -471,7 +467,7 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { } lastPrice := s.lastPrice - sourceBook := s.book.CopyDepth(1) + sourceBook := s.pricingBook.CopyDepth(1) switch side { case types.SideTypeBuy: @@ -608,7 +604,7 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } - bestBid, bestAsk, hasPrice := s.book.BestBidAndAsk() + bestBid, bestAsk, hasPrice := s.pricingBook.BestBidAndAsk() if !hasPrice { return } @@ -616,7 +612,7 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or // use mid-price for the last price s.lastPrice = bestBid.Price.Add(bestAsk.Price).Div(Two) - bookLastUpdateTime := s.book.LastUpdateTime() + bookLastUpdateTime := s.pricingBook.LastUpdateTime() if _, err := s.bidPriceHeartBeat.Update(bestBid, priceUpdateTimeout); err != nil { log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", @@ -632,7 +628,7 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } - sourceBook := s.book.CopyDepth(10) + sourceBook := s.pricingBook.CopyDepth(10) if valid, err := sourceBook.IsValid(); !valid { log.WithError(err).Errorf("%s invalid copied order book, skip quoting: %v", s.Symbol, err) return From e0686d11c8e64deffa1e9d34a29e8e97c0980939 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 16:15:35 +0800 Subject: [PATCH 250/422] xdepthmaker: clean up duplicated code --- pkg/strategy/xdepthmaker/strategy.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index d6b1d612cd..3ccf505307 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -302,31 +302,12 @@ func (s *Strategy) CrossRun( } } - if s.makerSession.MakerFeeRate.Sign() > 0 || s.makerSession.TakerFeeRate.Sign() > 0 { - s.Position.SetExchangeFeeRate(types.ExchangeName(s.MakerExchange), types.ExchangeFee{ - MakerFeeRate: s.makerSession.MakerFeeRate, - TakerFeeRate: s.makerSession.TakerFeeRate, - }) - } - - if s.hedgeSession.MakerFeeRate.Sign() > 0 || s.hedgeSession.TakerFeeRate.Sign() > 0 { - s.Position.SetExchangeFeeRate(types.ExchangeName(s.HedgeExchange), types.ExchangeFee{ - MakerFeeRate: s.hedgeSession.MakerFeeRate, - TakerFeeRate: s.hedgeSession.TakerFeeRate, - }) - } - s.pricingBook = types.NewStreamBook(s.Symbol) s.pricingBook.BindStream(s.hedgeSession.MarketDataStream) s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeMakerOrders.BindStream(s.makerSession.UserDataStream) - s.orderStore = core.NewOrderStore(s.Symbol) - s.orderStore.BindStream(s.hedgeSession.UserDataStream) - s.orderStore.BindStream(s.makerSession.UserDataStream) - s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore) - if s.NotifyTrade { s.tradeCollector.OnTrade(notifyTrade) } From 99723fc1f44e8988e3d40b4f0017f9d643f657d2 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 16:32:26 +0800 Subject: [PATCH 251/422] xdepthmaker: remove legacy s.activeMakerOrders --- pkg/strategy/xdepthmaker/strategy.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 3ccf505307..85e9fac1f2 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -132,8 +132,7 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( s.orderStore = core.NewOrderStore(s.Position.Symbol) s.orderStore.BindStream(hedgeSession.UserDataStream) s.orderStore.BindStream(makerSession.UserDataStream) - s.tradeCollector = core.NewTradeCollector(s.Position.Symbol, s.Position, s.orderStore) - + s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) return nil } @@ -197,8 +196,6 @@ type Strategy struct { // pricingBook is the order book (depth) from the hedging session pricingBook *types.StreamOrderBook - activeMakerOrders *bbgo.ActiveOrderBook - hedgeErrorLimiter *rate.Limiter hedgeErrorRateReservation *rate.Reservation @@ -305,9 +302,6 @@ func (s *Strategy) CrossRun( s.pricingBook = types.NewStreamBook(s.Symbol) s.pricingBook.BindStream(s.hedgeSession.MarketDataStream) - s.activeMakerOrders = bbgo.NewActiveOrderBook(s.Symbol) - s.activeMakerOrders.BindStream(s.makerSession.UserDataStream) - if s.NotifyTrade { s.tradeCollector.OnTrade(notifyTrade) } @@ -361,7 +355,7 @@ func (s *Strategy) CrossRun( defer reportTicker.Stop() defer func() { - if err := s.activeMakerOrders.GracefulCancel(context.Background(), s.makerSession.Exchange); err != nil { + if err := s.MakerOrderExecutor.GracefulCancel(context.Background()); err != nil { log.WithError(err).Errorf("can not cancel %s orders", s.Symbol) } }() @@ -425,7 +419,7 @@ func (s *Strategy) CrossRun( shutdownCtx, cancelShutdown := context.WithTimeout(context.TODO(), time.Minute) defer cancelShutdown() - if err := s.activeMakerOrders.GracefulCancel(shutdownCtx, s.makerSession.Exchange); err != nil { + if err := s.MakerOrderExecutor.GracefulCancel(shutdownCtx); err != nil { log.WithError(err).Errorf("graceful cancel error") } @@ -575,13 +569,13 @@ func (s *Strategy) tradeRecover(ctx context.Context) { } func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter) { - if err := s.activeMakerOrders.GracefulCancel(ctx, s.makerSession.Exchange); err != nil { + if err := s.MakerOrderExecutor.GracefulCancel(ctx); err != nil { log.Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) - s.activeMakerOrders.Print() + s.MakerOrderExecutor.ActiveMakerOrders().Print() return } - if s.activeMakerOrders.NumOfOrders() > 0 { + if s.MakerOrderExecutor.ActiveMakerOrders().NumOfOrders() > 0 { return } @@ -818,14 +812,11 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } - makerOrders, err := orderExecutionRouter.SubmitOrdersTo(ctx, s.MakerExchange, submitOrders...) + _, err := s.MakerOrderExecutor.SubmitOrders(ctx, submitOrders...) if err != nil { log.WithError(err).Errorf("order error: %s", err.Error()) return } - - s.activeMakerOrders.Add(makerOrders...) - s.orderStore.Add(makerOrders...) } func selectSessions2( From 10a71d83f16ba7ebd8d2072eb5ca4262b0c28f10 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 16:46:58 +0800 Subject: [PATCH 252/422] xdepthmaker: move global position profit handling --- pkg/strategy/xdepthmaker/strategy.go | 75 ++++++++++++---------------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 85e9fac1f2..35e935c80d 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -46,8 +46,10 @@ type CrossExchangeMarketMakingStrategy struct { makerSession, hedgeSession *bbgo.ExchangeSession makerMarket, hedgeMarket types.Market - Position *types.Position `json:"position,omitempty" persistence:"position"` - ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` + // persistence fields + Position *types.Position `json:"position,omitempty" persistence:"position"` + ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` + CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` MakerOrderExecutor, HedgeOrderExecutor *bbgo.GeneralOrderExecutor @@ -133,6 +135,28 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( s.orderStore.BindStream(hedgeSession.UserDataStream) s.orderStore.BindStream(makerSession.UserDataStream) s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) + + s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { + c := trade.PositionChange() + if trade.Exchange == s.hedgeSession.ExchangeName { + s.CoveredPosition.AtomicAdd(c) + } + + s.ProfitStats.AddTrade(trade) + + if profit.Compare(fixedpoint.Zero) == 0 { + s.Environ.RecordPosition(s.Position, trade, nil) + } else { + log.Infof("%s generated profit: %v", symbol, profit) + + p := s.Position.NewProfit(trade, profit, netProfit) + bbgo.Notify(&p) + s.ProfitStats.AddProfit(p) + + s.Environ.RecordPosition(s.Position, trade, &p) + } + }) + return nil } @@ -188,10 +212,6 @@ type Strategy struct { // -------------------------------- // private fields // -------------------------------- - state *State - - // persistence fields - CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` // pricingBook is the order book (depth) from the hedging session pricingBook *types.StreamOrderBook @@ -281,8 +301,6 @@ func (s *Strategy) Initialize() error { func (s *Strategy) CrossRun( ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, ) error { - instanceID := s.InstanceID() - makerSession, hedgeSession, err := selectSessions2(sessions, s.MakerExchange, s.HedgeExchange) if err != nil { return err @@ -293,12 +311,6 @@ func (s *Strategy) CrossRun( return err } - if s.CoveredPosition.IsZero() { - if s.state != nil && !s.CoveredPosition.IsZero() { - s.CoveredPosition = s.state.CoveredPosition - } - } - s.pricingBook = types.NewStreamBook(s.Symbol) s.pricingBook.BindStream(s.hedgeSession.MarketDataStream) @@ -306,29 +318,6 @@ func (s *Strategy) CrossRun( s.tradeCollector.OnTrade(notifyTrade) } - s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { - c := trade.PositionChange() - if trade.Exchange == s.hedgeSession.ExchangeName { - s.CoveredPosition = s.CoveredPosition.Add(c) - } - - s.ProfitStats.AddTrade(trade) - - if profit.Compare(fixedpoint.Zero) == 0 { - s.Environment.RecordPosition(s.Position, trade, nil) - } else { - log.Infof("%s generated profit: %v", s.Symbol, profit) - - p := s.Position.NewProfit(trade, profit, netProfit) - p.Strategy = ID - p.StrategyInstanceID = instanceID - bbgo.Notify(&p) - s.ProfitStats.AddProfit(p) - - s.Environment.RecordPosition(s.Position, trade, &p) - } - }) - s.tradeCollector.OnPositionUpdate(func(position *types.Position) { bbgo.Notify(position) }) @@ -354,12 +343,6 @@ func (s *Strategy) CrossRun( reportTicker := time.NewTicker(time.Hour) defer reportTicker.Stop() - defer func() { - if err := s.MakerOrderExecutor.GracefulCancel(context.Background()); err != nil { - log.WithError(err).Errorf("can not cancel %s orders", s.Symbol) - } - }() - for { select { @@ -420,7 +403,11 @@ func (s *Strategy) CrossRun( defer cancelShutdown() if err := s.MakerOrderExecutor.GracefulCancel(shutdownCtx); err != nil { - log.WithError(err).Errorf("graceful cancel error") + log.WithError(err).Errorf("graceful cancel %s order error", s.Symbol) + } + + if err := s.HedgeOrderExecutor.GracefulCancel(shutdownCtx); err != nil { + log.WithError(err).Errorf("graceful cancel %s order error", s.Symbol) } bbgo.Notify("%s: %s position", ID, s.Symbol, s.Position) From 18968c67a13081f62dbfe132c48c2b1a656f1fa7 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 16:58:54 +0800 Subject: [PATCH 253/422] xdepthmaker: remove disable hedge option --- pkg/strategy/xdepthmaker/strategy.go | 30 ++++++++++------------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 35e935c80d..68155ebc9f 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -134,8 +134,8 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( s.orderStore = core.NewOrderStore(s.Position.Symbol) s.orderStore.BindStream(hedgeSession.UserDataStream) s.orderStore.BindStream(makerSession.UserDataStream) - s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) + s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() if trade.Exchange == s.hedgeSession.ExchangeName { @@ -156,6 +156,8 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( s.Environ.RecordPosition(s.Position, trade, &p) } }) + s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) + s.tradeCollector.BindStream(s.makerSession.UserDataStream) return nil } @@ -195,8 +197,6 @@ type Strategy struct { // MaxExposurePosition defines the unhedged quantity of stop MaxExposurePosition fixedpoint.Value `json:"maxExposurePosition"` - DisableHedge bool `json:"disableHedge"` - NotifyTrade bool `json:"notifyTrade"` // RecoverTrade tries to find the missing trades via the REStful API @@ -240,7 +240,11 @@ func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { panic(err) } - hedgeSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) + hedgeSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{ + Depth: types.DepthLevelMedium, + Speed: types.SpeedHigh, + }) + hedgeSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) makerSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) } @@ -324,8 +328,6 @@ func (s *Strategy) CrossRun( s.tradeCollector.OnRecover(func(trade types.Trade) { bbgo.Notify("Recovered trade", trade) }) - s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) - s.tradeCollector.BindStream(s.makerSession.UserDataStream) s.stopC = make(chan struct{}) @@ -337,12 +339,6 @@ func (s *Strategy) CrossRun( posTicker := time.NewTicker(util.MillisecondsJitter(s.HedgeInterval.Duration(), 200)) defer posTicker.Stop() - quoteTicker := time.NewTicker(util.MillisecondsJitter(s.UpdateInterval.Duration(), 200)) - defer quoteTicker.Stop() - - reportTicker := time.NewTicker(time.Hour) - defer reportTicker.Stop() - for { select { @@ -354,12 +350,6 @@ func (s *Strategy) CrossRun( log.Warnf("%s maker goroutine stopped, due to the cancelled context", s.Symbol) return - case <-quoteTicker.C: - s.updateQuote(ctx, orderExecutionRouter) - - case <-reportTicker.C: - bbgo.Notify(s.ProfitStats) - case <-posTicker.C: // For positive position and positive covered position: // uncover position = +5 - +3 (covered position) = 2 @@ -377,7 +367,7 @@ func (s *Strategy) CrossRun( uncoverPosition := position.Sub(s.CoveredPosition) absPos := uncoverPosition.Abs() - if !s.DisableHedge && absPos.Compare(s.hedgeMarket.MinQuantity) > 0 { + if absPos.Compare(s.hedgeMarket.MinQuantity) > 0 { log.Infof("%s base position %v coveredPosition: %v uncoverPosition: %v", s.Symbol, position, @@ -795,7 +785,7 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or } if len(submitOrders) == 0 { - log.Warnf("no orders generated") + log.Warnf("no orders are generated") return } From 2c3792b290e0915cea725b9b1a87b5274a4dd093 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 28 Nov 2023 17:01:11 +0800 Subject: [PATCH 254/422] xdepthmaker: update Validate() method --- pkg/strategy/xdepthmaker/strategy.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 68155ebc9f..4b015c18a5 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -250,6 +250,14 @@ func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { } func (s *Strategy) Validate() error { + if s.MakerExchange == "" { + return errors.New("maker exchange is not configured") + } + + if s.HedgeExchange == "" { + return errors.New("maker exchange is not configured") + } + if s.Quantity.IsZero() || s.QuantityScale == nil { return errors.New("quantity or quantityScale can not be empty") } From 1e27f53891fd4214eaec0a53883410612d53ce7b Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 29 Nov 2023 09:39:03 +0800 Subject: [PATCH 255/422] xdepthmaker: use hedge order executor --- pkg/strategy/xdepthmaker/strategy.go | 44 ++++++++++++++++++++-------- pkg/types/orderbook.go | 30 +++++++++++++++---- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 4b015c18a5..b42335be8d 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -311,7 +311,8 @@ func (s *Strategy) Initialize() error { } func (s *Strategy) CrossRun( - ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, + ctx context.Context, _ bbgo.OrderExecutionRouter, + sessions map[string]*bbgo.ExchangeSession, ) error { makerSession, hedgeSession, err := selectSessions2(sessions, s.MakerExchange, s.HedgeExchange) if err != nil { @@ -333,14 +334,15 @@ func (s *Strategy) CrossRun( s.tradeCollector.OnPositionUpdate(func(position *types.Position) { bbgo.Notify(position) }) - s.tradeCollector.OnRecover(func(trade types.Trade) { - bbgo.Notify("Recovered trade", trade) - }) s.stopC = make(chan struct{}) if s.RecoverTrade { - go s.tradeRecover(ctx) + s.tradeCollector.OnRecover(func(trade types.Trade) { + bbgo.Notify("Recovered trade", trade) + }) + + go s.runTradeRecover(ctx) } go func() { @@ -358,6 +360,18 @@ func (s *Strategy) CrossRun( log.Warnf("%s maker goroutine stopped, due to the cancelled context", s.Symbol) return + case sig, ok := <-s.pricingBook.C: + // when any book change event happened + if !ok { + return + } + + switch sig.Type { + case types.BookSignalSnapshot: + case types.BookSignalUpdate: + + } + case <-posTicker.C: // For positive position and positive covered position: // uncover position = +5 - +3 (covered position) = 2 @@ -494,8 +508,8 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { log.Infof("submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) bbgo.Notify("Submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) - orderExecutor := &bbgo.ExchangeOrderExecutor{Session: s.hedgeSession} - returnOrders, err := orderExecutor.SubmitOrders(ctx, types.SubmitOrder{ + + createdOrders, err := s.HedgeOrderExecutor.SubmitOrders(ctx, types.SubmitOrder{ Market: s.hedgeMarket, Symbol: s.Symbol, Type: types.OrderTypeMarket, @@ -509,17 +523,17 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { return } - // if it's selling, than we should add positive position + s.orderStore.Add(createdOrders...) + + // if it's selling, then we should add positive position if side == types.SideTypeSell { s.CoveredPosition = s.CoveredPosition.Add(quantity) } else { s.CoveredPosition = s.CoveredPosition.Add(quantity.Neg()) } - - s.orderStore.Add(returnOrders...) } -func (s *Strategy) tradeRecover(ctx context.Context) { +func (s *Strategy) runTradeRecover(ctx context.Context) { tradeScanInterval := s.RecoverTradeScanPeriod.Duration() if tradeScanInterval == 0 { tradeScanInterval = 30 * time.Minute @@ -560,7 +574,9 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } - if s.MakerOrderExecutor.ActiveMakerOrders().NumOfOrders() > 0 { + numOfMakerOrders := s.MakerOrderExecutor.ActiveMakerOrders().NumOfOrders() + if numOfMakerOrders > 0 { + log.Warnf("maker orders are not all canceled") return } @@ -797,11 +813,13 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } - _, err := s.MakerOrderExecutor.SubmitOrders(ctx, submitOrders...) + createdOrders, err := s.MakerOrderExecutor.SubmitOrders(ctx, submitOrders...) if err != nil { log.WithError(err).Errorf("order error: %s", err.Error()) return } + + s.orderStore.Add(createdOrders...) } func selectSessions2( diff --git a/pkg/types/orderbook.go b/pkg/types/orderbook.go index 78d14ced38..3d32989c86 100644 --- a/pkg/types/orderbook.go +++ b/pkg/types/orderbook.go @@ -7,7 +7,6 @@ import ( "time" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/sigchan" ) type OrderBook interface { @@ -114,13 +113,26 @@ func (b *MutexOrderBook) Update(update SliceOrderBook) { b.Unlock() } -//go:generate callbackgen -type StreamOrderBook +type BookSignalType string + +const ( + BookSignalSnapshot BookSignalType = "snapshot" + BookSignalUpdate BookSignalType = "update" +) + +type BookSignal struct { + Type BookSignalType + Book SliceOrderBook +} + // StreamOrderBook receives streaming data from websocket connection and // update the order book with mutex lock, so you can safely access it. +// +//go:generate callbackgen -type StreamOrderBook type StreamOrderBook struct { *MutexOrderBook - C sigchan.Chan + C chan BookSignal updateCallbacks []func(update SliceOrderBook) snapshotCallbacks []func(snapshot SliceOrderBook) @@ -129,7 +141,7 @@ type StreamOrderBook struct { func NewStreamBook(symbol string) *StreamOrderBook { return &StreamOrderBook{ MutexOrderBook: NewMutexOrderBook(symbol), - C: sigchan.New(60), + C: make(chan BookSignal, 1), } } @@ -141,7 +153,9 @@ func (sb *StreamOrderBook) BindStream(stream Stream) { sb.Load(book) sb.EmitSnapshot(book) - sb.C.Emit() + + // when it's snapshot, it's very important to push the snapshot signal to the caller + sb.C <- BookSignal{Type: BookSignalSnapshot, Book: book} }) stream.OnBookUpdate(func(book SliceOrderBook) { @@ -151,6 +165,10 @@ func (sb *StreamOrderBook) BindStream(stream Stream) { sb.Update(book) sb.EmitUpdate(book) - sb.C.Emit() + + select { + case sb.C <- BookSignal{Type: BookSignalUpdate, Book: book}: + default: + } }) } From d123e89a1b2349ff1619c8041c35a2c25c5b1ed5 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 29 Nov 2023 16:19:22 +0800 Subject: [PATCH 256/422] xdepthmaker: document covered position --- pkg/strategy/xdepthmaker/strategy.go | 19 ++++++++++++++----- pkg/types/trade.go | 3 +++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index b42335be8d..7352adeb94 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -138,6 +138,14 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() + + // sync covered position + // sell trade -> negative delta -> + // 1) long position -> reduce long position + // 2) short position -> increase short position + // buy trade -> positive delta -> + // 1) short position -> reduce short position + // 2) short position -> increase short position if trade.Exchange == s.hedgeSession.ExchangeName { s.CoveredPosition.AtomicAdd(c) } @@ -525,11 +533,12 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { s.orderStore.Add(createdOrders...) - // if it's selling, then we should add positive position - if side == types.SideTypeSell { - s.CoveredPosition = s.CoveredPosition.Add(quantity) - } else { - s.CoveredPosition = s.CoveredPosition.Add(quantity.Neg()) + // if the hedge is on sell side, then we should add positive position + switch side { + case types.SideTypeSell: + s.CoveredPosition.AtomicAdd(quantity) + case types.SideTypeBuy: + s.CoveredPosition.AtomicAdd(quantity.Neg()) } } diff --git a/pkg/types/trade.go b/pkg/types/trade.go index 5a7eb0a7a0..9d7f57ac0e 100644 --- a/pkg/types/trade.go +++ b/pkg/types/trade.go @@ -120,6 +120,9 @@ func (trade Trade) CsvRecords() [][]string { } } +// PositionChange returns the position delta of this trade +// BUY trade -> positive quantity +// SELL trade -> negative quantity func (trade Trade) PositionChange() fixedpoint.Value { q := trade.Quantity switch trade.Side { From 263c0883d10e82a6ea95fc64e129eeecc3f015f5 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 29 Nov 2023 16:35:37 +0800 Subject: [PATCH 257/422] bbgo: solve the scale when unmarshalling the json --- pkg/bbgo/scale.go | 13 +++++++++++++ pkg/bbgo/scale_test.go | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/pkg/bbgo/scale.go b/pkg/bbgo/scale.go index 207a42589b..b973319b68 100644 --- a/pkg/bbgo/scale.go +++ b/pkg/bbgo/scale.go @@ -1,6 +1,7 @@ package bbgo import ( + "encoding/json" "fmt" "math" @@ -318,6 +319,18 @@ type LayerScale struct { LayerRule *SlideRule `json:"byLayer"` } +func (s *LayerScale) UnmarshalJSON(data []byte) error { + type T LayerScale + var p T + err := json.Unmarshal(data, &p) + if err != nil { + return err + } + + *s = LayerScale(p) + return nil +} + func (s *LayerScale) Scale(layer int) (quantity float64, err error) { if s.LayerRule == nil { err = errors.New("either price or volume scale is not defined") diff --git a/pkg/bbgo/scale_test.go b/pkg/bbgo/scale_test.go index 1ad86a0c4d..a193747cdc 100644 --- a/pkg/bbgo/scale_test.go +++ b/pkg/bbgo/scale_test.go @@ -1,6 +1,7 @@ package bbgo import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -8,6 +9,24 @@ import ( const delta = 1e-9 +func TestLayerScale_UnmarshalJSON(t *testing.T) { + var s LayerScale + err := json.Unmarshal([]byte(`{ + "byLayer": { + "linear": { + "domain": [ 1, 3 ], + "range": [ 10000.0, 30000.0 ] + } + } + }`), &s) + assert.NoError(t, err) + + if assert.NotNil(t, s.LayerRule) { + assert.NotNil(t, s.LayerRule.LinearScale.Range) + assert.NotNil(t, s.LayerRule.LinearScale.Domain) + } +} + func TestExponentialScale(t *testing.T) { // graph see: https://www.desmos.com/calculator/ip0ijbcbbf scale := ExponentialScale{ From 46b3a81b07ff3af2475f0c2c5740ff059b39500b Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 10:32:03 +0800 Subject: [PATCH 258/422] xdepthmaker: add tests to the generateMakerOrders --- config/xdepthmaker.yaml | 67 +++++++++++ pkg/strategy/xdepthmaker/strategy.go | 127 ++++++++++++++++++++- pkg/strategy/xdepthmaker/strategy_test.go | 72 ++++++++++++ pkg/testing/testhelper/assert_priceside.go | 2 +- pkg/types/price_volume_slice.go | 21 +++- 5 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 config/xdepthmaker.yaml create mode 100644 pkg/strategy/xdepthmaker/strategy_test.go diff --git a/config/xdepthmaker.yaml b/config/xdepthmaker.yaml new file mode 100644 index 0000000000..c278bbde96 --- /dev/null +++ b/config/xdepthmaker.yaml @@ -0,0 +1,67 @@ +--- +notifications: + slack: + defaultChannel: "dev-bbgo" + errorChannel: "bbgo-error" + + switches: + trade: true + orderUpdate: false + submitOrder: false + +persistence: + json: + directory: var/data + redis: + host: 127.0.0.1 + port: 6379 + db: 0 + +logging: + trade: true + order: true + fields: + env: staging + +sessions: + max: + exchange: max + envVarPrefix: max + + binance: + exchange: binance + envVarPrefix: binance + +crossExchangeStrategies: + +- xdepthmaker: + symbol: "BTCUSDT" + makerExchange: max + hedgeExchange: binance + + # disableHedge disables the hedge orders on the source exchange + # disableHedge: true + + hedgeInterval: 10s + notifyTrade: true + + margin: 0.004 + askMargin: 0.4% + bidMargin: 0.4% + + depthScale: + byLayer: + linear: + domain: [1, 30] + range: [50, 20_000] + + # numLayers means how many order we want to place on each side. 3 means we want 3 bid orders and 3 ask orders + numLayers: 30 + + # pips is the fraction numbers between each order. for BTC, 1 pip is 0.1, + # 0.1 pip is 0.01, here we use 10, so we will get 18000.00, 18001.00 and + # 18002.00 + pips: 10 + persistence: + type: redis + diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 7352adeb94..fea67795c8 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -202,6 +202,9 @@ type Strategy struct { // QuantityScale helps user to define the quantity by layer scale QuantityScale *bbgo.LayerScale `json:"quantityScale,omitempty"` + // DepthScale helps user to define the depth by layer scale + DepthScale *bbgo.LayerScale `json:"depthScale,omitempty"` + // MaxExposurePosition defines the unhedged quantity of stop MaxExposurePosition fixedpoint.Value `json:"maxExposurePosition"` @@ -266,8 +269,8 @@ func (s *Strategy) Validate() error { return errors.New("maker exchange is not configured") } - if s.Quantity.IsZero() || s.QuantityScale == nil { - return errors.New("quantity or quantityScale can not be empty") + if s.DepthScale == nil { + return errors.New("depthScale can not be empty") } if len(s.Symbol) == 0 { @@ -576,7 +579,106 @@ func (s *Strategy) runTradeRecover(ctx context.Context) { } } -func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter) { +func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook) ([]types.SubmitOrder, error) { + bestBid, bestAsk, hasPrice := pricingBook.BestBidAndAsk() + if !hasPrice { + return nil, nil + } + + bestBidPrice := bestBid.Price + bestAskPrice := bestAsk.Price + log.Infof("%s book ticker: best ask / best bid = %v / %v", s.Symbol, bestAskPrice, bestBidPrice) + + lastMidPrice := bestBidPrice.Add(bestAskPrice).Div(Two) + _ = lastMidPrice + + var submitOrders []types.SubmitOrder + var accumulatedBidQuantity = fixedpoint.Zero + var accumulatedAskQuantity = fixedpoint.Zero + var accumulatedBidQuoteQuantity = fixedpoint.Zero + + dupPricingBook := pricingBook.CopyDepth(0) + + for _, side := range []types.SideType{types.SideTypeBuy, types.SideTypeSell} { + for i := 1; i <= s.NumLayers; i++ { + requiredDepthFloat, err := s.DepthScale.Scale(i) + if err != nil { + return nil, errors.Wrapf(err, "depthScale scale error") + } + + // requiredDepth is the required depth in quote currency + requiredDepth := fixedpoint.NewFromFloat(requiredDepthFloat) + + sideBook := dupPricingBook.SideBook(side) + index := sideBook.IndexByQuoteVolumeDepth(requiredDepth) + + pvs := types.PriceVolumeSlice{} + if index == -1 { + pvs = sideBook[:] + } else { + pvs = sideBook[0 : index+1] + } + + log.Infof("required depth: %f, pvs: %+v", requiredDepth.Float64(), pvs) + + depthPrice, err := averageDepthPrice(pvs) + if err != nil { + log.WithError(err).Errorf("error aggregating depth price") + continue + } + + switch side { + case types.SideTypeBuy: + if s.BidMargin.Sign() > 0 { + depthPrice = depthPrice.Mul(fixedpoint.One.Sub(s.BidMargin)) + } + + depthPrice = depthPrice.Round(s.makerMarket.PricePrecision+1, fixedpoint.Down) + + case types.SideTypeSell: + if s.AskMargin.Sign() > 0 { + depthPrice = depthPrice.Mul(fixedpoint.One.Add(s.AskMargin)) + } + + depthPrice = depthPrice.Round(s.makerMarket.PricePrecision+1, fixedpoint.Up) + } + + depthPrice = s.makerMarket.TruncatePrice(depthPrice) + + quantity := requiredDepth.Div(depthPrice) + quantity = s.makerMarket.TruncateQuantity(quantity) + log.Infof("side: %s required depth: %f price: %f quantity: %f", side, requiredDepth.Float64(), depthPrice.Float64(), quantity.Float64()) + + switch side { + case types.SideTypeBuy: + quantity = quantity.Sub(accumulatedBidQuantity) + + accumulatedBidQuantity = accumulatedBidQuantity.Add(quantity) + quoteQuantity := fixedpoint.Mul(quantity, depthPrice) + quoteQuantity = quoteQuantity.Round(s.makerMarket.PricePrecision, fixedpoint.Up) + accumulatedBidQuoteQuantity = accumulatedBidQuoteQuantity.Add(quoteQuantity) + + case types.SideTypeSell: + quantity = quantity.Sub(accumulatedAskQuantity) + accumulatedAskQuantity = accumulatedAskQuantity.Add(quantity) + + } + + submitOrders = append(submitOrders, types.SubmitOrder{ + Symbol: s.Symbol, + Type: types.OrderTypeLimitMaker, + Market: s.makerMarket, + Side: side, + Price: depthPrice, + Quantity: quantity, + }) + } + } + + return submitOrders, nil +} + +func (s *Strategy) updateQuote(ctx context.Context) { if err := s.MakerOrderExecutor.GracefulCancel(ctx); err != nil { log.Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) s.MakerOrderExecutor.ActiveMakerOrders().Print() @@ -844,3 +946,22 @@ func selectSessions2( s2 = sessions[n2] return s1, s2, nil } + +func averageDepthPrice(pvs types.PriceVolumeSlice) (price fixedpoint.Value, err error) { + if len(pvs) == 0 { + return fixedpoint.Zero, fmt.Errorf("empty pv slice") + } + + totalQuoteAmount := fixedpoint.Zero + totalQuantity := fixedpoint.Zero + + for i := 0; i < len(pvs); i++ { + pv := pvs[i] + quoteAmount := fixedpoint.Mul(pv.Volume, pv.Price) + totalQuoteAmount = totalQuoteAmount.Add(quoteAmount) + totalQuantity = totalQuantity.Add(pv.Volume) + } + + price = totalQuoteAmount.Div(totalQuantity) + return price, nil +} diff --git a/pkg/strategy/xdepthmaker/strategy_test.go b/pkg/strategy/xdepthmaker/strategy_test.go new file mode 100644 index 0000000000..10ccced40c --- /dev/null +++ b/pkg/strategy/xdepthmaker/strategy_test.go @@ -0,0 +1,72 @@ +package xdepthmaker + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/bbgo" + . "github.com/c9s/bbgo/pkg/testing/testhelper" + "github.com/c9s/bbgo/pkg/types" +) + +func newTestBTCUSDTMarket() types.Market { + return types.Market{ + BaseCurrency: "BTC", + QuoteCurrency: "USDT", + TickSize: Number(0.01), + StepSize: Number(0.000001), + PricePrecision: 2, + VolumePrecision: 8, + MinNotional: Number(8.0), + MinQuantity: Number(0.0003), + } +} + +func TestStrategy_generateMakerOrders(t *testing.T) { + s := &Strategy{ + Symbol: "BTCUSDT", + NumLayers: 3, + DepthScale: &bbgo.LayerScale{ + LayerRule: &bbgo.SlideRule{ + LinearScale: &bbgo.LinearScale{ + Domain: [2]float64{1.0, 3.0}, + Range: [2]float64{1000.0, 15000.0}, + }, + }, + }, + CrossExchangeMarketMakingStrategy: &CrossExchangeMarketMakingStrategy{ + makerMarket: newTestBTCUSDTMarket(), + }, + } + + pricingBook := types.NewStreamBook("BTCUSDT") + pricingBook.OrderBook.Load(types.SliceOrderBook{ + Symbol: "BTCUSDT", + Bids: types.PriceVolumeSlice{ + {Price: Number("25000.00"), Volume: Number("0.1")}, + {Price: Number("24900.00"), Volume: Number("0.2")}, + {Price: Number("24800.00"), Volume: Number("0.3")}, + {Price: Number("24700.00"), Volume: Number("0.4")}, + }, + Asks: types.PriceVolumeSlice{ + {Price: Number("25100.00"), Volume: Number("0.1")}, + {Price: Number("25200.00"), Volume: Number("0.2")}, + {Price: Number("25300.00"), Volume: Number("0.3")}, + {Price: Number("25400.00"), Volume: Number("0.4")}, + }, + Time: time.Now(), + }) + + orders, err := s.generateMakerOrders(pricingBook) + assert.NoError(t, err) + AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ + {Side: types.SideTypeBuy, Price: Number("25000"), Quantity: Number("0.04")}, // =~ $1000.00 + {Side: types.SideTypeBuy, Price: Number("24866.66"), Quantity: Number("0.281715")}, // =~ $7005.3111219, accumulated amount =~ $1000.00 + $7005.3111219 = $8005.3111219 + {Side: types.SideTypeBuy, Price: Number("24800"), Quantity: Number("0.283123")}, // =~ $7021.4504, accumulated amount =~ $1000.00 + $7005.3111219 + $7021.4504 = $8005.3111219 + $7021.4504 =~ $15026.7615219 + {Side: types.SideTypeSell, Price: Number("25100"), Quantity: Number("0.03984")}, + {Side: types.SideTypeSell, Price: Number("25233.33"), Quantity: Number("0.2772")}, + {Side: types.SideTypeSell, Price: Number("25233.33"), Quantity: Number("0.277411")}, + }, orders) +} diff --git a/pkg/testing/testhelper/assert_priceside.go b/pkg/testing/testhelper/assert_priceside.go index 7c45cdd9d8..0d7f74df1d 100644 --- a/pkg/testing/testhelper/assert_priceside.go +++ b/pkg/testing/testhelper/assert_priceside.go @@ -32,7 +32,7 @@ type PriceSideQuantityAssert struct { func AssertOrdersPriceSideQuantity( t *testing.T, asserts []PriceSideQuantityAssert, orders []types.SubmitOrder, ) { - assert.Equalf(t, len(orders), len(asserts), "expecting %d orders", len(asserts)) + assert.Equalf(t, len(asserts), len(orders), "expecting %d orders", len(asserts)) var assertPrices, orderPrices fixedpoint.Slice var assertPricesFloat, orderPricesFloat []float64 diff --git a/pkg/types/price_volume_slice.go b/pkg/types/price_volume_slice.go index fdaaaf7716..1aa1756fcf 100644 --- a/pkg/types/price_volume_slice.go +++ b/pkg/types/price_volume_slice.go @@ -38,7 +38,7 @@ func (slice PriceVolumeSlice) Trim() (pvs PriceVolumeSlice) { } func (slice PriceVolumeSlice) CopyDepth(depth int) PriceVolumeSlice { - if depth > len(slice) { + if depth == 0 || depth > len(slice) { return slice.Copy() } @@ -67,8 +67,23 @@ func (slice PriceVolumeSlice) First() (PriceVolume, bool) { return PriceVolume{}, false } +func (slice PriceVolumeSlice) IndexByQuoteVolumeDepth(requiredQuoteVolume fixedpoint.Value) int { + var totalQuoteVolume = fixedpoint.Zero + for x, pv := range slice { + // this should use float64 multiply + quoteVolume := fixedpoint.Mul(pv.Volume, pv.Price) + totalQuoteVolume = totalQuoteVolume.Add(quoteVolume) + if totalQuoteVolume.Compare(requiredQuoteVolume) >= 0 { + return x + } + } + + // depth not enough + return -1 +} + func (slice PriceVolumeSlice) IndexByVolumeDepth(requiredVolume fixedpoint.Value) int { - var tv fixedpoint.Value = fixedpoint.Zero + var tv = fixedpoint.Zero for x, el := range slice { tv = tv.Add(el.Volume) if tv.Compare(requiredVolume) >= 0 { @@ -76,7 +91,7 @@ func (slice PriceVolumeSlice) IndexByVolumeDepth(requiredVolume fixedpoint.Value } } - // not deep enough + // depth not enough return -1 } From e0e98769021d23e7f1cd63e4e88e3d5fcc31a627 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 13:44:35 +0800 Subject: [PATCH 259/422] improve price hart beat usage --- pkg/strategy/xdepthmaker/strategy.go | 8 +++--- pkg/strategy/xmaker/strategy.go | 16 +++++++++--- pkg/types/price_volume_heartbeat.go | 32 ++++++++++++++++-------- pkg/types/price_volume_heartbeat_test.go | 9 +++---- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index fea67795c8..f5c77ceae7 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -230,7 +230,7 @@ type Strategy struct { hedgeErrorLimiter *rate.Limiter hedgeErrorRateReservation *rate.Reservation - askPriceHeartBeat, bidPriceHeartBeat types.PriceHeartBeat + askPriceHeartBeat, bidPriceHeartBeat *types.PriceHeartBeat lastPrice fixedpoint.Value @@ -318,6 +318,8 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { + s.bidPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) + s.askPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) return nil } @@ -701,14 +703,14 @@ func (s *Strategy) updateQuote(ctx context.Context) { bookLastUpdateTime := s.pricingBook.LastUpdateTime() - if _, err := s.bidPriceHeartBeat.Update(bestBid, priceUpdateTimeout); err != nil { + if _, err := s.bidPriceHeartBeat.Update(bestBid); err != nil { log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) return } - if _, err := s.askPriceHeartBeat.Update(bestAsk, priceUpdateTimeout); err != nil { + if _, err := s.askPriceHeartBeat.Update(bestAsk); err != nil { log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go index b99added28..c8c376a5eb 100644 --- a/pkg/strategy/xmaker/strategy.go +++ b/pkg/strategy/xmaker/strategy.go @@ -112,7 +112,7 @@ type Strategy struct { orderStore *core.OrderStore tradeCollector *core.TradeCollector - askPriceHeartBeat, bidPriceHeartBeat types.PriceHeartBeat + askPriceHeartBeat, bidPriceHeartBeat *types.PriceHeartBeat lastPrice fixedpoint.Value groupID uint32 @@ -170,6 +170,12 @@ func aggregatePrice(pvs types.PriceVolumeSlice, requiredQuantity fixedpoint.Valu return price } +func (s *Strategy) Initialize() error { + s.bidPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) + s.askPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) + return nil +} + func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter) { if err := s.activeMakerOrders.GracefulCancel(ctx, s.makerSession.Exchange); err != nil { log.Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) @@ -191,14 +197,14 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or bookLastUpdateTime := s.book.LastUpdateTime() - if _, err := s.bidPriceHeartBeat.Update(bestBid, priceUpdateTimeout); err != nil { + if _, err := s.bidPriceHeartBeat.Update(bestBid); err != nil { log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) return } - if _, err := s.askPriceHeartBeat.Update(bestAsk, priceUpdateTimeout); err != nil { + if _, err := s.askPriceHeartBeat.Update(bestAsk); err != nil { log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) @@ -639,7 +645,9 @@ func (s *Strategy) Validate() error { return nil } -func (s *Strategy) CrossRun(ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession) error { +func (s *Strategy) CrossRun( + ctx context.Context, orderExecutionRouter bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, +) error { if s.BollBandInterval == "" { s.BollBandInterval = types.Interval1m } diff --git a/pkg/types/price_volume_heartbeat.go b/pkg/types/price_volume_heartbeat.go index 51b4a5bfac..9f249a67a1 100644 --- a/pkg/types/price_volume_heartbeat.go +++ b/pkg/types/price_volume_heartbeat.go @@ -7,24 +7,34 @@ import ( // PriceHeartBeat is used for monitoring the price volume update. type PriceHeartBeat struct { - PriceVolume PriceVolume - LastTime time.Time + last PriceVolume + lastUpdatedTime time.Time + timeout time.Duration +} + +func NewPriceHeartBeat(timeout time.Duration) *PriceHeartBeat { + return &PriceHeartBeat{ + timeout: timeout, + } } // Update updates the price volume object and the last update time // It returns (bool, error), when the price is successfully updated, it returns true. // If the price is not updated (same price) and the last time exceeded the timeout, // Then false, and an error will be returned -func (b *PriceHeartBeat) Update(pv PriceVolume, timeout time.Duration) (bool, error) { - if b.PriceVolume.Price.IsZero() || b.PriceVolume != pv { - b.PriceVolume = pv - b.LastTime = time.Now() +func (b *PriceHeartBeat) Update(current PriceVolume) (bool, error) { + if b.last.Price.IsZero() || b.last != current { + b.last = current + b.lastUpdatedTime = time.Now() return true, nil // successfully updated - } else if time.Since(b.LastTime) > timeout { - return false, fmt.Errorf("price %s has not been updating for %s, last update: %s, skip quoting", - b.PriceVolume.String(), - time.Since(b.LastTime), - b.LastTime) + } else { + // if price and volume is not changed + if b.last.Equals(current) && time.Since(b.lastUpdatedTime) > b.timeout { + return false, fmt.Errorf("price %s has not been updating for %s, last update: %s, skip quoting", + b.last.String(), + time.Since(b.lastUpdatedTime), + b.lastUpdatedTime) + } } return false, nil diff --git a/pkg/types/price_volume_heartbeat_test.go b/pkg/types/price_volume_heartbeat_test.go index 16d0c28784..c443a46042 100644 --- a/pkg/types/price_volume_heartbeat_test.go +++ b/pkg/types/price_volume_heartbeat_test.go @@ -2,7 +2,6 @@ package types import ( "testing" - "time" "github.com/stretchr/testify/assert" @@ -11,19 +10,19 @@ import ( func TestPriceHeartBeat_Update(t *testing.T) { hb := PriceHeartBeat{} - updated, err := hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(22.0), Volume: fixedpoint.NewFromFloat(100.0)}, time.Minute) + updated, err := hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(22.0), Volume: fixedpoint.NewFromFloat(100.0)}) assert.NoError(t, err) assert.True(t, updated) - updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(22.0), Volume: fixedpoint.NewFromFloat(100.0)}, time.Minute) + updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(22.0), Volume: fixedpoint.NewFromFloat(100.0)}) assert.NoError(t, err) assert.False(t, updated, "should not be updated when pv is not changed") - updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(23.0), Volume: fixedpoint.NewFromFloat(100.0)}, time.Minute) + updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(23.0), Volume: fixedpoint.NewFromFloat(100.0)}) assert.NoError(t, err) assert.True(t, updated, "should be updated when the price is changed") - updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(23.0), Volume: fixedpoint.NewFromFloat(200.0)}, time.Minute) + updated, err = hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(23.0), Volume: fixedpoint.NewFromFloat(200.0)}) assert.NoError(t, err) assert.True(t, updated, "should be updated when the volume is changed") } From 2f1a700b892f2a62d33eff35174a7f0a3fa8ba49 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 13:53:22 +0800 Subject: [PATCH 260/422] remove xpuremaker --- pkg/cmd/strategy/builtin.go | 1 - pkg/strategy/xpuremaker/strategy.go | 197 ---------------------------- 2 files changed, 198 deletions(-) delete mode 100644 pkg/strategy/xpuremaker/strategy.go diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 867c72dc2a..937292a84b 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -50,5 +50,4 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/xgap" _ "github.com/c9s/bbgo/pkg/strategy/xmaker" _ "github.com/c9s/bbgo/pkg/strategy/xnav" - _ "github.com/c9s/bbgo/pkg/strategy/xpuremaker" ) diff --git a/pkg/strategy/xpuremaker/strategy.go b/pkg/strategy/xpuremaker/strategy.go deleted file mode 100644 index a5d441bdc1..0000000000 --- a/pkg/strategy/xpuremaker/strategy.go +++ /dev/null @@ -1,197 +0,0 @@ -package xpuremaker - -import ( - "context" - "math" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/c9s/bbgo/pkg/bbgo" - "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/types" -) - -const ID = "xpuremaker" - -var Ten = fixedpoint.NewFromInt(10) - -func init() { - bbgo.RegisterStrategy(ID, &Strategy{}) -} - -type Strategy struct { - Symbol string `json:"symbol"` - Side string `json:"side"` - NumOrders int `json:"numOrders"` - BehindVolume fixedpoint.Value `json:"behindVolume"` - PriceTick fixedpoint.Value `json:"priceTick"` - BaseQuantity fixedpoint.Value `json:"baseQuantity"` - BuySellRatio float64 `json:"buySellRatio"` - - book *types.StreamOrderBook - activeOrders map[string]types.Order -} - -func (s *Strategy) ID() string { - return ID -} - -func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - session.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{}) -} - -func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - - s.book = types.NewStreamBook(s.Symbol) - s.book.BindStream(session.UserDataStream) - - s.activeOrders = make(map[string]types.Order) - - // We can move the go routine to the parent level. - go func() { - ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() - - s.update(orderExecutor, session) - - for { - select { - case <-ctx.Done(): - return - - case <-s.book.C: - s.update(orderExecutor, session) - - case <-ticker.C: - s.update(orderExecutor, session) - } - } - }() - - return nil -} - -func (s *Strategy) cancelOrders(session *bbgo.ExchangeSession) { - var deletedIDs []string - for clientOrderID, o := range s.activeOrders { - log.Infof("canceling order: %+v", o) - - if err := session.Exchange.CancelOrders(context.Background(), o); err != nil { - log.WithError(err).Error("cancel order error") - continue - } - - deletedIDs = append(deletedIDs, clientOrderID) - } - s.book.C.Drain(1*time.Second, 3*time.Second) - - for _, id := range deletedIDs { - delete(s.activeOrders, id) - } -} - -func (s *Strategy) update(orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) { - s.cancelOrders(session) - - switch s.Side { - case "buy": - s.updateOrders(orderExecutor, session, types.SideTypeBuy) - case "sell": - s.updateOrders(orderExecutor, session, types.SideTypeSell) - case "both": - s.updateOrders(orderExecutor, session, types.SideTypeBuy) - s.updateOrders(orderExecutor, session, types.SideTypeSell) - - default: - log.Panicf("undefined side: %s", s.Side) - } - - s.book.C.Drain(1*time.Second, 3*time.Second) -} - -func (s *Strategy) updateOrders(orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession, side types.SideType) { - var book = s.book.Copy() - var pvs = book.SideBook(side) - if len(pvs) == 0 { - log.Warnf("empty side: %s", side) - return - } - - log.Infof("placing order behind volume: %f", s.BehindVolume.Float64()) - - idx := pvs.IndexByVolumeDepth(s.BehindVolume) - if idx == -1 || idx > len(pvs)-1 { - // do not place orders - log.Warn("depth is not enough") - return - } - - var depthPrice = pvs[idx].Price - var orders = s.generateOrders(s.Symbol, side, depthPrice, s.PriceTick, s.BaseQuantity, s.NumOrders) - if len(orders) == 0 { - log.Warn("empty orders") - return - } - - createdOrders, err := orderExecutor.SubmitOrders(context.Background(), orders...) - if err != nil { - log.WithError(err).Errorf("order submit error") - return - } - - // add created orders to the list - for i, o := range createdOrders { - s.activeOrders[o.ClientOrderID] = createdOrders[i] - } -} - -func (s *Strategy) generateOrders(symbol string, side types.SideType, price, priceTick, baseQuantity fixedpoint.Value, numOrders int) (orders []types.SubmitOrder) { - var expBase = fixedpoint.Zero - - switch side { - case types.SideTypeBuy: - if priceTick.Sign() > 0 { - priceTick = priceTick.Neg() - } - - case types.SideTypeSell: - if priceTick.Sign() < 0 { - priceTick = priceTick.Neg() - } - } - - decdigits := priceTick.Abs().NumIntDigits() - step := priceTick.Abs().MulExp(-decdigits + 1) - - for i := 0; i < numOrders; i++ { - quantityExp := fixedpoint.NewFromFloat(math.Exp(expBase.Float64())) - volume := baseQuantity.Mul(quantityExp) - amount := volume.Mul(price) - // skip order less than 10usd - if amount.Compare(Ten) < 0 { - log.Warnf("amount too small (< 10usd). price=%s volume=%s amount=%s", - price.String(), volume.String(), amount.String()) - continue - } - - orders = append(orders, types.SubmitOrder{ - Symbol: symbol, - Side: side, - Type: types.OrderTypeLimit, - Price: price, - Quantity: volume, - }) - - log.Infof("%s order: %s @ %s", side, volume.String(), price.String()) - - if len(orders) >= numOrders { - break - } - - price = price.Add(priceTick) - expBase = expBase.Add(step) - } - - return orders -} From a82bc8645561681d341d4bab54b876429d310f75 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 13:55:46 +0800 Subject: [PATCH 261/422] xdepthmaker: update updateQuote method --- pkg/strategy/xdepthmaker/strategy.go | 212 +-------------------------- 1 file changed, 8 insertions(+), 204 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index f5c77ceae7..4b5ebe1970 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -589,7 +589,6 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook) ([]ty bestBidPrice := bestBid.Price bestAskPrice := bestAsk.Price - log.Infof("%s book ticker: best ask / best bid = %v / %v", s.Symbol, bestAskPrice, bestBidPrice) lastMidPrice := bestBidPrice.Add(bestAskPrice).Div(Two) _ = lastMidPrice @@ -698,8 +697,11 @@ func (s *Strategy) updateQuote(ctx context.Context) { return } - // use mid-price for the last price - s.lastPrice = bestBid.Price.Add(bestAsk.Price).Div(Two) + bestBidPrice := bestBid.Price + bestAskPrice := bestAsk.Price + log.Infof("%s book ticker: best ask / best bid = %v / %v", s.Symbol, bestAskPrice, bestBidPrice) + + s.lastPrice = bestBidPrice.Add(bestAskPrice).Div(Two) bookLastUpdateTime := s.pricingBook.LastUpdateTime() @@ -717,210 +719,12 @@ func (s *Strategy) updateQuote(ctx context.Context) { return } - sourceBook := s.pricingBook.CopyDepth(10) - if valid, err := sourceBook.IsValid(); !valid { - log.WithError(err).Errorf("%s invalid copied order book, skip quoting: %v", s.Symbol, err) - return - } - - var disableMakerBid = false - var disableMakerAsk = false - - // check maker's balance quota - // we load the balances from the account while we're generating the orders, - // the balance may have a chance to be deducted by other strategies or manual orders submitted by the user - makerBalances := s.makerSession.GetAccount().Balances() - makerQuota := &bbgo.QuotaTransaction{} - if b, ok := makerBalances[s.makerMarket.BaseCurrency]; ok { - if b.Available.Compare(s.makerMarket.MinQuantity) > 0 { - makerQuota.BaseAsset.Add(b.Available) - } else { - disableMakerAsk = true - } - } - - if b, ok := makerBalances[s.makerMarket.QuoteCurrency]; ok { - if b.Available.Compare(s.makerMarket.MinNotional) > 0 { - makerQuota.QuoteAsset.Add(b.Available) - } else { - disableMakerBid = true - } - } - - hedgeBalances := s.hedgeSession.GetAccount().Balances() - hedgeQuota := &bbgo.QuotaTransaction{} - if b, ok := hedgeBalances[s.hedgeMarket.BaseCurrency]; ok { - // to make bid orders, we need enough base asset in the foreign exchange, - // if the base asset balance is not enough for selling - if s.StopHedgeBaseBalance.Sign() > 0 { - minAvailable := s.StopHedgeBaseBalance.Add(s.hedgeMarket.MinQuantity) - if b.Available.Compare(minAvailable) > 0 { - hedgeQuota.BaseAsset.Add(b.Available.Sub(minAvailable)) - } else { - log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) - disableMakerBid = true - } - } else if b.Available.Compare(s.hedgeMarket.MinQuantity) > 0 { - hedgeQuota.BaseAsset.Add(b.Available) - } else { - log.Warnf("%s maker bid disabled: insufficient base balance %s", s.Symbol, b.String()) - disableMakerBid = true - } - } - - if b, ok := hedgeBalances[s.hedgeMarket.QuoteCurrency]; ok { - // to make ask orders, we need enough quote asset in the foreign exchange, - // if the quote asset balance is not enough for buying - if s.StopHedgeQuoteBalance.Sign() > 0 { - minAvailable := s.StopHedgeQuoteBalance.Add(s.hedgeMarket.MinNotional) - if b.Available.Compare(minAvailable) > 0 { - hedgeQuota.QuoteAsset.Add(b.Available.Sub(minAvailable)) - } else { - log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) - disableMakerAsk = true - } - } else if b.Available.Compare(s.hedgeMarket.MinNotional) > 0 { - hedgeQuota.QuoteAsset.Add(b.Available) - } else { - log.Warnf("%s maker ask disabled: insufficient quote balance %s", s.Symbol, b.String()) - disableMakerAsk = true - } - } - - // if max exposure position is configured, we should not: - // 1. place bid orders when we already bought too much - // 2. place ask orders when we already sold too much - if s.MaxExposurePosition.Sign() > 0 { - pos := s.Position.GetBase() - - if pos.Compare(s.MaxExposurePosition.Neg()) > 0 { - // stop sell if we over-sell - disableMakerAsk = true - } else if pos.Compare(s.MaxExposurePosition) > 0 { - // stop buy if we over buy - disableMakerBid = true - } - } - - if disableMakerAsk && disableMakerBid { - log.Warnf("%s bid/ask maker is disabled due to insufficient balances", s.Symbol) + submitOrders, err := s.generateMakerOrders(s.pricingBook) + if err != nil { + log.WithError(err).Errorf("generate order error") return } - bestBidPrice := bestBid.Price - bestAskPrice := bestAsk.Price - log.Infof("%s book ticker: best ask / best bid = %v / %v", s.Symbol, bestAskPrice, bestBidPrice) - - var submitOrders []types.SubmitOrder - var accumulativeBidQuantity, accumulativeAskQuantity fixedpoint.Value - var bidQuantity = s.Quantity - var askQuantity = s.Quantity - var bidMargin = s.BidMargin - var askMargin = s.AskMargin - var pips = s.Pips - - bidPrice := bestBidPrice - askPrice := bestAskPrice - for i := 0; i < s.NumLayers; i++ { - // for maker bid orders - if !disableMakerBid { - if s.QuantityScale != nil { - qf, err := s.QuantityScale.Scale(i + 1) - if err != nil { - log.WithError(err).Errorf("quantityScale error") - return - } - - log.Infof("%s scaling bid #%d quantity to %f", s.Symbol, i+1, qf) - - // override the default bid quantity - bidQuantity = fixedpoint.NewFromFloat(qf) - } - - accumulativeBidQuantity = accumulativeBidQuantity.Add(bidQuantity) - if s.UseDepthPrice { - if s.DepthQuantity.Sign() > 0 { - bidPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeBuy), s.DepthQuantity) - } else { - bidPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeBuy), accumulativeBidQuantity) - } - } - - bidPrice = bidPrice.Mul(fixedpoint.One.Sub(bidMargin)) - if i > 0 && pips.Sign() > 0 { - bidPrice = bidPrice.Sub(pips.Mul(fixedpoint.NewFromInt(int64(i)). - Mul(s.makerMarket.TickSize))) - } - - if makerQuota.QuoteAsset.Lock(bidQuantity.Mul(bidPrice)) && hedgeQuota.BaseAsset.Lock(bidQuantity) { - // if we bought, then we need to sell the base from the hedge session - submitOrders = append(submitOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Type: types.OrderTypeLimit, - Side: types.SideTypeBuy, - Price: bidPrice, - Quantity: bidQuantity, - TimeInForce: types.TimeInForceGTC, - }) - - makerQuota.Commit() - hedgeQuota.Commit() - } else { - makerQuota.Rollback() - hedgeQuota.Rollback() - } - } - - // for maker ask orders - if !disableMakerAsk { - if s.QuantityScale != nil { - qf, err := s.QuantityScale.Scale(i + 1) - if err != nil { - log.WithError(err).Errorf("quantityScale error") - return - } - - log.Infof("%s scaling ask #%d quantity to %f", s.Symbol, i+1, qf) - - // override the default bid quantity - askQuantity = fixedpoint.NewFromFloat(qf) - } - accumulativeAskQuantity = accumulativeAskQuantity.Add(askQuantity) - - if s.UseDepthPrice { - if s.DepthQuantity.Sign() > 0 { - askPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeSell), s.DepthQuantity) - } else { - askPrice = aggregatePrice(sourceBook.SideBook(types.SideTypeSell), accumulativeAskQuantity) - } - } - - askPrice = askPrice.Mul(fixedpoint.One.Add(askMargin)) - if i > 0 && pips.Sign() > 0 { - askPrice = askPrice.Add(pips.Mul(fixedpoint.NewFromInt(int64(i)).Mul(s.makerMarket.TickSize))) - } - - if makerQuota.BaseAsset.Lock(askQuantity) && hedgeQuota.QuoteAsset.Lock(askQuantity.Mul(askPrice)) { - // if we bought, then we need to sell the base from the hedge session - submitOrders = append(submitOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Market: s.makerMarket, - Type: types.OrderTypeLimit, - Side: types.SideTypeSell, - Price: askPrice, - Quantity: askQuantity, - TimeInForce: types.TimeInForceGTC, - }) - makerQuota.Commit() - hedgeQuota.Commit() - } else { - makerQuota.Rollback() - hedgeQuota.Rollback() - } - - } - } - if len(submitOrders) == 0 { log.Warnf("no orders are generated") return From c2c1eca4c9f9254e9d9ffd297057f0a070b23104 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:09:07 +0800 Subject: [PATCH 262/422] types: fix price heart beat alert tests --- pkg/types/price_volume_heartbeat_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/types/price_volume_heartbeat_test.go b/pkg/types/price_volume_heartbeat_test.go index c443a46042..470fdb9a0d 100644 --- a/pkg/types/price_volume_heartbeat_test.go +++ b/pkg/types/price_volume_heartbeat_test.go @@ -2,6 +2,7 @@ package types import ( "testing" + "time" "github.com/stretchr/testify/assert" @@ -9,7 +10,8 @@ import ( ) func TestPriceHeartBeat_Update(t *testing.T) { - hb := PriceHeartBeat{} + hb := NewPriceHeartBeat(time.Minute) + updated, err := hb.Update(PriceVolume{Price: fixedpoint.NewFromFloat(22.0), Volume: fixedpoint.NewFromFloat(100.0)}) assert.NoError(t, err) assert.True(t, updated) From 96f6f9e0d05a46349b642b2eb19df81e5ce61b01 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:09:25 +0800 Subject: [PATCH 263/422] exchange/retry: add QueryOrderUntilCancelled --- pkg/exchange/retry/order.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index 66a690a542..f06f682c34 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -3,6 +3,7 @@ package retry import ( "context" "errors" + "fmt" "strconv" "github.com/cenkalti/backoff/v4" @@ -16,6 +17,34 @@ type advancedOrderCancelService interface { CancelOrdersByGroupID(ctx context.Context, groupID uint32) ([]types.Order, error) } +func QueryOrderUntilCanceled( + ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64, +) (o *types.Order, err error) { + var op = func() (err2 error) { + o, err2 = queryOrderService.QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(orderId, 10), + }) + + if err2 != nil { + return err2 + } + + if o == nil { + return fmt.Errorf("order #%d response is nil", orderId) + } + + if o.Status == types.OrderStatusCanceled || o.Status == types.OrderStatusFilled { + return nil + } + + return fmt.Errorf("order #%d is not canceled yet: %s", o.OrderID, o.Status) + } + + err = GeneralBackoff(ctx, op) + return o, err +} + func QueryOrderUntilFilled( ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64, ) (o *types.Order, err error) { From f21170aa5d0191b9f2e8fdc3583a53e854802187 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:09:54 +0800 Subject: [PATCH 264/422] types: add order sorting by price --- pkg/types/sort.go | 18 +++++++++++++++ pkg/types/sort_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/pkg/types/sort.go b/pkg/types/sort.go index 9c0c606130..31840cc5b3 100644 --- a/pkg/types/sort.go +++ b/pkg/types/sort.go @@ -20,6 +20,24 @@ func SortOrdersAscending(orders []Order) []Order { return orders } +// SortOrdersByPrice sorts by creation time ascending-ly +func SortOrdersByPrice(orders []Order, descending bool) []Order { + var f func(i, j int) bool + + if descending { + f = func(i, j int) bool { + return orders[i].Price.Compare(orders[j].Price) > 0 + } + } else { + f = func(i, j int) bool { + return orders[i].Price.Compare(orders[j].Price) < 0 + } + } + + sort.Slice(orders, f) + return orders +} + // SortOrdersAscending sorts by update time ascending-ly func SortOrdersUpdateTimeAscending(orders []Order) []Order { sort.Slice(orders, func(i, j int) bool { diff --git a/pkg/types/sort_test.go b/pkg/types/sort_test.go index 4e5171acea..a93dc9a2e9 100644 --- a/pkg/types/sort_test.go +++ b/pkg/types/sort_test.go @@ -5,6 +5,8 @@ import ( "time" "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" ) func TestSortTradesAscending(t *testing.T) { @@ -29,3 +31,52 @@ func TestSortTradesAscending(t *testing.T) { trades = SortTradesAscending(trades) assert.True(t, trades[0].Time.Before(trades[1].Time.Time())) } + +func getOrderPrices(orders []Order) (prices fixedpoint.Slice) { + for _, o := range orders { + prices = append(prices, o.Price) + } + + return prices +} + +func TestSortOrdersByPrice(t *testing.T) { + + t.Run("ascending", func(t *testing.T) { + orders := []Order{ + {SubmitOrder: SubmitOrder{Price: number("10.0")}}, + {SubmitOrder: SubmitOrder{Price: number("30.0")}}, + {SubmitOrder: SubmitOrder{Price: number("20.0")}}, + {SubmitOrder: SubmitOrder{Price: number("25.0")}}, + {SubmitOrder: SubmitOrder{Price: number("15.0")}}, + } + orders = SortOrdersByPrice(orders, false) + prices := getOrderPrices(orders) + assert.Equal(t, fixedpoint.Slice{ + number(10.0), + number(15.0), + number(20.0), + number(25.0), + number(30.0), + }, prices) + }) + + t.Run("descending", func(t *testing.T) { + orders := []Order{ + {SubmitOrder: SubmitOrder{Price: number("10.0")}}, + {SubmitOrder: SubmitOrder{Price: number("30.0")}}, + {SubmitOrder: SubmitOrder{Price: number("20.0")}}, + {SubmitOrder: SubmitOrder{Price: number("25.0")}}, + {SubmitOrder: SubmitOrder{Price: number("15.0")}}, + } + orders = SortOrdersByPrice(orders, true) + prices := getOrderPrices(orders) + assert.Equal(t, fixedpoint.Slice{ + number(30.0), + number(25.0), + number(20.0), + number(15.0), + number(10.0), + }, prices) + }) +} From 888a550c80a5d95177106f9e8bbec57bc79ba11b Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:10:09 +0800 Subject: [PATCH 265/422] xdepthmaker: support partial maker order replenish --- pkg/strategy/xdepthmaker/strategy.go | 53 +++++++++++++++++++---- pkg/strategy/xdepthmaker/strategy_test.go | 2 +- pkg/types/ordermap.go | 13 ++++++ pkg/types/price_volume_heartbeat.go | 4 ++ pkg/types/price_volume_slice.go | 20 +++++++++ 5 files changed, 82 insertions(+), 10 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 4b5ebe1970..9169faadf5 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -2,6 +2,7 @@ package xdepthmaker import ( "context" + stderrors "errors" "fmt" "sync" "time" @@ -381,8 +382,10 @@ func (s *Strategy) CrossRun( switch sig.Type { case types.BookSignalSnapshot: - case types.BookSignalUpdate: + s.updateQuote(ctx, 0) + case types.BookSignalUpdate: + s.updateQuote(ctx, 5) } case <-posTicker.C: @@ -581,7 +584,7 @@ func (s *Strategy) runTradeRecover(ctx context.Context) { } } -func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook) ([]types.SubmitOrder, error) { +func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLayer int) ([]types.SubmitOrder, error) { bestBid, bestAsk, hasPrice := pricingBook.BestBidAndAsk() if !hasPrice { return nil, nil @@ -600,8 +603,12 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook) ([]ty dupPricingBook := pricingBook.CopyDepth(0) + if maxLayer == 0 || maxLayer > s.NumLayers { + maxLayer = s.NumLayers + } + for _, side := range []types.SideType{types.SideTypeBuy, types.SideTypeSell} { - for i := 1; i <= s.NumLayers; i++ { + for i := 1; i <= maxLayer; i++ { requiredDepthFloat, err := s.DepthScale.Scale(i) if err != nil { return nil, errors.Wrapf(err, "depthScale scale error") @@ -679,11 +686,31 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook) ([]ty return submitOrders, nil } -func (s *Strategy) updateQuote(ctx context.Context) { - if err := s.MakerOrderExecutor.GracefulCancel(ctx); err != nil { - log.Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) - s.MakerOrderExecutor.ActiveMakerOrders().Print() - return +func (s *Strategy) partiallyCancelOrders(ctx context.Context, maxLayer int) error { + buyOrders, sellOrders := s.MakerOrderExecutor.ActiveMakerOrders().Orders().SeparateBySide() + buyOrders = types.SortOrdersByPrice(buyOrders, true) + sellOrders = types.SortOrdersByPrice(sellOrders, false) + + buyOrdersToCancel := buyOrders[0:min(maxLayer, len(buyOrders))] + sellOrdersToCancel := sellOrders[0:min(maxLayer, len(sellOrders))] + + err1 := s.MakerOrderExecutor.GracefulCancel(ctx, buyOrdersToCancel...) + err2 := s.MakerOrderExecutor.GracefulCancel(ctx, sellOrdersToCancel...) + return stderrors.Join(err1, err2) +} + +func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { + if maxLayer == 0 { + if err := s.MakerOrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Warnf("there are some %s orders not canceled, skipping placing maker orders", s.Symbol) + s.MakerOrderExecutor.ActiveMakerOrders().Print() + return + } + } else { + if err := s.partiallyCancelOrders(ctx, maxLayer); err != nil { + log.WithError(err).Warnf("%s partial order cancel failed", s.Symbol) + return + } } numOfMakerOrders := s.MakerOrderExecutor.ActiveMakerOrders().NumOfOrders() @@ -719,7 +746,7 @@ func (s *Strategy) updateQuote(ctx context.Context) { return } - submitOrders, err := s.generateMakerOrders(s.pricingBook) + submitOrders, err := s.generateMakerOrders(s.pricingBook, maxLayer) if err != nil { log.WithError(err).Errorf("generate order error") return @@ -771,3 +798,11 @@ func averageDepthPrice(pvs types.PriceVolumeSlice) (price fixedpoint.Value, err price = totalQuoteAmount.Div(totalQuantity) return price, nil } + +func min(a, b int) int { + if a < b { + return a + } + + return b +} diff --git a/pkg/strategy/xdepthmaker/strategy_test.go b/pkg/strategy/xdepthmaker/strategy_test.go index 10ccced40c..5f40af7f63 100644 --- a/pkg/strategy/xdepthmaker/strategy_test.go +++ b/pkg/strategy/xdepthmaker/strategy_test.go @@ -59,7 +59,7 @@ func TestStrategy_generateMakerOrders(t *testing.T) { Time: time.Now(), }) - orders, err := s.generateMakerOrders(pricingBook) + orders, err := s.generateMakerOrders(pricingBook, 0) assert.NoError(t, err) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ {Side: types.SideTypeBuy, Price: Number("25000"), Quantity: Number("0.04")}, // =~ $1000.00 diff --git a/pkg/types/ordermap.go b/pkg/types/ordermap.go index e7cfca1a4d..a086d12ed8 100644 --- a/pkg/types/ordermap.go +++ b/pkg/types/ordermap.go @@ -243,3 +243,16 @@ func (m *SyncOrderMap) Orders() (slice OrderSlice) { } type OrderSlice []Order + +func (s OrderSlice) SeparateBySide() (buyOrders, sellOrders []Order) { + for _, o := range s { + switch o.Side { + case SideTypeBuy: + buyOrders = append(buyOrders, o) + case SideTypeSell: + sellOrders = append(sellOrders, o) + } + } + + return buyOrders, sellOrders +} diff --git a/pkg/types/price_volume_heartbeat.go b/pkg/types/price_volume_heartbeat.go index 9f249a67a1..17c3ed55db 100644 --- a/pkg/types/price_volume_heartbeat.go +++ b/pkg/types/price_volume_heartbeat.go @@ -18,6 +18,10 @@ func NewPriceHeartBeat(timeout time.Duration) *PriceHeartBeat { } } +func (b *PriceHeartBeat) Last() PriceVolume { + return b.last +} + // Update updates the price volume object and the last update time // It returns (bool, error), when the price is successfully updated, it returns true. // If the price is not updated (same price) and the last time exceeded the timeout, diff --git a/pkg/types/price_volume_slice.go b/pkg/types/price_volume_slice.go index 1aa1756fcf..53fb7d9136 100644 --- a/pkg/types/price_volume_slice.go +++ b/pkg/types/price_volume_slice.go @@ -82,6 +82,26 @@ func (slice PriceVolumeSlice) IndexByQuoteVolumeDepth(requiredQuoteVolume fixedp return -1 } +func (slice PriceVolumeSlice) SumDepth() fixedpoint.Value { + var total = fixedpoint.Zero + for _, pv := range slice { + total = total.Add(pv.Volume) + } + + return total +} + +func (slice PriceVolumeSlice) SumDepthInQuote() fixedpoint.Value { + var total = fixedpoint.Zero + + for _, pv := range slice { + quoteVolume := fixedpoint.Mul(pv.Price, pv.Volume) + total = total.Add(quoteVolume) + } + + return total +} + func (slice PriceVolumeSlice) IndexByVolumeDepth(requiredVolume fixedpoint.Value) int { var tv = fixedpoint.Zero for x, el := range slice { From 25b04cb36cd991bfc7a84791e5ed67eef1d8ea9d Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:11:21 +0800 Subject: [PATCH 266/422] xdepthmaker: add fullReplenishTicker --- pkg/strategy/xdepthmaker/strategy.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 9169faadf5..6f9497db5a 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -363,6 +363,9 @@ func (s *Strategy) CrossRun( posTicker := time.NewTicker(util.MillisecondsJitter(s.HedgeInterval.Duration(), 200)) defer posTicker.Stop() + fullReplenishTicker := time.NewTicker(util.MillisecondsJitter(15*time.Minute, 200)) + defer fullReplenishTicker.Stop() + for { select { @@ -374,6 +377,9 @@ func (s *Strategy) CrossRun( log.Warnf("%s maker goroutine stopped, due to the cancelled context", s.Symbol) return + case <-fullReplenishTicker.C: + s.updateQuote(ctx, 0) + case sig, ok := <-s.pricingBook.C: // when any book change event happened if !ok { From d14527b5cfdba6054ec5ed825025acb3f0366e2b Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 30 Nov 2023 17:18:05 +0800 Subject: [PATCH 267/422] xdepthmaker: apply FullReplenishInterval from config --- pkg/strategy/xdepthmaker/strategy.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 6f9497db5a..81896ead15 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -184,15 +184,16 @@ type Strategy struct { // MakerExchange session name MakerExchange string `json:"makerExchange"` - UpdateInterval types.Duration `json:"updateInterval"` - HedgeInterval types.Duration `json:"hedgeInterval"` + UpdateInterval types.Duration `json:"updateInterval"` + HedgeInterval types.Duration `json:"hedgeInterval"` + + FullReplenishInterval types.Duration `json:"fullReplenishInterval"` + OrderCancelWaitTime types.Duration `json:"orderCancelWaitTime"` - Margin fixedpoint.Value `json:"margin"` - BidMargin fixedpoint.Value `json:"bidMargin"` - AskMargin fixedpoint.Value `json:"askMargin"` - UseDepthPrice bool `json:"useDepthPrice"` - DepthQuantity fixedpoint.Value `json:"depthQuantity"` + Margin fixedpoint.Value `json:"margin"` + BidMargin fixedpoint.Value `json:"bidMargin"` + AskMargin fixedpoint.Value `json:"askMargin"` StopHedgeQuoteBalance fixedpoint.Value `json:"stopHedgeQuoteBalance"` StopHedgeBaseBalance fixedpoint.Value `json:"stopHedgeBaseBalance"` @@ -286,6 +287,10 @@ func (s *Strategy) Defaults() error { s.UpdateInterval = types.Duration(time.Second) } + if s.FullReplenishInterval == 0 { + s.FullReplenishInterval = types.Duration(15 * time.Minute) + } + if s.HedgeInterval == 0 { s.HedgeInterval = types.Duration(3 * time.Second) } @@ -363,7 +368,7 @@ func (s *Strategy) CrossRun( posTicker := time.NewTicker(util.MillisecondsJitter(s.HedgeInterval.Duration(), 200)) defer posTicker.Stop() - fullReplenishTicker := time.NewTicker(util.MillisecondsJitter(15*time.Minute, 200)) + fullReplenishTicker := time.NewTicker(util.MillisecondsJitter(s.FullReplenishInterval.Duration(), 200)) defer fullReplenishTicker.Stop() for { From f03ac52ce5351e49aefb538784daa9cd50c5a882 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 6 Dec 2023 17:55:25 +0800 Subject: [PATCH 268/422] activeOrderBook: use orderMap instead of orderStore --- pkg/bbgo/activeorderbook.go | 6 ++---- pkg/types/ordermap.go | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index ee1cdf0567..b65ae47774 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -10,7 +10,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/c9s/bbgo/pkg/core" "github.com/c9s/bbgo/pkg/sigchan" "github.com/c9s/bbgo/pkg/types" ) @@ -218,11 +217,10 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, continue } - openOrderStore := core.NewOrderStore(symbol) - openOrderStore.Add(openOrders...) + orderMap := types.NewOrderMap(openOrders...) for _, o := range orders { // if it's not on the order book (open orders), we should remove it from our local side - if !openOrderStore.Exists(o.OrderID) { + if !orderMap.Exists(o.OrderID) { b.Remove(o) } else { leftOrders = append(leftOrders, o) diff --git a/pkg/types/ordermap.go b/pkg/types/ordermap.go index a086d12ed8..c091d6c418 100644 --- a/pkg/types/ordermap.go +++ b/pkg/types/ordermap.go @@ -8,6 +8,14 @@ import ( // OrderMap is used for storing orders by their order id type OrderMap map[uint64]Order +func NewOrderMap(os ...Order) OrderMap { + m := OrderMap{} + if len(os) > 0 { + m.Add(os...) + } + return m +} + func (m OrderMap) Backup() (orderForms []SubmitOrder) { for _, order := range m { orderForms = append(orderForms, order.Backup()) @@ -17,8 +25,10 @@ func (m OrderMap) Backup() (orderForms []SubmitOrder) { } // Add the order the the map -func (m OrderMap) Add(o Order) { - m[o.OrderID] = o +func (m OrderMap) Add(os ...Order) { + for _, o := range os { + m[o.OrderID] = o + } } // Update only updates the order when the order ID exists in the map From b8fb2ac478933b34eeaf6b8bdbb36d96d4852483 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 6 Dec 2023 18:01:30 +0800 Subject: [PATCH 269/422] bbgo: fix active orderbook symbol order grouping --- pkg/bbgo/activeorderbook.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index b65ae47774..7c9a9a2dc6 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -178,7 +178,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, waitTime := CancelOrderWaitTime startTime := time.Now() - // ensure every order is cancelled + // ensure every order is canceled for { // Some orders in the variable are not created on the server side yet, // If we cancel these orders directly, we will get an unsent order error @@ -203,14 +203,18 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, // verify the current open orders via the RESTful API log.Warnf("[ActiveOrderBook] using REStful API to verify active orders...") - var symbols = map[string]struct{}{} + var symbolOrdersMap = map[string]types.OrderSlice{} for _, order := range orders { - symbols[order.Symbol] = struct{}{} - + symbolOrdersMap[order.Symbol] = append(symbolOrdersMap[order.Symbol], order) } + var leftOrders []types.Order + for symbol := range symbolOrdersMap { + symbolOrders, ok := symbolOrdersMap[symbol] + if !ok { + continue + } - for symbol := range symbols { openOrders, err := ex.QueryOpenOrders(ctx, symbol) if err != nil { log.WithError(err).Errorf("can not query %s open orders", symbol) @@ -218,7 +222,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, } orderMap := types.NewOrderMap(openOrders...) - for _, o := range orders { + for _, o := range symbolOrders { // if it's not on the order book (open orders), we should remove it from our local side if !orderMap.Exists(o.OrderID) { b.Remove(o) @@ -228,6 +232,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, } } + // update order slice for the next try orders = leftOrders } From 35dabe8a72a908ffd00dc509057a2da4fa231b77 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 6 Dec 2023 18:05:33 +0800 Subject: [PATCH 270/422] xdepthmaker: fix aggregatePrice quantity issue --- pkg/strategy/xdepthmaker/aggregate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/strategy/xdepthmaker/aggregate.go b/pkg/strategy/xdepthmaker/aggregate.go index a9ff5298f7..35c1df1d99 100644 --- a/pkg/strategy/xdepthmaker/aggregate.go +++ b/pkg/strategy/xdepthmaker/aggregate.go @@ -27,6 +27,6 @@ func aggregatePrice(pvs types.PriceVolumeSlice, requiredQuantity fixedpoint.Valu totalAmount = totalAmount.Add(pv.Volume.Mul(pv.Price)) } - price = totalAmount.Div(requiredQuantity) + price = totalAmount.Div(requiredQuantity.Sub(q)) return price } From e82605f658088fd20f720cb83242bcf29743c4d4 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 14:38:34 +0800 Subject: [PATCH 271/422] xdepthmaker: skip test for dnum --- pkg/strategy/xdepthmaker/strategy_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/strategy/xdepthmaker/strategy_test.go b/pkg/strategy/xdepthmaker/strategy_test.go index 5f40af7f63..182f0bc8ac 100644 --- a/pkg/strategy/xdepthmaker/strategy_test.go +++ b/pkg/strategy/xdepthmaker/strategy_test.go @@ -1,3 +1,5 @@ +//go:build !dnum + package xdepthmaker import ( From 4b11289c7cfd15d4630550088635eb77556108ac Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 15:14:12 +0800 Subject: [PATCH 272/422] github flow: bump go version to 1.19 --- .github/workflows/go.yml | 4 ++-- .github/workflows/release.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a0e21c855c..5d0cfaedd7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,9 +16,9 @@ jobs: strategy: matrix: redis-version: - - 6.2 + - "6.2" go-version: - - 1.18 + - "1.20" env: MYSQL_DATABASE: bbgo MYSQL_USER: "root" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a23ef5ade7..98def0eb02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.19 - name: Install Node uses: actions/setup-node@v2 with: From 214f9fe75e2621942836756c73e5aa65aa509edb Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 16:31:37 +0800 Subject: [PATCH 273/422] bitget: improve bitget websocket depth subscription --- pkg/exchange/bitget/stream.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 864b5ba244..4d8ec96a86 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -75,7 +75,7 @@ func (s *Stream) syncSubscriptions(opType WsEventType) error { } logger := log.WithField("opType", opType) - args := []WsArg{} + var args []WsArg for _, subscription := range s.Subscriptions { arg, err := convertSubscription(subscription) if err != nil { @@ -244,9 +244,11 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { arg.Channel = ChannelOrderBook5 switch sub.Options.Depth { - case types.DepthLevel15: + case types.DepthLevel5: + arg.Channel = ChannelOrderBook5 + case types.DepthLevel15, types.DepthLevelMedium: arg.Channel = ChannelOrderBook15 - case types.DepthLevel200: + case types.DepthLevel200, types.DepthLevelFull: log.Warn("*** The subscription events for the order book may return fewer than 200 bids/asks at a depth of 200. ***") arg.Channel = ChannelOrderBook } From cd06ffd21feece14b55d846201bb531baaa5965f Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 17:38:39 +0800 Subject: [PATCH 274/422] xdepthmaker: fix order call --- pkg/strategy/xdepthmaker/strategy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 81896ead15..a446a1289a 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -255,7 +255,7 @@ func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { hedgeSession.Subscribe(types.BookChannel, s.Symbol, types.SubscribeOptions{ Depth: types.DepthLevelMedium, - Speed: types.SpeedHigh, + Speed: types.SpeedLow, }) hedgeSession.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: "1m"}) @@ -371,6 +371,8 @@ func (s *Strategy) CrossRun( fullReplenishTicker := time.NewTicker(util.MillisecondsJitter(s.FullReplenishInterval.Duration(), 200)) defer fullReplenishTicker.Stop() + s.updateQuote(ctx, 0) + for { select { From ab3579700f81ed3c017354a965bf8ceb2b141f73 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 17:48:35 +0800 Subject: [PATCH 275/422] builtin: register xdepthmaker --- pkg/cmd/strategy/builtin.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 937292a84b..b2fbcf14f9 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -45,6 +45,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/wall" _ "github.com/c9s/bbgo/pkg/strategy/xalign" _ "github.com/c9s/bbgo/pkg/strategy/xbalance" + _ "github.com/c9s/bbgo/pkg/strategy/xdepthmaker" _ "github.com/c9s/bbgo/pkg/strategy/xfixedmaker" _ "github.com/c9s/bbgo/pkg/strategy/xfunding" _ "github.com/c9s/bbgo/pkg/strategy/xgap" From 3048a13f0b7a4c283b07f67b8caadbaa9c26b6f5 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 8 Dec 2023 00:21:53 +0800 Subject: [PATCH 276/422] xdepthmaker: replace AtomicAdd with Add --- pkg/fixedpoint/dec.go | 4 ++-- pkg/strategy/xdepthmaker/strategy.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/fixedpoint/dec.go b/pkg/fixedpoint/dec.go index 41318f76d2..17c66cac3e 100644 --- a/pkg/fixedpoint/dec.go +++ b/pkg/fixedpoint/dec.go @@ -1197,7 +1197,7 @@ func align(x, y *Value) bool { } yshift = e // check(0 <= yshift && yshift <= 20) - //y.coef = (y.coef + halfpow10[yshift]) / pow10[yshift] + // y.coef = (y.coef + halfpow10[yshift]) / pow10[yshift] y.coef = (y.coef) / pow10[yshift] // check(int(y.exp)+yshift == int(x.exp)) return true @@ -1364,4 +1364,4 @@ func (x Value) Clamp(min, max Value) Value { return max } return x -} \ No newline at end of file +} diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index a446a1289a..eb7d2a9234 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -148,7 +148,8 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // 1) short position -> reduce short position // 2) short position -> increase short position if trade.Exchange == s.hedgeSession.ExchangeName { - s.CoveredPosition.AtomicAdd(c) + // TODO: make this atomic + s.CoveredPosition = s.CoveredPosition.Add(c) } s.ProfitStats.AddTrade(trade) @@ -557,9 +558,9 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { // if the hedge is on sell side, then we should add positive position switch side { case types.SideTypeSell: - s.CoveredPosition.AtomicAdd(quantity) + s.CoveredPosition = s.CoveredPosition.Add(quantity) case types.SideTypeBuy: - s.CoveredPosition.AtomicAdd(quantity.Neg()) + s.CoveredPosition = s.CoveredPosition.Add(quantity.Neg()) } } From 33f0571511183179bda0c98f4157a8bef2eac56c Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 7 Dec 2023 16:07:23 +0800 Subject: [PATCH 277/422] bbgo: fix since time override --- pkg/bbgo/environment.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index ce8b98fb50..e4c67de1a3 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -451,11 +451,13 @@ func (environ *Environment) syncWithUserConfig(ctx context.Context, userConfig * sessions = environ.SelectSessions(selectedSessions...) } - since := time.Now().AddDate(0, -6, 0) + since := defaultSyncSinceTime() if userConfig.Sync.Since != nil { since = userConfig.Sync.Since.Time() } + environ.SetSyncStartTime(since) + syncSymbolMap, restSymbols := categorizeSyncSymbol(userConfig.Sync.Symbols) for _, session := range sessions { syncSymbols := restSymbols @@ -463,7 +465,7 @@ func (environ *Environment) syncWithUserConfig(ctx context.Context, userConfig * syncSymbols = append(syncSymbols, ss...) } - if err := environ.syncSession(ctx, session, syncSymbols...); err != nil { + if err := environ.syncSession(ctx, session, since, syncSymbols...); err != nil { return err } @@ -520,8 +522,9 @@ func (environ *Environment) Sync(ctx context.Context, userConfig ...*Config) err } // the default sync logics + since := defaultSyncSinceTime() for _, session := range environ.sessions { - if err := environ.syncSession(ctx, session); err != nil { + if err := environ.syncSession(ctx, session, since); err != nil { return err } } @@ -616,10 +619,13 @@ func (environ *Environment) SyncSession(ctx context.Context, session *ExchangeSe environ.setSyncing(Syncing) defer environ.setSyncing(SyncDone) - return environ.syncSession(ctx, session, defaultSymbols...) + since := defaultSyncSinceTime() + return environ.syncSession(ctx, session, since, defaultSymbols...) } -func (environ *Environment) syncSession(ctx context.Context, session *ExchangeSession, defaultSymbols ...string) error { +func (environ *Environment) syncSession( + ctx context.Context, session *ExchangeSession, syncStartTime time.Time, defaultSymbols ...string, +) error { symbols, err := session.getSessionSymbols(defaultSymbols...) if err != nil { return err @@ -627,7 +633,7 @@ func (environ *Environment) syncSession(ctx context.Context, session *ExchangeSe log.Infof("syncing symbols %v from session %s", symbols, session.Name) - return environ.SyncService.SyncSessionSymbols(ctx, session.Exchange, environ.syncStartTime, symbols...) + return environ.SyncService.SyncSessionSymbols(ctx, session.Exchange, syncStartTime, symbols...) } func (environ *Environment) ConfigureNotificationSystem(ctx context.Context, userConfig *Config) error { @@ -1014,3 +1020,7 @@ func (session *ExchangeSession) getSessionSymbols(defaultSymbols ...string) ([]s return session.FindPossibleSymbols() } + +func defaultSyncSinceTime() time.Time { + return time.Now().AddDate(0, -6, 0) +} From d9986540d2dc5979b8616a850ad8d5f8439b23dd Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 8 Dec 2023 09:40:17 +0800 Subject: [PATCH 278/422] update readme for requirement --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bfddf9cb06..9ec4117ec3 100644 --- a/README.md +++ b/README.md @@ -139,15 +139,19 @@ the implementation. ## Requirements -Get your exchange API key and secret after you register the accounts (you can choose one or more exchanges): +* Go SDK 1.20 -- MAX: -- Binance: -- OKEx: -- Kucoin: +* Linux / MacOS / Windows (WSL) -This project is maintained and supported by a small group of team. If you would like to support this project, please -register on the exchanges using the provided links with referral codes above. +* Get your exchange API key and secret after you register the accounts (you can choose one or more exchanges): + + - MAX: + - Binance: + - OKEx: + - Kucoin: + + This project is maintained and supported by a small group of team. If you would like to support this project, please + register on the exchanges using the provided links with referral codes above. ## Installation From b9c4002704fa73e68fc6a796b841dfa2603564a2 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 8 Dec 2023 09:45:53 +0800 Subject: [PATCH 279/422] bitget: handle order type limit maker --- pkg/exchange/bitget/convert.go | 6 ++++-- pkg/exchange/bitget/exchange.go | 36 ++++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 0b6a4b9216..94c1b06782 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -274,7 +274,9 @@ func toGlobalOrder(order v2.OrderDetail) (*types.Order, error) { // If the order status is Filled, return the filled base quantity instead of the buy quantity, because a market order on the buy side // cannot execute all. // Otherwise, return zero. -func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoint.Value, orderStatus v2.OrderStatus) (fixedpoint.Value, error) { +func processMarketBuyQuantity( + filledQty, filledPrice, priceAvg, buyQty fixedpoint.Value, orderStatus v2.OrderStatus, +) (fixedpoint.Value, error) { switch orderStatus { case v2.OrderStatusInit, v2.OrderStatusNew, v2.OrderStatusLive, v2.OrderStatusCancelled: return fixedpoint.Zero, nil @@ -302,7 +304,7 @@ func processMarketBuyQuantity(filledQty, filledPrice, priceAvg, buyQty fixedpoin func toLocalOrderType(orderType types.OrderType) (v2.OrderType, error) { switch orderType { - case types.OrderTypeLimit: + case types.OrderTypeLimit, types.OrderTypeLimitMaker: return v2.OrderTypeLimit, nil case types.OrderTypeMarket: diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index ad429acffc..62ab09091a 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -166,7 +166,9 @@ func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[str // // The end time has different limits. 1m, 5m can query for one month,15m can query for 52 days,30m can query for 62 days, // 1H can query for 83 days,4H can query for 240 days,6H can query for 360 days. -func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { +func (e *Exchange) QueryKLines( + ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions, +) ([]types.KLine, error) { req := e.v2client.NewGetKLineRequest().Symbol(symbol) intervalStr, found := toLocalGranularity[interval] if !found { @@ -263,6 +265,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr if err != nil { return nil, err } + req.OrderType(orderType) // set side @@ -270,6 +273,7 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr if err != nil { return nil, err } + req.Side(side) // set quantity @@ -282,30 +286,38 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr } qty = order.Quantity.Mul(ticker.Buy) } + req.Size(order.Market.FormatQuantity(qty)) - // we support only GTC/PostOnly, this is because: - // 1. We support only SPOT trading. + // set TimeInForce + // we only support GTC/PostOnly, because: + // 1. we only support SPOT trading. // 2. The query open/closed order does not include the `force` in SPOT. // If we support FOK/IOC, but you can't query them, that would be unreasonable. // The other case to consider is 'PostOnly', which is a trade-off because we want to support 'xmaker'. if len(order.TimeInForce) != 0 && order.TimeInForce != types.TimeInForceGTC { return nil, fmt.Errorf("time-in-force %s not supported", order.TimeInForce) } - req.Force(v2.OrderForceGTC) + + switch order.Type { + case types.OrderTypeLimitMaker: + req.Force(v2.OrderForcePostOnly) + default: + req.Force(v2.OrderForceGTC) + } + // set price - if order.Type == types.OrderTypeLimit || order.Type == types.OrderTypeLimitMaker { + switch order.Type { + case types.OrderTypeLimit, types.OrderTypeLimitMaker: req.Price(order.Market.FormatPrice(order.Price)) - if order.Type == types.OrderTypeLimitMaker { - req.Force(v2.OrderForcePostOnly) - } } // set client order id if len(order.ClientOrderID) > maxOrderIdLen { return nil, fmt.Errorf("unexpected length of order id, got: %d", len(order.ClientOrderID)) } + if len(order.ClientOrderID) > 0 { req.ClientOrderId(order.ClientOrderID) } @@ -401,7 +413,9 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ // ** Since is inclusive, Until is exclusive. If you use a time range to query, you must provide both a start time and an end time. ** // ** Since and Until cannot exceed 90 days. ** // ** Since from the last 90 days can be queried ** -func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { +func (e *Exchange) QueryClosedOrders( + ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64, +) (orders []types.Order, err error) { newSince := since now := time.Now() @@ -507,7 +521,9 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) (err // REMARK: If your start time is 90 days earlier, we will update it to now - 90 days. // ** StartTime is inclusive, EndTime is exclusive. If you use the EndTime, the StartTime is required. ** // ** StartTime and EndTime cannot exceed 90 days. ** -func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) (trades []types.Trade, err error) { +func (e *Exchange) QueryTrades( + ctx context.Context, symbol string, options *types.TradeQueryOptions, +) (trades []types.Trade, err error) { if options.LastTradeID != 0 { log.Warn("!!!BITGET EXCHANGE API NOTICE!!! The trade of response is in descending order, so the last trade id not supported.") } From d4a201b9874c549027fd07b0eba37f15d056a85e Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 8 Dec 2023 15:44:15 +0800 Subject: [PATCH 280/422] upgrade go requirement to 1.20 --- Dockerfile | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09a1f6482f..66c50268c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # First stage container -FROM golang:1.18.10-alpine3.17 AS builder +FROM golang:1.20-alpine3.18 AS builder RUN apk add --no-cache git ca-certificates gcc musl-dev libc-dev pkgconfig # gcc is for github.com/mattn/go-sqlite3 # ADD . $GOPATH/src/github.com/c9s/bbgo diff --git a/go.mod b/go.mod index 57f0bf0994..ff2d965e5c 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/c9s/bbgo -go 1.18 +go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 From c90e5d1907302cf711cb409368f6e5ef4ba7c598 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 8 Dec 2023 15:45:11 +0800 Subject: [PATCH 281/422] upgrade docker image for go 1.20 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 66c50268c2..d4530a3b27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN go get github.com/mattn/go-sqlite3 RUN go build -o $GOPATH_ORIG/bin/bbgo ./cmd/bbgo # Second stage container -FROM alpine:3.17 +FROM alpine:3.18 # Create the default user 'bbgo' and assign to env 'USER' ENV USER=bbgo From 2c3ccdf03086849627c2249f9eccd03c065e0448 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 16:56:19 +0800 Subject: [PATCH 282/422] xdepthmaker: more improvements - place orders with balance quota calculation - wait for authed event - clean up open orders on start --- pkg/strategy/xdepthmaker/strategy.go | 142 ++++++++++++++++++++-- pkg/strategy/xdepthmaker/strategy_test.go | 3 +- 2 files changed, 131 insertions(+), 14 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index eb7d2a9234..2ee47c318d 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -13,6 +13,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" @@ -237,7 +238,7 @@ type Strategy struct { lastPrice fixedpoint.Value - stopC chan struct{} + stopC, authedC chan struct{} } func (s *Strategy) ID() string { @@ -365,13 +366,47 @@ func (s *Strategy) CrossRun( go s.runTradeRecover(ctx) } + s.authedC = make(chan struct{}, 2) + s.makerSession.UserDataStream.OnAuth(func() { + select { + case s.authedC <- struct{}{}: + default: + } + }) + s.hedgeSession.UserDataStream.OnAuth(func() { + select { + case s.authedC <- struct{}{}: + default: + } + }) + go func() { + log.Infof("waiting for user data stream to get authenticated") + select { + case <-ctx.Done(): + return + case <-s.authedC: + } + + select { + case <-ctx.Done(): + return + case <-s.authedC: + } + + log.Infof("user data stream authenticated, start placing orders...") + posTicker := time.NewTicker(util.MillisecondsJitter(s.HedgeInterval.Duration(), 200)) defer posTicker.Stop() fullReplenishTicker := time.NewTicker(util.MillisecondsJitter(s.FullReplenishInterval.Duration(), 200)) defer fullReplenishTicker.Stop() + // clean up the previous open orders + if err := s.cleanUpOpenOrders(ctx); err != nil { + log.WithError(err).Errorf("error cleaning up open orders") + } + s.updateQuote(ctx, 0) for { @@ -598,18 +633,14 @@ func (s *Strategy) runTradeRecover(ctx context.Context) { } } -func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLayer int) ([]types.SubmitOrder, error) { - bestBid, bestAsk, hasPrice := pricingBook.BestBidAndAsk() +func (s *Strategy) generateMakerOrders( + pricingBook *types.StreamOrderBook, maxLayer int, availableBase fixedpoint.Value, availableQuote fixedpoint.Value, +) ([]types.SubmitOrder, error) { + _, _, hasPrice := pricingBook.BestBidAndAsk() if !hasPrice { return nil, nil } - bestBidPrice := bestBid.Price - bestAskPrice := bestAsk.Price - - lastMidPrice := bestBidPrice.Add(bestAskPrice).Div(Two) - _ = lastMidPrice - var submitOrders []types.SubmitOrder var accumulatedBidQuantity = fixedpoint.Zero var accumulatedAskQuantity = fixedpoint.Zero @@ -621,8 +652,33 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLa maxLayer = s.NumLayers } + var availableBalances = map[types.SideType]fixedpoint.Value{ + types.SideTypeBuy: availableQuote, + types.SideTypeSell: availableBase, + } + for _, side := range []types.SideType{types.SideTypeBuy, types.SideTypeSell} { + sideBook := dupPricingBook.SideBook(side) + if sideBook.Len() == 0 { + log.Warnf("orderbook %s side is empty", side) + continue + } + + availableSideBalance, ok := availableBalances[side] + if !ok { + log.Warnf("no available balance for side %s side", side) + continue + } + + layerLoop: for i := 1; i <= maxLayer; i++ { + // simple break, we need to check the market minNotional and minQuantity later + if !availableSideBalance.Eq(fixedpoint.PosInf) { + if availableSideBalance.IsZero() || availableSideBalance.Sign() < 0 { + break layerLoop + } + } + requiredDepthFloat, err := s.DepthScale.Scale(i) if err != nil { return nil, errors.Wrapf(err, "depthScale scale error") @@ -631,7 +687,6 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLa // requiredDepth is the required depth in quote currency requiredDepth := fixedpoint.NewFromFloat(requiredDepthFloat) - sideBook := dupPricingBook.SideBook(side) index := sideBook.IndexByQuoteVolumeDepth(requiredDepth) pvs := types.PriceVolumeSlice{} @@ -641,7 +696,11 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLa pvs = sideBook[0 : index+1] } - log.Infof("required depth: %f, pvs: %+v", requiredDepth.Float64(), pvs) + if len(pvs) == 0 { + continue + } + + log.Infof("side: %s required depth: %f, pvs: %+v", side, requiredDepth.Float64(), pvs) depthPrice, err := averageDepthPrice(pvs) if err != nil { @@ -678,12 +737,36 @@ func (s *Strategy) generateMakerOrders(pricingBook *types.StreamOrderBook, maxLa accumulatedBidQuantity = accumulatedBidQuantity.Add(quantity) quoteQuantity := fixedpoint.Mul(quantity, depthPrice) quoteQuantity = quoteQuantity.Round(s.makerMarket.PricePrecision, fixedpoint.Up) + + if !availableSideBalance.Eq(fixedpoint.PosInf) && availableSideBalance.Compare(quoteQuantity) <= 0 { + quoteQuantity = availableSideBalance + quantity = quoteQuantity.Div(depthPrice).Round(s.makerMarket.PricePrecision, fixedpoint.Down) + } + + if quantity.Compare(s.makerMarket.MinQuantity) <= 0 || quoteQuantity.Compare(s.makerMarket.MinNotional) <= 0 { + break layerLoop + } + + availableSideBalance = availableSideBalance.Sub(quoteQuantity) + accumulatedBidQuoteQuantity = accumulatedBidQuoteQuantity.Add(quoteQuantity) case types.SideTypeSell: quantity = quantity.Sub(accumulatedAskQuantity) - accumulatedAskQuantity = accumulatedAskQuantity.Add(quantity) + quoteQuantity := quantity.Mul(depthPrice) + + // balance check + if !availableSideBalance.Eq(fixedpoint.PosInf) && availableSideBalance.Compare(quantity) <= 0 { + break layerLoop + } + + if quantity.Compare(s.makerMarket.MinQuantity) <= 0 || quoteQuantity.Compare(s.makerMarket.MinNotional) <= 0 { + break layerLoop + } + + availableSideBalance = availableSideBalance.Sub(quantity) + accumulatedAskQuantity = accumulatedAskQuantity.Add(quantity) } submitOrders = append(submitOrders, types.SubmitOrder{ @@ -760,7 +843,27 @@ func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { return } - submitOrders, err := s.generateMakerOrders(s.pricingBook, maxLayer) + balances, err := s.MakerOrderExecutor.Session().Exchange.QueryAccountBalances(ctx) + if err != nil { + log.WithError(err).Errorf("balance query error") + return + } + + log.Infof("balances: %+v", balances.NotZero()) + + quoteBalance, ok := balances[s.makerMarket.QuoteCurrency] + if !ok { + return + } + + baseBalance, ok := balances[s.makerMarket.BaseCurrency] + if !ok { + return + } + + log.Infof("quote balance: %s, base balance: %s", quoteBalance, baseBalance) + + submitOrders, err := s.generateMakerOrders(s.pricingBook, maxLayer, baseBalance.Available, quoteBalance.Available) if err != nil { log.WithError(err).Errorf("generate order error") return @@ -780,6 +883,19 @@ func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { s.orderStore.Add(createdOrders...) } +func (s *Strategy) cleanUpOpenOrders(ctx context.Context) error { + openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.makerSession.Exchange, s.Symbol) + if err != nil { + return err + } + + if err := s.makerSession.Exchange.CancelOrders(ctx, openOrders...); err != nil { + return err + } + + return nil +} + func selectSessions2( sessions map[string]*bbgo.ExchangeSession, n1, n2 string, ) (s1, s2 *bbgo.ExchangeSession, err error) { diff --git a/pkg/strategy/xdepthmaker/strategy_test.go b/pkg/strategy/xdepthmaker/strategy_test.go index 182f0bc8ac..faa3cf2ba5 100644 --- a/pkg/strategy/xdepthmaker/strategy_test.go +++ b/pkg/strategy/xdepthmaker/strategy_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" . "github.com/c9s/bbgo/pkg/testing/testhelper" "github.com/c9s/bbgo/pkg/types" ) @@ -61,7 +62,7 @@ func TestStrategy_generateMakerOrders(t *testing.T) { Time: time.Now(), }) - orders, err := s.generateMakerOrders(pricingBook, 0) + orders, err := s.generateMakerOrders(pricingBook, 0, fixedpoint.PosInf, fixedpoint.PosInf) assert.NoError(t, err) AssertOrdersPriceSideQuantity(t, []PriceSideQuantityAssert{ {Side: types.SideTypeBuy, Price: Number("25000"), Quantity: Number("0.04")}, // =~ $1000.00 From de7eb8453b454455b684ef8f369769faaf0deb24 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:00:25 +0800 Subject: [PATCH 283/422] xdepthmaker: refactor auth binding to bindAuthSignal --- pkg/strategy/xdepthmaker/strategy.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 2ee47c318d..89804d7cc9 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -367,18 +367,8 @@ func (s *Strategy) CrossRun( } s.authedC = make(chan struct{}, 2) - s.makerSession.UserDataStream.OnAuth(func() { - select { - case s.authedC <- struct{}{}: - default: - } - }) - s.hedgeSession.UserDataStream.OnAuth(func() { - select { - case s.authedC <- struct{}{}: - default: - } - }) + bindAuthSignal(ctx, s.makerSession.UserDataStream, s.authedC) + bindAuthSignal(ctx, s.hedgeSession.UserDataStream, s.authedC) go func() { log.Infof("waiting for user data stream to get authenticated") @@ -936,3 +926,14 @@ func min(a, b int) int { return b } + +func bindAuthSignal(ctx context.Context, stream types.Stream, c chan<- struct{}) { + stream.OnAuth(func() { + select { + case <-ctx.Done(): + return + case c <- struct{}{}: + default: + } + }) +} From cedd7900661ceff59b510a64a9da7629f420ad94 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:02:17 +0800 Subject: [PATCH 284/422] xdepthmaker: add lastOrderReplenishTime to prevent replacing orders too frequent --- pkg/strategy/xdepthmaker/strategy.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 89804d7cc9..65f03b9a3e 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -399,6 +399,7 @@ func (s *Strategy) CrossRun( s.updateQuote(ctx, 0) + lastOrderReplenishTime := time.Now() for { select { @@ -412,6 +413,7 @@ func (s *Strategy) CrossRun( case <-fullReplenishTicker.C: s.updateQuote(ctx, 0) + lastOrderReplenishTime = time.Now() case sig, ok := <-s.pricingBook.C: // when any book change event happened @@ -419,6 +421,10 @@ func (s *Strategy) CrossRun( return } + if time.Since(lastOrderReplenishTime) < time.Minute { + continue + } + switch sig.Type { case types.BookSignalSnapshot: s.updateQuote(ctx, 0) @@ -427,6 +433,8 @@ func (s *Strategy) CrossRun( s.updateQuote(ctx, 5) } + lastOrderReplenishTime = time.Now() + case <-posTicker.C: // For positive position and positive covered position: // uncover position = +5 - +3 (covered position) = 2 From 98468b39c7ba430bfb2e3b76ff53fc8e25f9898b Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:05:07 +0800 Subject: [PATCH 285/422] xdepthmaker: change priceHeartBeat alert to warning --- pkg/strategy/xdepthmaker/strategy.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 65f03b9a3e..d6bfc2b6fa 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -421,7 +421,7 @@ func (s *Strategy) CrossRun( return } - if time.Since(lastOrderReplenishTime) < time.Minute { + if time.Since(lastOrderReplenishTime) < 10*time.Second { continue } @@ -828,14 +828,14 @@ func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { bookLastUpdateTime := s.pricingBook.LastUpdateTime() if _, err := s.bidPriceHeartBeat.Update(bestBid); err != nil { - log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", + log.WithError(err).Warnf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) return } if _, err := s.askPriceHeartBeat.Update(bestAsk); err != nil { - log.WithError(err).Errorf("quote update error, %s price not updating, order book last update: %s ago", + log.WithError(err).Warnf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) return From 8c13092d8b099feda9d206aea234bdd0d26233fb Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:58:48 +0800 Subject: [PATCH 286/422] types: add slice book test for copy depth --- pkg/types/sliceorderbook_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 pkg/types/sliceorderbook_test.go diff --git a/pkg/types/sliceorderbook_test.go b/pkg/types/sliceorderbook_test.go new file mode 100644 index 0000000000..06aedd3ae8 --- /dev/null +++ b/pkg/types/sliceorderbook_test.go @@ -0,0 +1,27 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSliceOrderBook_CopyDepth(t *testing.T) { + b := &SliceOrderBook{ + Bids: PriceVolumeSlice{ + {Price: number(0.119), Volume: number(100.0)}, + {Price: number(0.118), Volume: number(100.0)}, + {Price: number(0.117), Volume: number(100.0)}, + {Price: number(0.116), Volume: number(100.0)}, + }, + Asks: PriceVolumeSlice{ + {Price: number(0.120), Volume: number(100.0)}, + {Price: number(0.121), Volume: number(100.0)}, + {Price: number(0.122), Volume: number(100.0)}, + }, + } + + copied := b.CopyDepth(0) + assert.Equal(t, 3, len(copied.SideBook(SideTypeSell))) + assert.Equal(t, 4, len(copied.SideBook(SideTypeBuy))) +} From 9f14215ce8c5154db849b1613dc9f4e44b913a92 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:59:02 +0800 Subject: [PATCH 287/422] bbgo: reduce logs --- pkg/bbgo/activeorderbook.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 7c9a9a2dc6..ef5234a562 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -392,10 +392,10 @@ func (b *ActiveOrderBook) add(order types.Order) { if pendingOrder, ok := b.pendingOrderUpdates.Get(order.OrderID); ok { // if the pending order update time is newer than the adding order // we should use the pending order rather than the adding order. - // if pending order is older, than we should add the new one, and drop the pending order - log.Infof("found pending order update") + // if the pending order is older, then we should add the new one, and drop the pending order + log.Debugf("found pending order update: %+v", pendingOrder) if isNewerOrderUpdate(pendingOrder, order) { - log.Infof("pending order update is newer") + log.Infof("pending order update is newer: %+v", pendingOrder) order = pendingOrder } From 8c6724b2648b0659ae99a04d11288e05468f0360 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 17:59:16 +0800 Subject: [PATCH 288/422] xdepthmaker: fix pricing book copy by avoiding using CopyDepth --- pkg/strategy/xdepthmaker/strategy.go | 15 ++++++-- pkg/strategy/xdepthmaker/strategy_test.go | 2 +- pkg/types/orderbook.go | 47 ++++++++++++++--------- pkg/types/price_volume_slice.go | 2 +- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index d6bfc2b6fa..7173cec99d 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -25,7 +25,7 @@ var defaultMargin = fixedpoint.NewFromFloat(0.003) var Two = fixedpoint.NewFromInt(2) -const priceUpdateTimeout = 30 * time.Second +const priceUpdateTimeout = 5 * time.Minute const ID = "xdepthmaker" @@ -644,7 +644,16 @@ func (s *Strategy) generateMakerOrders( var accumulatedAskQuantity = fixedpoint.Zero var accumulatedBidQuoteQuantity = fixedpoint.Zero - dupPricingBook := pricingBook.CopyDepth(0) + // copy the pricing book because during the generation the book data could change + dupPricingBook := pricingBook.Copy() + + log.Infof("dupPricingBook: \n\tbids: %+v \n\tasks: %+v", + dupPricingBook.SideBook(types.SideTypeBuy), + dupPricingBook.SideBook(types.SideTypeSell)) + + log.Infof("pricingBook: \n\tbids: %+v \n\tasks: %+v", + pricingBook.SideBook(types.SideTypeBuy), + pricingBook.SideBook(types.SideTypeSell)) if maxLayer == 0 || maxLayer > s.NumLayers { maxLayer = s.NumLayers @@ -831,14 +840,12 @@ func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { log.WithError(err).Warnf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) - return } if _, err := s.askPriceHeartBeat.Update(bestAsk); err != nil { log.WithError(err).Warnf("quote update error, %s price not updating, order book last update: %s ago", s.Symbol, time.Since(bookLastUpdateTime)) - return } balances, err := s.MakerOrderExecutor.Session().Exchange.QueryAccountBalances(ctx) diff --git a/pkg/strategy/xdepthmaker/strategy_test.go b/pkg/strategy/xdepthmaker/strategy_test.go index faa3cf2ba5..2df6424c0d 100644 --- a/pkg/strategy/xdepthmaker/strategy_test.go +++ b/pkg/strategy/xdepthmaker/strategy_test.go @@ -45,7 +45,7 @@ func TestStrategy_generateMakerOrders(t *testing.T) { } pricingBook := types.NewStreamBook("BTCUSDT") - pricingBook.OrderBook.Load(types.SliceOrderBook{ + pricingBook.Load(types.SliceOrderBook{ Symbol: "BTCUSDT", Bids: types.PriceVolumeSlice{ {Price: Number("25000.00"), Volume: Number("0.1")}, diff --git a/pkg/types/orderbook.go b/pkg/types/orderbook.go index 3d32989c86..6fbc92ba95 100644 --- a/pkg/types/orderbook.go +++ b/pkg/types/orderbook.go @@ -26,8 +26,9 @@ type OrderBook interface { type MutexOrderBook struct { sync.Mutex - Symbol string - OrderBook OrderBook + Symbol string + + orderBook OrderBook } func NewMutexOrderBook(symbol string) *MutexOrderBook { @@ -39,20 +40,27 @@ func NewMutexOrderBook(symbol string) *MutexOrderBook { return &MutexOrderBook{ Symbol: symbol, - OrderBook: book, + orderBook: book, } } func (b *MutexOrderBook) IsValid() (ok bool, err error) { b.Lock() - ok, err = b.OrderBook.IsValid() + ok, err = b.orderBook.IsValid() b.Unlock() return ok, err } +func (b *MutexOrderBook) SideBook(sideType SideType) PriceVolumeSlice { + b.Lock() + sideBook := b.orderBook.SideBook(sideType) + b.Unlock() + return sideBook +} + func (b *MutexOrderBook) LastUpdateTime() time.Time { b.Lock() - t := b.OrderBook.LastUpdateTime() + t := b.orderBook.LastUpdateTime() b.Unlock() return t } @@ -60,8 +68,8 @@ func (b *MutexOrderBook) LastUpdateTime() time.Time { func (b *MutexOrderBook) BestBidAndAsk() (bid, ask PriceVolume, ok bool) { var ok1, ok2 bool b.Lock() - bid, ok1 = b.OrderBook.BestBid() - ask, ok2 = b.OrderBook.BestAsk() + bid, ok1 = b.orderBook.BestBid() + ask, ok2 = b.orderBook.BestAsk() b.Unlock() ok = ok1 && ok2 return bid, ask, ok @@ -69,48 +77,49 @@ func (b *MutexOrderBook) BestBidAndAsk() (bid, ask PriceVolume, ok bool) { func (b *MutexOrderBook) BestBid() (pv PriceVolume, ok bool) { b.Lock() - pv, ok = b.OrderBook.BestBid() + pv, ok = b.orderBook.BestBid() b.Unlock() return pv, ok } func (b *MutexOrderBook) BestAsk() (pv PriceVolume, ok bool) { b.Lock() - pv, ok = b.OrderBook.BestAsk() + pv, ok = b.orderBook.BestAsk() b.Unlock() return pv, ok } func (b *MutexOrderBook) Load(book SliceOrderBook) { b.Lock() - b.OrderBook.Load(book) + b.orderBook.Load(book) b.Unlock() } func (b *MutexOrderBook) Reset() { b.Lock() - b.OrderBook.Reset() + b.orderBook.Reset() b.Unlock() } func (b *MutexOrderBook) CopyDepth(depth int) OrderBook { b.Lock() - book := b.OrderBook.CopyDepth(depth) - b.Unlock() - return book + defer b.Unlock() + + return b.orderBook.CopyDepth(depth) } func (b *MutexOrderBook) Copy() OrderBook { b.Lock() - book := b.OrderBook.Copy() - b.Unlock() - return book + defer b.Unlock() + + return b.orderBook.Copy() } func (b *MutexOrderBook) Update(update SliceOrderBook) { b.Lock() - b.OrderBook.Update(update) - b.Unlock() + defer b.Unlock() + + b.orderBook.Update(update) } type BookSignalType string diff --git a/pkg/types/price_volume_slice.go b/pkg/types/price_volume_slice.go index 53fb7d9136..03a6c8b008 100644 --- a/pkg/types/price_volume_slice.go +++ b/pkg/types/price_volume_slice.go @@ -17,7 +17,7 @@ func (p PriceVolume) Equals(b PriceVolume) bool { } func (p PriceVolume) String() string { - return fmt.Sprintf("PriceVolume{ price: %s, volume: %s }", p.Price.String(), p.Volume.String()) + return fmt.Sprintf("PriceVolume{ Price: %s, Volume: %s }", p.Price.String(), p.Volume.String()) } type PriceVolumeSlice []PriceVolume From 158c48b8079f2f045dd2f2bd5b91caa10331bade Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 20:46:17 +0800 Subject: [PATCH 289/422] bbgo: change verbose info log to debug log --- pkg/bbgo/activeorderbook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index ef5234a562..b0045b84b3 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -395,7 +395,7 @@ func (b *ActiveOrderBook) add(order types.Order) { // if the pending order is older, then we should add the new one, and drop the pending order log.Debugf("found pending order update: %+v", pendingOrder) if isNewerOrderUpdate(pendingOrder, order) { - log.Infof("pending order update is newer: %+v", pendingOrder) + log.Debugf("pending order update is newer: %+v", pendingOrder) order = pendingOrder } From f39474f9f5b099d1aaea6302c8bbb0c9f189e4ea Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 22:01:05 +0800 Subject: [PATCH 290/422] github: upgrade golang linter to v1.54 --- .github/workflows/golang-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml index 420724cb71..326e2c5b7d 100644 --- a/.github/workflows/golang-lint.yml +++ b/.github/workflows/golang-lint.yml @@ -12,11 +12,11 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version: 1.21 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.46.2 + version: v1.54 From 15fa83268b1661bd71963e28911d9508ab89ca25 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 11 Dec 2023 22:06:48 +0800 Subject: [PATCH 291/422] github: upgrade to actions/setup-go@v4 --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5d0cfaedd7..79ca5258cf 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -52,7 +52,7 @@ jobs: # auto-start: "false" - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} From c5282a8f9b838455ca26d776867cfb19d3b115c3 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 16:37:43 +0800 Subject: [PATCH 292/422] bitget: add more debug logs --- .../v2/get_unfilled_orders_request.go | 14 ++++-- pkg/exchange/bitget/convert.go | 3 +- pkg/exchange/bitget/exchange.go | 50 ++++++++++++++----- pkg/strategy/xdepthmaker/strategy.go | 6 +-- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go index d597df1d06..3bf57ddc62 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_unfilled_orders_request.go @@ -38,13 +38,17 @@ type GetUnfilledOrdersRequest struct { client requestgen.AuthenticatedAPIClient symbol *string `param:"symbol,query"` - // Limit number default 100 max 100 + + // limit number default 100 max 100 limit *string `param:"limit,query"` + // idLessThan requests the content on the page before this ID (older data), the value input should be the orderId of the corresponding interface. - idLessThan *string `param:"idLessThan,query"` - startTime *time.Time `param:"startTime,milliseconds,query"` - endTime *time.Time `param:"endTime,milliseconds,query"` - orderId *string `param:"orderId,query"` + idLessThan *string `param:"idLessThan,query"` + + startTime *time.Time `param:"startTime,milliseconds,query"` + endTime *time.Time `param:"endTime,milliseconds,query"` + + orderId *string `param:"orderId,query"` } func (c *Client) NewGetUnfilledOrdersRequest() *GetUnfilledOrdersRequest { diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 94c1b06782..ff5fb62588 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -191,7 +191,7 @@ func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { // The market order will be executed immediately, so this check is used to handle corner cases. if orderType == types.OrderTypeMarket { qty = order.BaseVolume - log.Warnf("!!! The price(%f) and quantity(%f) are not verified for market orders, because we only receive limit orders in the test environment !!!", price.Float64(), qty.Float64()) + log.Warnf("!!! The price(%f) and quantity(%f) are not verified for market orders, because we only receive limit orders in the test environment !!!", price.Float64(), qty.Float64()) } return &types.Order{ @@ -202,6 +202,7 @@ func unfilledOrderToGlobalOrder(order v2.UnfilledOrder) (*types.Order, error) { Type: orderType, Quantity: qty, Price: price, + // Bitget does not include the "time-in-force" field in its API response for spot trading, so we set GTC. TimeInForce: types.TimeInForceGTC, }, diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 62ab09091a..de873205e3 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -14,6 +14,7 @@ import ( "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" ) const ( @@ -34,26 +35,45 @@ var log = logrus.WithFields(logrus.Fields{ var ( // queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryAccountRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-account-assets queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryOpenOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Unfilled-Orders queryOpenOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // closedQueryOrdersRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Get-History-Orders closedQueryOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/15), 5) - // submitOrdersRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order - submitOrdersRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + + // submitOrderRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Place-Order + submitOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTradeRateLimiter has its own rate limit. https://www.bitget.com/zh-CN/api-doc/spot/trade/Get-Fills queryTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // cancelOrderRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/trade/Cancel-Order cancelOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // kLineRateLimiter has its own rate limit. https://www.bitget.com/api-doc/spot/market/Get-Candle-Data - kLineOrderRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + kLineRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) +var debugf func(msg string, args ...interface{}) + +func init() { + if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { + debugf = log.Infof + } else { + debugf = func(msg string, args ...interface{}) {} + } +} + type Exchange struct { key, secret, passphrase string @@ -199,7 +219,7 @@ func (e *Exchange) QueryKLines( req.EndTime(*options.EndTime) } - if err := kLineOrderRateLimiter.Wait(ctx); err != nil { + if err := kLineRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("query klines rate limiter wait error: %w", err) } @@ -322,26 +342,34 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr req.ClientOrderId(order.ClientOrderID) } - if err := submitOrdersRateLimiter.Wait(ctx); err != nil { + if err := submitOrderRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("place order rate limiter wait error: %w", err) } + res, err := req.Do(ctx) if err != nil { return nil, fmt.Errorf("failed to place order, order: %#v, err: %w", order, err) } + debugf("order created: %+v", res) + if len(res.OrderId) == 0 || (len(order.ClientOrderID) != 0 && res.ClientOrderId != order.ClientOrderID) { return nil, fmt.Errorf("unexpected order id, resp: %#v, order: %#v", res, order) } orderId := res.OrderId + + debugf("fetching unfilled order info for order #%s", orderId) ordersResp, err := e.v2client.NewGetUnfilledOrdersRequest().OrderId(orderId).Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query open order by order id: %s, err: %w", orderId, err) } - switch len(ordersResp) { - case 0: + debugf("unfilled order response: %+v", ordersResp) + + if len(ordersResp) == 1 { + return unfilledOrderToGlobalOrder(ordersResp[0]) + } else if len(ordersResp) == 0 { // The market order will be executed immediately, so we cannot retrieve it through the NewGetUnfilledOrdersRequest API. // Try to get the order from the NewGetHistoryOrdersRequest API. ordersResp, err := e.v2client.NewGetHistoryOrdersRequest().OrderId(orderId).Do(ctx) @@ -354,13 +382,9 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (cr } return toGlobalOrder(ordersResp[0]) - - case 1: - return unfilledOrderToGlobalOrder(ordersResp[0]) - - default: - return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) } + + return nil, fmt.Errorf("unexpected order length, order id: %s", orderId) } func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 7173cec99d..5bdf04cb08 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -647,14 +647,10 @@ func (s *Strategy) generateMakerOrders( // copy the pricing book because during the generation the book data could change dupPricingBook := pricingBook.Copy() - log.Infof("dupPricingBook: \n\tbids: %+v \n\tasks: %+v", + log.Infof("pricingBook: \n\tbids: %+v \n\tasks: %+v", dupPricingBook.SideBook(types.SideTypeBuy), dupPricingBook.SideBook(types.SideTypeSell)) - log.Infof("pricingBook: \n\tbids: %+v \n\tasks: %+v", - pricingBook.SideBook(types.SideTypeBuy), - pricingBook.SideBook(types.SideTypeSell)) - if maxLayer == 0 || maxLayer > s.NumLayers { maxLayer = s.NumLayers } From c2724c4f626d6387c422627146444ec26fcef3e3 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 12 Dec 2023 17:30:51 +0800 Subject: [PATCH 293/422] pkg/exchange: fix price is zero when order not executed --- pkg/exchange/bitget/convert.go | 7 ++++++- pkg/exchange/bitget/convert_test.go | 6 ++++-- pkg/exchange/bitget/stream_test.go | 1 + pkg/exchange/bitget/types.go | 22 ++++++++++++---------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index ff5fb62588..c0167d16fc 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -430,6 +430,11 @@ func (o *Order) toGlobalOrder() (types.Order, error) { } } + price := o.Price + if orderType == types.OrderTypeMarket { + price = o.PriceAvg + } + return types.Order{ SubmitOrder: types.SubmitOrder{ ClientOrderID: o.ClientOrderId, @@ -437,7 +442,7 @@ func (o *Order) toGlobalOrder() (types.Order, error) { Side: side, Type: orderType, Quantity: qty, - Price: o.PriceAvg, + Price: price, TimeInForce: timeInForce, }, Exchange: types.ExchangeBitget, diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index da083929f5..a0898e21de 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -924,6 +924,7 @@ func TestOrder_toGlobalOrder(t *testing.T) { // } t.Run("limit buy", func(t *testing.T) { newO := o + newO.Price = fixedpoint.NewFromFloat(0.49998) newO.OrderType = v2.OrderTypeLimit res, err := newO.toGlobalOrder() @@ -935,7 +936,7 @@ func TestOrder_toGlobalOrder(t *testing.T) { Side: types.SideTypeBuy, Type: types.OrderTypeLimit, Quantity: newO.Size, - Price: newO.PriceAvg, + Price: newO.Price, TimeInForce: types.TimeInForceGTC, }, Exchange: types.ExchangeBitget, @@ -983,6 +984,7 @@ func TestOrder_toGlobalOrder(t *testing.T) { newO := o newO.OrderType = v2.OrderTypeLimit newO.Side = v2.SideTypeSell + newO.Price = fixedpoint.NewFromFloat(0.48710) res, err := newO.toGlobalOrder() assert.NoError(t, err) @@ -993,7 +995,7 @@ func TestOrder_toGlobalOrder(t *testing.T) { Side: types.SideTypeSell, Type: types.OrderTypeLimit, Quantity: newO.Size, - Price: newO.PriceAvg, + Price: newO.Price, TimeInForce: types.TimeInForceGTC, }, Exchange: types.ExchangeBitget, diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index 011210abeb..cfe952e6a2 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -125,6 +125,7 @@ func TestStream(t *testing.T) { }) t.Run("private test", func(t *testing.T) { + s.SetPrivateChannelSymbols([]string{"BTCUSDT"}) err := s.Connect(context.Background()) assert.NoError(t, err) diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index c9c8468f5e..1634797134 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -488,16 +488,18 @@ type Order struct { // Size is base coin when orderType=limit; quote coin when orderType=market Size fixedpoint.Value `json:"size"` // Buy amount, returned when buying at market price - Notional fixedpoint.Value `json:"notional"` - OrderType v2.OrderType `json:"orderType"` - Force v2.OrderForce `json:"force"` - Side v2.SideType `json:"side"` - AccBaseVolume fixedpoint.Value `json:"accBaseVolume"` - PriceAvg fixedpoint.Value `json:"priceAvg"` - Status v2.OrderStatus `json:"status"` - CreatedTime types.MillisecondTimestamp `json:"cTime"` - UpdatedTime types.MillisecondTimestamp `json:"uTime"` - FeeDetail []struct { + Notional fixedpoint.Value `json:"notional"` + OrderType v2.OrderType `json:"orderType"` + Force v2.OrderForce `json:"force"` + Side v2.SideType `json:"side"` + AccBaseVolume fixedpoint.Value `json:"accBaseVolume"` + PriceAvg fixedpoint.Value `json:"priceAvg"` + // The Price field is only applicable to limit orders. + Price fixedpoint.Value `json:"price"` + Status v2.OrderStatus `json:"status"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` + FeeDetail []struct { FeeCoin string `json:"feeCoin"` Fee string `json:"fee"` } `json:"feeDetail"` From 8025d05eac324d97519bb5538805a653d3b625fd Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:18:34 +0800 Subject: [PATCH 294/422] core: log trades pruning --- pkg/core/tradestore.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/core/tradestore.go b/pkg/core/tradestore.go index 485f820dcf..d54d804cb7 100644 --- a/pkg/core/tradestore.go +++ b/pkg/core/tradestore.go @@ -4,6 +4,8 @@ import ( "sync" "time" + log "github.com/sirupsen/logrus" + "github.com/c9s/bbgo/pkg/types" ) @@ -112,14 +114,16 @@ func (s *TradeStore) touchLastTradeTime(trade types.Trade) { } } -// pruneExpiredTrades prunes trades that are older than the expiry time -// see TradeExpiryTime -func (s *TradeStore) pruneExpiredTrades(curTime time.Time) { +// Prune prunes trades that are older than the expiry time +// see TradeExpiryTime (24 hours) +func (s *TradeStore) Prune(curTime time.Time) { s.Lock() defer s.Unlock() var trades = make(map[uint64]types.Trade) var cutOffTime = curTime.Add(-TradeExpiryTime) + + log.Infof("pruning expired trades, cutoff time = %s", cutOffTime.String()) for _, trade := range s.trades { if trade.Time.Before(cutOffTime) { continue @@ -129,15 +133,13 @@ func (s *TradeStore) pruneExpiredTrades(curTime time.Time) { } s.trades = trades -} -func (s *TradeStore) Prune(curTime time.Time) { - s.pruneExpiredTrades(curTime) + log.Infof("trade pruning done, size: %d", len(trades)) } func (s *TradeStore) isCoolTrade(trade types.Trade) bool { - // if the time of last trade is over 1 hour, we call it's cool trade - return s.lastTradeTime != (time.Time{}) && time.Time(trade.Time).Sub(s.lastTradeTime) > time.Hour + // if the duration between the current trade and the last trade is over 1 hour, we call it "cool trade" + return !s.lastTradeTime.IsZero() && time.Time(trade.Time).Sub(s.lastTradeTime) > time.Hour } func (s *TradeStore) BindStream(stream types.Stream) { From 97c39921bd95b02b312c72a176249b1b7c7e9855 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:18:59 +0800 Subject: [PATCH 295/422] core: adjust TradeExpiryTime to 3 hour --- pkg/core/tradestore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/core/tradestore.go b/pkg/core/tradestore.go index d54d804cb7..9f1cc6af04 100644 --- a/pkg/core/tradestore.go +++ b/pkg/core/tradestore.go @@ -9,7 +9,7 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -const TradeExpiryTime = 24 * time.Hour +const TradeExpiryTime = 3 * time.Hour const PruneTriggerNumOfTrades = 10_000 type TradeStore struct { @@ -115,7 +115,7 @@ func (s *TradeStore) touchLastTradeTime(trade types.Trade) { } // Prune prunes trades that are older than the expiry time -// see TradeExpiryTime (24 hours) +// see TradeExpiryTime (3 hours) func (s *TradeStore) Prune(curTime time.Time) { s.Lock() defer s.Unlock() From 685f332495180ec610506838ae61b682e2d2208c Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:21:22 +0800 Subject: [PATCH 296/422] core: enable trade store's trade pruning in NewTradeCollector --- pkg/core/tradecollector.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/core/tradecollector.go b/pkg/core/tradecollector.go index 07c92b3dde..e6e5d515f5 100644 --- a/pkg/core/tradecollector.go +++ b/pkg/core/tradecollector.go @@ -34,12 +34,15 @@ type TradeCollector struct { } func NewTradeCollector(symbol string, position *types.Position, orderStore *OrderStore) *TradeCollector { + tradeStore := NewTradeStore() + tradeStore.EnablePrune = true + return &TradeCollector{ Symbol: symbol, orderSig: sigchan.New(1), tradeC: make(chan types.Trade, 100), - tradeStore: NewTradeStore(), + tradeStore: tradeStore, doneTrades: make(map[types.TradeKey]struct{}), position: position, orderStore: orderStore, @@ -88,7 +91,9 @@ func (c *TradeCollector) Emit() { c.orderSig.Emit() } -func (c *TradeCollector) Recover(ctx context.Context, ex types.ExchangeTradeHistoryService, symbol string, from time.Time) error { +func (c *TradeCollector) Recover( + ctx context.Context, ex types.ExchangeTradeHistoryService, symbol string, from time.Time, +) error { logrus.Debugf("recovering %s trades...", symbol) trades, err := ex.QueryTrades(ctx, symbol, &types.TradeQueryOptions{ From 21c8593c45d400a2edfc55690eda6e505387ebee Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:24:33 +0800 Subject: [PATCH 297/422] core: add exceededMaximumTradeStoreSize check --- pkg/core/tradestore.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/core/tradestore.go b/pkg/core/tradestore.go index 9f1cc6af04..340e3df150 100644 --- a/pkg/core/tradestore.go +++ b/pkg/core/tradestore.go @@ -10,7 +10,7 @@ import ( ) const TradeExpiryTime = 3 * time.Hour -const PruneTriggerNumOfTrades = 10_000 +const MaximumTradeStoreSize = 1_000 type TradeStore struct { // any created trades for tracking trades @@ -142,6 +142,10 @@ func (s *TradeStore) isCoolTrade(trade types.Trade) bool { return !s.lastTradeTime.IsZero() && time.Time(trade.Time).Sub(s.lastTradeTime) > time.Hour } +func (s *TradeStore) exceededMaximumTradeStoreSize() bool { + return len(s.trades) > MaximumTradeStoreSize +} + func (s *TradeStore) BindStream(stream types.Stream) { stream.OnTradeUpdate(func(trade types.Trade) { s.Add(trade) @@ -149,7 +153,7 @@ func (s *TradeStore) BindStream(stream types.Stream) { if s.EnablePrune { stream.OnTradeUpdate(func(trade types.Trade) { - if s.isCoolTrade(trade) { + if s.isCoolTrade(trade) || s.exceededMaximumTradeStoreSize() { s.Prune(time.Time(trade.Time)) } }) From 4e26b9d2adf2de88d78c38b1a7b4d20d45f73d1c Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:25:09 +0800 Subject: [PATCH 298/422] core: pull out cool trade period to a constant --- pkg/core/tradestore.go | 3 ++- pkg/core/tradestore_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/core/tradestore.go b/pkg/core/tradestore.go index 340e3df150..f1acdb2448 100644 --- a/pkg/core/tradestore.go +++ b/pkg/core/tradestore.go @@ -10,6 +10,7 @@ import ( ) const TradeExpiryTime = 3 * time.Hour +const CoolTradePeriod = 1 * time.Hour const MaximumTradeStoreSize = 1_000 type TradeStore struct { @@ -139,7 +140,7 @@ func (s *TradeStore) Prune(curTime time.Time) { func (s *TradeStore) isCoolTrade(trade types.Trade) bool { // if the duration between the current trade and the last trade is over 1 hour, we call it "cool trade" - return !s.lastTradeTime.IsZero() && time.Time(trade.Time).Sub(s.lastTradeTime) > time.Hour + return !s.lastTradeTime.IsZero() && time.Time(trade.Time).Sub(s.lastTradeTime) > CoolTradePeriod } func (s *TradeStore) exceededMaximumTradeStoreSize() bool { diff --git a/pkg/core/tradestore_test.go b/pkg/core/tradestore_test.go index 431572ab4f..c820a2a02e 100644 --- a/pkg/core/tradestore_test.go +++ b/pkg/core/tradestore_test.go @@ -30,7 +30,7 @@ func TestTradeStore_Prune(t *testing.T) { store := NewTradeStore() store.Add( types.Trade{ID: 1, Time: types.Time(now.Add(-25 * time.Hour))}, - types.Trade{ID: 2, Time: types.Time(now.Add(-23 * time.Hour))}, + types.Trade{ID: 2, Time: types.Time(now.Add(-2 * time.Hour))}, types.Trade{ID: 3, Time: types.Time(now.Add(-2 * time.Minute))}, types.Trade{ID: 4, Time: types.Time(now.Add(-1 * time.Minute))}, ) From 6d944f0f2715ebc7114e510f0245fb08c448752f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 01:07:06 +0000 Subject: [PATCH 299/422] build(deps): bump @babel/traverse in /apps/frontend Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.18.5 to 7.23.6. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.23.6/packages/babel-traverse) --- updated-dependencies: - dependency-name: "@babel/traverse" dependency-type: indirect ... Signed-off-by: dependabot[bot] --- apps/frontend/yarn.lock | 160 +++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 28 deletions(-) diff --git a/apps/frontend/yarn.lock b/apps/frontend/yarn.lock index 8537ee4f1f..6ef54bb3fb 100644 --- a/apps/frontend/yarn.lock +++ b/apps/frontend/yarn.lock @@ -17,6 +17,14 @@ dependencies: "@babel/highlight" "^7.16.7" +"@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" + integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA== + dependencies: + "@babel/highlight" "^7.23.4" + chalk "^2.4.2" + "@babel/compat-data@^7.17.10": version "7.18.5" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.18.5.tgz#acac0c839e317038c73137fbb6ef71a1d6238471" @@ -52,6 +60,16 @@ "@jridgewell/gen-mapping" "^0.3.0" jsesc "^2.5.1" +"@babel/generator@^7.23.6": + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.6.tgz#9e1fca4811c77a10580d17d26b57b036133f3c2e" + integrity sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw== + dependencies: + "@babel/types" "^7.23.6" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-compilation-targets@^7.18.2": version "7.18.2" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.2.tgz#67a85a10cbd5fc7f1457fec2e7f45441dc6c754b" @@ -62,25 +80,30 @@ browserslist "^4.20.2" semver "^6.3.0" -"@babel/helper-environment-visitor@^7.16.7", "@babel/helper-environment-visitor@^7.18.2": +"@babel/helper-environment-visitor@^7.16.7": version "7.18.2" resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.2.tgz#8a6d2dedb53f6bf248e31b4baf38739ee4a637bd" integrity sha512-14GQKWkX9oJzPiQQ7/J36FTXcD4kSp8egKjO9nINlSKiHITRA9q/R74qu8S9xlc/b/yjsJItQUeeh3xnGN0voQ== -"@babel/helper-function-name@^7.17.9": - version "7.17.9" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.17.9.tgz#136fcd54bc1da82fcb47565cf16fd8e444b1ff12" - integrity sha512-7cRisGlVtiVqZ0MW0/yFB4atgpGLWEHUVYnb448hZK4x+vih0YO5UoS11XIYtZYqHd0dIPMdUSv8q5K4LdMnIg== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== dependencies: - "@babel/template" "^7.16.7" - "@babel/types" "^7.17.0" + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" -"@babel/helper-hoist-variables@^7.16.7": - version "7.16.7" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz#86bcb19a77a509c7b77d0e22323ef588fa58c246" - integrity sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg== +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== dependencies: - "@babel/types" "^7.16.7" + "@babel/types" "^7.22.5" "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.16.7": version "7.16.7" @@ -122,11 +145,28 @@ dependencies: "@babel/types" "^7.16.7" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-string-parser@^7.23.4": + version "7.23.4" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83" + integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ== + "@babel/helper-validator-identifier@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.16.7.tgz#b203ce62ce5fe153899b617c08957de860de4d23" @@ -150,11 +190,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.23.4": + version "7.23.4" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b" + integrity sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + "@babel/parser@^7.16.7", "@babel/parser@^7.18.5": version "7.18.5" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.5.tgz#337062363436a893a2d22faa60be5bb37091c83c" integrity sha512-YZWVaglMiplo7v8f1oMQ5ZPQr0vn7HPeZXxXWsxXJRjGVrzUFn9OxFQl1sb5wzfootjA/yChhW84BV+383FSOw== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.6": + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.6.tgz#ba1c9e512bda72a47e285ae42aff9d2a635a9e3b" + integrity sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ== + "@babel/plugin-syntax-jsx@^7.12.13": version "7.17.12" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.17.12.tgz#834035b45061983a491f60096f61a2e7c5674a47" @@ -178,23 +232,32 @@ "@babel/parser" "^7.16.7" "@babel/types" "^7.16.7" -"@babel/traverse@^7.18.0", "@babel/traverse@^7.18.2", "@babel/traverse@^7.18.5": - version "7.18.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.18.5.tgz#94a8195ad9642801837988ab77f36e992d9a20cd" - integrity sha512-aKXj1KT66sBj0vVzk6rEeAO6Z9aiiQ68wfDgge3nHhA/my6xMM/7HGQUNumKZaoa2qUPQ5whJG9aAifsxUKfLA== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.16.7" - "@babel/generator" "^7.18.2" - "@babel/helper-environment-visitor" "^7.18.2" - "@babel/helper-function-name" "^7.17.9" - "@babel/helper-hoist-variables" "^7.16.7" - "@babel/helper-split-export-declaration" "^7.16.7" - "@babel/parser" "^7.18.5" - "@babel/types" "^7.18.4" - debug "^4.1.0" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.18.0", "@babel/traverse@^7.18.2", "@babel/traverse@^7.18.5": + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.6.tgz#b53526a2367a0dd6edc423637f3d2d0f2521abc5" + integrity sha512-czastdK1e8YByZqezMPFiZ8ahwVMh/ESl9vPgvgdB9AmFMGP5jfpFax74AQgl5zj4XHzqeYAg2l8PuUeRS1MgQ== + dependencies: + "@babel/code-frame" "^7.23.5" + "@babel/generator" "^7.23.6" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.6" + "@babel/types" "^7.23.6" + debug "^4.3.1" globals "^11.1.0" -"@babel/types@^7.16.7", "@babel/types@^7.17.0", "@babel/types@^7.18.0", "@babel/types@^7.18.2", "@babel/types@^7.18.4": +"@babel/types@^7.16.7", "@babel/types@^7.18.0", "@babel/types@^7.18.2", "@babel/types@^7.18.4": version "7.18.4" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.18.4.tgz#27eae9b9fd18e9dccc3f9d6ad051336f307be354" integrity sha512-ThN1mBcMq5pG/Vm2IcBmPPfyPXbd8S02rS+OBIDENdufvqC7Z/jHPCv9IcP01277aKtDI8g/2XysBN4hA8niiw== @@ -202,6 +265,15 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6": + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd" + integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg== + dependencies: + "@babel/helper-string-parser" "^7.23.4" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@date-io/core@^2.14.0": version "2.14.0" resolved "https://registry.yarnpkg.com/@date-io/core/-/core-2.14.0.tgz#03e9b9b9fc8e4d561c32dd324df0f3ccd967ef14" @@ -694,21 +766,53 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/gen-mapping@^0.3.2": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" + integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + "@jridgewell/resolve-uri@^3.0.3": version "3.0.7" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz#30cd49820a962aff48c8fffc5cd760151fca61fe" integrity sha512-8cXDaBBHOr2pQ7j77Y6Vp5VDT2sIqWyWQ56TjEq4ih/a4iST3dItRe8Q9fp0rrIl9DoKhWQtUQz/YpOxLkXbNA== +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + "@jridgewell/set-array@^1.0.0": version "1.1.1" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.1.tgz#36a6acc93987adcf0ba50c66908bd0b70de8afea" integrity sha512-Ct5MqZkLGEXTVmQYbGtx9SVqD2fqwvdubdps5D3djjAkgkKwT918VNOz65pEHFaYTeWcukmJmH5SwsA9Tn2ObQ== +"@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.13" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.13.tgz#b6461fb0c2964356c469e115f504c95ad97ab88c" integrity sha512-GryiOJmNcWbovBxTfZSF71V/mXbgcV3MewDe3kIMCLyIh5e7SKAeUZs+rMnJ8jkMolZ/4/VsdBmMrw3l+VdZ3w== +"@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + +"@jridgewell/trace-mapping@^0.3.17": + version "0.3.20" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz#72e45707cf240fa6b081d0366f8265b0cd10197f" + integrity sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@jridgewell/trace-mapping@^0.3.9": version "0.3.13" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.13.tgz#dcfe3e95f224c8fe97a87a5235defec999aa92ea" @@ -1233,7 +1337,7 @@ caniuse-lite@^1.0.30001332, caniuse-lite@^1.0.30001349: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001357.tgz#dec7fc4158ef6ad24690d0eec7b91f32b8cb1b5d" integrity sha512-b+KbWHdHePp+ZpNj+RDHFChZmuN+J5EvuQUlee9jOQIUAdhv9uvAZeEtUeLAknXbkiu1uxjQ9NLp1ie894CuWg== -chalk@^2.0.0: +chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1386,7 +1490,7 @@ d3-time@^1.0.11: resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.1.0.tgz#b1e19d307dae9c900b7e5b25ffc5dcc249a8a0f1" integrity sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA== -debug@^4.1.0, debug@^4.1.1: +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== From b022a6119f6411f4006788b75bb290767f3639e1 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 17:53:32 +0800 Subject: [PATCH 300/422] bitget: add bitget log prefix --- pkg/exchange/bitget/exchange.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index de873205e3..7fbd4f12e4 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -68,7 +68,9 @@ var debugf func(msg string, args ...interface{}) func init() { if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { - debugf = log.Infof + debugf = func(msg string, args ...interface{}) { + log.Infof("[BITGET] "+msg, args...) + } } else { debugf = func(msg string, args ...interface{}) {} } From f3ce4c2cc67e766bd49da16ba940d807760a78a9 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 12 Dec 2023 18:04:16 +0800 Subject: [PATCH 301/422] bitget: refactor debug function tool --- pkg/exchange/bitget/exchange.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 7fbd4f12e4..2174063269 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -8,6 +8,7 @@ import ( "time" "github.com/sirupsen/logrus" + prefixed "github.com/x-cray/logrus-prefixed-formatter" "go.uber.org/multierr" "golang.org/x/time/rate" @@ -64,16 +65,31 @@ var ( kLineRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) -var debugf func(msg string, args ...interface{}) +type LogFunction func(msg string, args ...interface{}) -func init() { +var debugf LogFunction + +func isPrefixFormatterConfigured() bool { + _, isPrefixFormatter := logrus.StandardLogger().Formatter.(*prefixed.TextFormatter) + return isPrefixFormatter +} + +func getDebugFunction() LogFunction { if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { - debugf = func(msg string, args ...interface{}) { - log.Infof("[BITGET] "+msg, args...) + if isPrefixFormatterConfigured() { + return func(msg string, args ...interface{}) { + log.Infof("[BITGET] "+msg, args...) + } } - } else { - debugf = func(msg string, args ...interface{}) {} + + return log.Infof } + + return func(msg string, args ...interface{}) {} +} + +func init() { + debugf = getDebugFunction() } type Exchange struct { From 6cbb17fb763c15d4e4dbf3239e4aa29bfc2f6f18 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 13 Dec 2023 09:47:18 +0800 Subject: [PATCH 302/422] all: refactor log formatter functions --- pkg/bbgo/envvar.go | 48 +++++++++++++++++++++++++++++++++ pkg/cmd/root.go | 25 ++++------------- pkg/exchange/bitget/exchange.go | 12 --------- 3 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 pkg/bbgo/envvar.go diff --git a/pkg/bbgo/envvar.go b/pkg/bbgo/envvar.go new file mode 100644 index 0000000000..10ac022b35 --- /dev/null +++ b/pkg/bbgo/envvar.go @@ -0,0 +1,48 @@ +package bbgo + +import ( + "os" + + log "github.com/sirupsen/logrus" + prefixed "github.com/x-cray/logrus-prefixed-formatter" +) + +func GetCurrentEnv() string { + env := os.Getenv("BBGO_ENV") + if env == "" { + env = "development" + } + + return env +} + +func NewLogFormatterWithEnv(env string) log.Formatter { + switch env { + case "production", "prod", "stag", "staging": + // always use json formatter for production and staging + return &log.JSONFormatter{} + } + + return &prefixed.TextFormatter{} +} + +type LogFormatterType string + +const ( + LogFormatterTypePrefixed LogFormatterType = "prefixed" + LogFormatterTypeText LogFormatterType = "text" + LogFormatterTypeJson LogFormatterType = "json" +) + +func NewLogFormatter(logFormatter LogFormatterType) log.Formatter { + switch logFormatter { + case LogFormatterTypePrefixed: + return &prefixed.TextFormatter{} + case LogFormatterTypeText: + return &log.TextFormatter{} + case LogFormatterTypeJson: + return &log.JSONFormatter{} + } + + return &prefixed.TextFormatter{} +} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 67ccd6bba8..417e6196ce 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -17,7 +17,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - prefixed "github.com/x-cray/logrus-prefixed-formatter" "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/util" @@ -46,10 +45,7 @@ var RootCmd = &cobra.Command{ log.SetLevel(log.DebugLevel) } - env := os.Getenv("BBGO_ENV") - if env == "" { - env = "development" - } + env := bbgo.GetCurrentEnv() logFormatter, err := cmd.Flags().GetString("log-formatter") if err != nil { @@ -57,22 +53,11 @@ var RootCmd = &cobra.Command{ } if len(logFormatter) == 0 { - switch env { - case "production", "prod", "stag", "staging": - // always use json formatter for production and staging - log.SetFormatter(&log.JSONFormatter{}) - default: - log.SetFormatter(&prefixed.TextFormatter{}) - } + formatter := bbgo.NewLogFormatterWithEnv(env) + log.SetFormatter(formatter) } else { - switch logFormatter { - case "prefixed": - log.SetFormatter(&prefixed.TextFormatter{}) - case "text": - log.SetFormatter(&log.TextFormatter{}) - case "json": - log.SetFormatter(&log.JSONFormatter{}) - } + formatter := bbgo.NewLogFormatter(bbgo.LogFormatterType(logFormatter)) + log.SetFormatter(formatter) } if token := viper.GetString("rollbar-token"); token != "" { diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 2174063269..dc0ca8e7ab 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -8,7 +8,6 @@ import ( "time" "github.com/sirupsen/logrus" - prefixed "github.com/x-cray/logrus-prefixed-formatter" "go.uber.org/multierr" "golang.org/x/time/rate" @@ -69,19 +68,8 @@ type LogFunction func(msg string, args ...interface{}) var debugf LogFunction -func isPrefixFormatterConfigured() bool { - _, isPrefixFormatter := logrus.StandardLogger().Formatter.(*prefixed.TextFormatter) - return isPrefixFormatter -} - func getDebugFunction() LogFunction { if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { - if isPrefixFormatterConfigured() { - return func(msg string, args ...interface{}) { - log.Infof("[BITGET] "+msg, args...) - } - } - return log.Infof } From 29550f0013f23459b1946b3f3f7a15fa8e18c1e7 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 22 Nov 2023 16:08:07 +0800 Subject: [PATCH 303/422] pkg/exchange: we don't need the fee rate in the public stream --- pkg/exchange/bybit/stream.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index 3b61661943..fd09ec4c8f 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -85,6 +85,12 @@ func NewStream(key, secret string, userDataProvider StreamDataProvider) *Stream stream.SetDispatcher(stream.dispatchEvent) stream.SetHeartBeat(stream.ping) stream.SetBeforeConnect(func(ctx context.Context) (err error) { + if stream.PublicOnly { + // we don't need the fee rate in the public stream. + return + } + + // get account fee rate go stream.feeRateProvider.Start(ctx) stream.marketsInfo, err = stream.streamDataProvider.QueryMarkets(ctx) From 115c2dc139d8985bbb02b2295f8849e50cc8d457 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 13 Dec 2023 14:00:53 +0800 Subject: [PATCH 304/422] bbgo: refactor active orderbook --- pkg/bbgo/activeorderbook.go | 40 ++++++++++++++++++++++++++----------- pkg/types/ordermap.go | 9 +++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index b0045b84b3..8afde5c9f2 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -14,7 +14,7 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -const CancelOrderWaitTime = 20 * time.Millisecond +const DefaultCancelOrderWaitTime = 20 * time.Millisecond // ActiveOrderBook manages the local active order books. // @@ -34,6 +34,8 @@ type ActiveOrderBook struct { C sigchan.Chan mu sync.Mutex + + cancelOrderWaitTime time.Duration } func NewActiveOrderBook(symbol string) *ActiveOrderBook { @@ -42,9 +44,14 @@ func NewActiveOrderBook(symbol string) *ActiveOrderBook { orders: types.NewSyncOrderMap(), pendingOrderUpdates: types.NewSyncOrderMap(), C: sigchan.New(1), + cancelOrderWaitTime: DefaultCancelOrderWaitTime, } } +func (b *ActiveOrderBook) SetCancelOrderWaitTime(duration time.Duration) { + b.cancelOrderWaitTime = duration +} + func (b *ActiveOrderBook) MarshalJSON() ([]byte, error) { orders := b.Backup() return json.Marshal(orders) @@ -175,7 +182,8 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, } log.Debugf("[ActiveOrderBook] gracefully cancelling %s orders...", b.Symbol) - waitTime := CancelOrderWaitTime + waitTime := b.cancelOrderWaitTime + orderCancelTimeout := 5 * time.Second startTime := time.Now() // ensure every order is canceled @@ -192,7 +200,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, log.Debugf("[ActiveOrderBook] waiting %s for %s orders to be cancelled...", waitTime, b.Symbol) - clear, err := b.waitAllClear(ctx, waitTime, 5*time.Second) + clear, err := b.waitAllClear(ctx, waitTime, orderCancelTimeout) if clear || err != nil { break } @@ -203,12 +211,9 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, // verify the current open orders via the RESTful API log.Warnf("[ActiveOrderBook] using REStful API to verify active orders...") - var symbolOrdersMap = map[string]types.OrderSlice{} - for _, order := range orders { - symbolOrdersMap[order.Symbol] = append(symbolOrdersMap[order.Symbol], order) - } + var symbolOrdersMap = categorizeOrderBySymbol(orders) - var leftOrders []types.Order + var leftOrders types.OrderSlice for symbol := range symbolOrdersMap { symbolOrders, ok := symbolOrdersMap[symbol] if !ok { @@ -221,13 +226,14 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, continue } - orderMap := types.NewOrderMap(openOrders...) + openOrderMap := types.NewOrderMap(openOrders...) for _, o := range symbolOrders { - // if it's not on the order book (open orders), we should remove it from our local side - if !orderMap.Exists(o.OrderID) { + // if it's not on the order book (open orders), + // we should remove it from our local side + if !openOrderMap.Exists(o.OrderID) { b.Remove(o) } else { - leftOrders = append(leftOrders, o) + leftOrders.Add(o) } } } @@ -441,3 +447,13 @@ func (b *ActiveOrderBook) Orders() types.OrderSlice { func (b *ActiveOrderBook) Lookup(f func(o types.Order) bool) *types.Order { return b.orders.Lookup(f) } + +func categorizeOrderBySymbol(orders types.OrderSlice) map[string]types.OrderSlice { + orderMap := map[string]types.OrderSlice{} + + for _, order := range orders { + orderMap[order.Symbol] = append(orderMap[order.Symbol], order) + } + + return orderMap +} diff --git a/pkg/types/ordermap.go b/pkg/types/ordermap.go index c091d6c418..b00da9bff2 100644 --- a/pkg/types/ordermap.go +++ b/pkg/types/ordermap.go @@ -254,6 +254,15 @@ func (m *SyncOrderMap) Orders() (slice OrderSlice) { type OrderSlice []Order +func (s *OrderSlice) Add(o Order) { + *s = append(*s, o) +} + +// Map builds up an OrderMap by the order id +func (s OrderSlice) Map() OrderMap { + return NewOrderMap(s...) +} + func (s OrderSlice) SeparateBySide() (buyOrders, sellOrders []Order) { for _, o := range s { switch o.Side { From 092d5cfb074f9a47d2174124d43cc0b233533d8a Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Fri, 24 Nov 2023 16:04:48 +0800 Subject: [PATCH 305/422] FEATURE: cancel maker orders and open take profit order --- pkg/strategy/dca2/openPosition.go | 25 ++++++++++++++ pkg/strategy/dca2/openPosition_test.go | 10 ++++-- pkg/strategy/dca2/takeProfit.go | 42 +++++++++++++++++++++++ pkg/strategy/dca2/takeProfit_test.go | 47 ++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 pkg/strategy/dca2/takeProfit.go create mode 100644 pkg/strategy/dca2/takeProfit_test.go diff --git a/pkg/strategy/dca2/openPosition.go b/pkg/strategy/dca2/openPosition.go index f79a0c5aee..59988396e0 100644 --- a/pkg/strategy/dca2/openPosition.go +++ b/pkg/strategy/dca2/openPosition.go @@ -9,6 +9,10 @@ import ( "github.com/c9s/bbgo/pkg/types" ) +type cancelOrdersByGroupIDApi interface { + CancelOrdersByGroupID(ctx context.Context, groupID int64) ([]types.Order, error) +} + func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { s.logger.Infof("[DCA] start placing open position orders") price, err := getBestPriceUntilSuccess(ctx, s.Session.Exchange, s.Symbol, s.Short) @@ -111,3 +115,24 @@ func calculateNotionalAndNum(market types.Market, short bool, budget fixedpoint. return fixedpoint.Zero, 0 } + +func (s *Strategy) cancelOpenPositionOrders(ctx context.Context) error { + s.logger.Info("[DCA] cancel open position orders") + e, ok := s.Session.Exchange.(cancelOrdersByGroupIDApi) + if ok { + cancelledOrders, err := e.CancelOrdersByGroupID(ctx, int64(s.OrderGroupID)) + if err != nil { + return err + } + + for _, cancelledOrder := range cancelledOrders { + s.logger.Info("CANCEL ", cancelledOrder.String()) + } + } else { + if err := s.OrderExecutor.ActiveMakerOrders().GracefulCancel(ctx, s.Session.Exchange); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/strategy/dca2/openPosition_test.go b/pkg/strategy/dca2/openPosition_test.go index ede4c9f31b..440dadaf1a 100644 --- a/pkg/strategy/dca2/openPosition_test.go +++ b/pkg/strategy/dca2/openPosition_test.go @@ -32,9 +32,13 @@ func newTestStrategy(va ...string) *Strategy { market := newTestMarket() s := &Strategy{ - logger: logrus.NewEntry(logrus.New()), - Symbol: symbol, - Market: market, + logger: logrus.NewEntry(logrus.New()), + Symbol: symbol, + Market: market, + Short: false, + TakeProfitRatio: Number("10%"), + makerSide: types.SideTypeBuy, + takeProfitSide: types.SideTypeSell, } return s } diff --git a/pkg/strategy/dca2/takeProfit.go b/pkg/strategy/dca2/takeProfit.go new file mode 100644 index 0000000000..ea46f0b0d0 --- /dev/null +++ b/pkg/strategy/dca2/takeProfit.go @@ -0,0 +1,42 @@ +package dca2 + +import ( + "context" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) openTakeProfitOrders(ctx context.Context) error { + s.logger.Info("[DCA] open take profit orders") + takeProfitOrder := s.generateTakeProfitOrder(s.Short, s.Position) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, takeProfitOrder) + if err != nil { + return err + } + + for _, createdOrder := range createdOrders { + s.logger.Info("SUBMIT TAKE PROFIT ORDER ", createdOrder.String()) + } + + return nil +} + +func (s *Strategy) generateTakeProfitOrder(short bool, position *types.Position) types.SubmitOrder { + takeProfitRatio := s.TakeProfitRatio + if s.Short { + takeProfitRatio = takeProfitRatio.Neg() + } + takeProfitPrice := s.Market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) + return types.SubmitOrder{ + Symbol: s.Symbol, + Market: s.Market, + Type: types.OrderTypeLimit, + Price: takeProfitPrice, + Side: s.takeProfitSide, + TimeInForce: types.TimeInForceGTC, + Quantity: position.GetBase().Abs(), + Tag: orderTag, + GroupID: s.OrderGroupID, + } +} diff --git a/pkg/strategy/dca2/takeProfit_test.go b/pkg/strategy/dca2/takeProfit_test.go new file mode 100644 index 0000000000..47c69ddb07 --- /dev/null +++ b/pkg/strategy/dca2/takeProfit_test.go @@ -0,0 +1,47 @@ +package dca2 + +import ( + "testing" + + . "github.com/c9s/bbgo/pkg/testing/testhelper" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestGenerateTakeProfitOrder(t *testing.T) { + assert := assert.New(t) + + strategy := newTestStrategy() + + position := types.NewPositionFromMarket(strategy.Market) + position.AddTrade(types.Trade{ + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + Price: Number("28500"), + Quantity: Number("1"), + QuoteQuantity: Number("28500"), + Fee: Number("0.0015"), + FeeCurrency: strategy.Market.BaseCurrency, + }) + + o := strategy.generateTakeProfitOrder(false, position) + assert.Equal(Number("31397.09"), o.Price) + assert.Equal(Number("0.9985"), o.Quantity) + assert.Equal(types.SideTypeSell, o.Side) + assert.Equal(strategy.Symbol, o.Symbol) + + position.AddTrade(types.Trade{ + Side: types.SideTypeBuy, + Price: Number("27000"), + Quantity: Number("0.5"), + QuoteQuantity: Number("13500"), + Fee: Number("0.00075"), + FeeCurrency: strategy.Market.BaseCurrency, + }) + o = strategy.generateTakeProfitOrder(false, position) + assert.Equal(Number("30846.26"), o.Price) + assert.Equal(Number("1.49775"), o.Quantity) + assert.Equal(types.SideTypeSell, o.Side) + assert.Equal(strategy.Symbol, o.Symbol) + +} From e3d51777d394adc34dbc3e40f10524fea31e2327 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Thu, 7 Dec 2023 14:38:13 +0800 Subject: [PATCH 306/422] rename --- .../{openPosition.go => open_position.go} | 25 ++++++----- ...Position_test.go => open_position_test.go} | 17 ++++---- pkg/strategy/dca2/strategy.go | 6 +-- pkg/strategy/dca2/takeProfit.go | 42 ------------------ pkg/strategy/dca2/take_profit.go | 43 +++++++++++++++++++ ...takeProfit_test.go => take_profit_test.go} | 4 +- 6 files changed, 72 insertions(+), 65 deletions(-) rename pkg/strategy/dca2/{openPosition.go => open_position.go} (81%) rename pkg/strategy/dca2/{openPosition_test.go => open_position_test.go} (81%) delete mode 100644 pkg/strategy/dca2/takeProfit.go create mode 100644 pkg/strategy/dca2/take_profit.go rename pkg/strategy/dca2/{takeProfit_test.go => take_profit_test.go} (84%) diff --git a/pkg/strategy/dca2/openPosition.go b/pkg/strategy/dca2/open_position.go similarity index 81% rename from pkg/strategy/dca2/openPosition.go rename to pkg/strategy/dca2/open_position.go index 59988396e0..6a25c5c649 100644 --- a/pkg/strategy/dca2/openPosition.go +++ b/pkg/strategy/dca2/open_position.go @@ -20,7 +20,7 @@ func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { return err } - orders, err := s.generateOpenPositionOrders(s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum) + orders, err := generateOpenPositionOrders(s.Market, s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum, s.OrderGroupID) if err != nil { return err } @@ -48,7 +48,7 @@ func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol str } } -func (s *Strategy) generateOpenPositionOrders(short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64) ([]types.SubmitOrder, error) { +func generateOpenPositionOrders(market types.Market, short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64, orderGroupID uint32) ([]types.SubmitOrder, error) { factor := fixedpoint.One.Sub(priceDeviation) if short { factor = fixedpoint.One.Add(priceDeviation) @@ -60,32 +60,37 @@ func (s *Strategy) generateOpenPositionOrders(short bool, budget, price, priceDe if i > 0 { price = price.Mul(factor) } - price = s.Market.TruncatePrice(price) - if price.Compare(s.Market.MinPrice) < 0 { + price = market.TruncatePrice(price) + if price.Compare(market.MinPrice) < 0 { break } prices = append(prices, price) } - notional, orderNum := calculateNotionalAndNum(s.Market, short, budget, prices) + notional, orderNum := calculateNotionalAndNum(market, short, budget, prices) if orderNum == 0 { return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, budget: %s", price, budget) } + side := types.SideTypeBuy + if short { + side = types.SideTypeSell + } + var submitOrders []types.SubmitOrder for i := 0; i < orderNum; i++ { - quantity := s.Market.TruncateQuantity(notional.Div(prices[i])) + quantity := market.TruncateQuantity(notional.Div(prices[i])) submitOrders = append(submitOrders, types.SubmitOrder{ - Symbol: s.Symbol, - Market: s.Market, + Symbol: market.Symbol, + Market: market, Type: types.OrderTypeLimit, Price: prices[i], - Side: s.makerSide, + Side: side, TimeInForce: types.TimeInForceGTC, Quantity: quantity, Tag: orderTag, - GroupID: s.OrderGroupID, + GroupID: orderGroupID, }) } diff --git a/pkg/strategy/dca2/openPosition_test.go b/pkg/strategy/dca2/open_position_test.go similarity index 81% rename from pkg/strategy/dca2/openPosition_test.go rename to pkg/strategy/dca2/open_position_test.go index 440dadaf1a..3d51bf6af0 100644 --- a/pkg/strategy/dca2/openPosition_test.go +++ b/pkg/strategy/dca2/open_position_test.go @@ -12,6 +12,7 @@ import ( func newTestMarket() types.Market { return types.Market{ + Symbol: "BTCUSDT", BaseCurrency: "BTC", QuoteCurrency: "USDT", TickSize: Number(0.01), @@ -32,13 +33,13 @@ func newTestStrategy(va ...string) *Strategy { market := newTestMarket() s := &Strategy{ - logger: logrus.NewEntry(logrus.New()), - Symbol: symbol, - Market: market, - Short: false, - TakeProfitRatio: Number("10%"), - makerSide: types.SideTypeBuy, - takeProfitSide: types.SideTypeSell, + logger: logrus.NewEntry(logrus.New()), + Symbol: symbol, + Market: market, + Short: false, + TakeProfitRatio: Number("10%"), + openPositionSide: types.SideTypeBuy, + takeProfitSide: types.SideTypeSell, } return s } @@ -52,7 +53,7 @@ func TestGenerateOpenPositionOrders(t *testing.T) { budget := Number("10500") askPrice := Number("30000") margin := Number("0.05") - submitOrders, err := strategy.generateOpenPositionOrders(false, budget, askPrice, margin, 4) + submitOrders, err := generateOpenPositionOrders(strategy.Market, false, budget, askPrice, margin, 4, strategy.OrderGroupID) if !assert.NoError(err) { return } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 55e13433bc..597eb69da4 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -50,7 +50,7 @@ type Strategy struct { // private field mu sync.Mutex - makerSide types.SideType + openPositionSide types.SideType takeProfitSide types.SideType takeProfitPrice fixedpoint.Value startTimeOfNextRound time.Time @@ -106,10 +106,10 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. instanceID := s.InstanceID() if s.Short { - s.makerSide = types.SideTypeSell + s.openPositionSide = types.SideTypeSell s.takeProfitSide = types.SideTypeBuy } else { - s.makerSide = types.SideTypeBuy + s.openPositionSide = types.SideTypeBuy s.takeProfitSide = types.SideTypeSell } diff --git a/pkg/strategy/dca2/takeProfit.go b/pkg/strategy/dca2/takeProfit.go deleted file mode 100644 index ea46f0b0d0..0000000000 --- a/pkg/strategy/dca2/takeProfit.go +++ /dev/null @@ -1,42 +0,0 @@ -package dca2 - -import ( - "context" - - "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/types" -) - -func (s *Strategy) openTakeProfitOrders(ctx context.Context) error { - s.logger.Info("[DCA] open take profit orders") - takeProfitOrder := s.generateTakeProfitOrder(s.Short, s.Position) - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, takeProfitOrder) - if err != nil { - return err - } - - for _, createdOrder := range createdOrders { - s.logger.Info("SUBMIT TAKE PROFIT ORDER ", createdOrder.String()) - } - - return nil -} - -func (s *Strategy) generateTakeProfitOrder(short bool, position *types.Position) types.SubmitOrder { - takeProfitRatio := s.TakeProfitRatio - if s.Short { - takeProfitRatio = takeProfitRatio.Neg() - } - takeProfitPrice := s.Market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) - return types.SubmitOrder{ - Symbol: s.Symbol, - Market: s.Market, - Type: types.OrderTypeLimit, - Price: takeProfitPrice, - Side: s.takeProfitSide, - TimeInForce: types.TimeInForceGTC, - Quantity: position.GetBase().Abs(), - Tag: orderTag, - GroupID: s.OrderGroupID, - } -} diff --git a/pkg/strategy/dca2/take_profit.go b/pkg/strategy/dca2/take_profit.go new file mode 100644 index 0000000000..148312abbf --- /dev/null +++ b/pkg/strategy/dca2/take_profit.go @@ -0,0 +1,43 @@ +package dca2 + +import ( + "context" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) placeTakeProfitOrders(ctx context.Context) error { + s.logger.Info("[DCA] start placing take profit orders") + order := generateTakeProfitOrder(s.Market, s.Short, s.TakeProfitRatio, s.Position, s.OrderGroupID) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, order) + if err != nil { + return err + } + + for _, createdOrder := range createdOrders { + s.logger.Info("SUBMIT TAKE PROFIT ORDER ", createdOrder.String()) + } + + return nil +} + +func generateTakeProfitOrder(market types.Market, short bool, takeProfitRatio fixedpoint.Value, position *types.Position, orderGroupID uint32) types.SubmitOrder { + side := types.SideTypeSell + if short { + takeProfitRatio = takeProfitRatio.Neg() + side = types.SideTypeBuy + } + takeProfitPrice := market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) + return types.SubmitOrder{ + Symbol: market.Symbol, + Market: market, + Type: types.OrderTypeLimit, + Price: takeProfitPrice, + Side: side, + TimeInForce: types.TimeInForceGTC, + Quantity: position.GetBase().Abs(), + Tag: orderTag, + GroupID: orderGroupID, + } +} diff --git a/pkg/strategy/dca2/takeProfit_test.go b/pkg/strategy/dca2/take_profit_test.go similarity index 84% rename from pkg/strategy/dca2/takeProfit_test.go rename to pkg/strategy/dca2/take_profit_test.go index 47c69ddb07..1080e3ba85 100644 --- a/pkg/strategy/dca2/takeProfit_test.go +++ b/pkg/strategy/dca2/take_profit_test.go @@ -24,7 +24,7 @@ func TestGenerateTakeProfitOrder(t *testing.T) { FeeCurrency: strategy.Market.BaseCurrency, }) - o := strategy.generateTakeProfitOrder(false, position) + o := generateTakeProfitOrder(strategy.Market, false, strategy.TakeProfitRatio, position, strategy.OrderGroupID) assert.Equal(Number("31397.09"), o.Price) assert.Equal(Number("0.9985"), o.Quantity) assert.Equal(types.SideTypeSell, o.Side) @@ -38,7 +38,7 @@ func TestGenerateTakeProfitOrder(t *testing.T) { Fee: Number("0.00075"), FeeCurrency: strategy.Market.BaseCurrency, }) - o = strategy.generateTakeProfitOrder(false, position) + o = generateTakeProfitOrder(strategy.Market, false, strategy.TakeProfitRatio, position, strategy.OrderGroupID) assert.Equal(Number("30846.26"), o.Price) assert.Equal(Number("1.49775"), o.Quantity) assert.Equal(types.SideTypeSell, o.Side) From c170eac99145c1922a83d4ff26cc76405c518e3d Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 13 Dec 2023 15:25:52 +0800 Subject: [PATCH 307/422] bbgo: fix active order book graceful cancel checking logics --- pkg/bbgo/activeorderbook.go | 55 +++++++++++++++++++++++-------------- pkg/types/ordermap.go | 8 ++++++ 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index 8afde5c9f2..e33830e1b5 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -3,7 +3,6 @@ package bbgo import ( "context" "encoding/json" - "sort" "sync" "time" @@ -163,10 +162,14 @@ func (b *ActiveOrderBook) FastCancel(ctx context.Context, ex types.Exchange, ord } // GracefulCancel cancels the active orders gracefully -func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, orders ...types.Order) error { +func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, specifiedOrders ...types.Order) error { + cancelAll := false + orders := specifiedOrders + // if no orders are given, set to cancelAll - if len(orders) == 0 { + if len(specifiedOrders) == 0 { orders = b.Orders() + cancelAll = true } else { // simple check on given input hasSymbol := b.Symbol != "" @@ -176,6 +179,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, } } } + // optimize order cancel for back-testing if IsBackTesting { return ex.CancelOrders(context.Background(), orders...) @@ -200,16 +204,24 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, log.Debugf("[ActiveOrderBook] waiting %s for %s orders to be cancelled...", waitTime, b.Symbol) - clear, err := b.waitAllClear(ctx, waitTime, orderCancelTimeout) - if clear || err != nil { - break - } + if cancelAll { + clear, err := b.waitAllClear(ctx, waitTime, orderCancelTimeout) + if clear || err != nil { + break + } - log.Warnf("[ActiveOrderBook] %d %s orders are not cancelled yet:", b.NumOfOrders(), b.Symbol) - b.Print() + log.Warnf("[ActiveOrderBook] %d %s orders are not cancelled yet:", b.NumOfOrders(), b.Symbol) + b.Print() + + } else { + existingOrders := b.filterExistingOrders(orders) + if len(existingOrders) == 0 { + break + } + } // verify the current open orders via the RESTful API - log.Warnf("[ActiveOrderBook] using REStful API to verify active orders...") + log.Warnf("[ActiveOrderBook] using open orders API to verify the active orders...") var symbolOrdersMap = categorizeOrderBySymbol(orders) @@ -252,17 +264,8 @@ func (b *ActiveOrderBook) orderUpdateHandler(order types.Order) { func (b *ActiveOrderBook) Print() { orders := b.orders.Orders() - - // sort orders by price - sort.Slice(orders, func(i, j int) bool { - o1 := orders[i] - o2 := orders[j] - return o1.Price.Compare(o2.Price) > 0 - }) - - for _, o := range orders { - log.Infof("%s", o) - } + orders = types.SortOrdersByPrice(orders, true) + orders.Print() } // Update updates the order by the order status and emit the related events. @@ -448,6 +451,16 @@ func (b *ActiveOrderBook) Lookup(f func(o types.Order) bool) *types.Order { return b.orders.Lookup(f) } +func (b *ActiveOrderBook) filterExistingOrders(orders []types.Order) (existingOrders types.OrderSlice) { + for _, o := range orders { + if b.Exists(o) { + existingOrders.Add(o) + } + } + + return existingOrders +} + func categorizeOrderBySymbol(orders types.OrderSlice) map[string]types.OrderSlice { orderMap := map[string]types.OrderSlice{} diff --git a/pkg/types/ordermap.go b/pkg/types/ordermap.go index b00da9bff2..af63dd7a74 100644 --- a/pkg/types/ordermap.go +++ b/pkg/types/ordermap.go @@ -3,6 +3,8 @@ package types import ( "sync" "time" + + "github.com/sirupsen/logrus" ) // OrderMap is used for storing orders by their order id @@ -275,3 +277,9 @@ func (s OrderSlice) SeparateBySide() (buyOrders, sellOrders []Order) { return buyOrders, sellOrders } + +func (s OrderSlice) Print() { + for _, o := range s { + logrus.Infof("%s", o) + } +} From c870defd4791e7f6a47f0378fe50d9c45c0abca1 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 13 Dec 2023 16:29:07 +0800 Subject: [PATCH 308/422] xdepthmaker: improve shutdown process --- pkg/bbgo/activeorderbook.go | 14 ++++++++++++-- pkg/strategy/xdepthmaker/strategy.go | 14 +++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/bbgo/activeorderbook.go b/pkg/bbgo/activeorderbook.go index e33830e1b5..6234c56930 100644 --- a/pkg/bbgo/activeorderbook.go +++ b/pkg/bbgo/activeorderbook.go @@ -199,14 +199,23 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, // since ctx might be canceled, we should use background context here if err := ex.CancelOrders(context.Background(), orders...); err != nil { - log.WithError(err).Errorf("[ActiveOrderBook] can not cancel %s orders", b.Symbol) + log.WithError(err).Warnf("[ActiveOrderBook] can not cancel %s orders", b.Symbol) } log.Debugf("[ActiveOrderBook] waiting %s for %s orders to be cancelled...", waitTime, b.Symbol) if cancelAll { clear, err := b.waitAllClear(ctx, waitTime, orderCancelTimeout) - if clear || err != nil { + if err != nil { + if !errors.Is(err, context.Canceled) { + log.WithError(err).Errorf("order cancel error") + } + + break + } + + if clear { + log.Debugf("[ActiveOrderBook] %s orders are canceled", b.Symbol) break } @@ -216,6 +225,7 @@ func (b *ActiveOrderBook) GracefulCancel(ctx context.Context, ex types.Exchange, } else { existingOrders := b.filterExistingOrders(orders) if len(existingOrders) == 0 { + log.Debugf("[ActiveOrderBook] orders are canceled") break } } diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 5bdf04cb08..a553c98242 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -474,14 +474,11 @@ func (s *Strategy) CrossRun( // wait for the quoter to stop time.Sleep(s.UpdateInterval.Duration()) - shutdownCtx, cancelShutdown := context.WithTimeout(context.TODO(), time.Minute) - defer cancelShutdown() - - if err := s.MakerOrderExecutor.GracefulCancel(shutdownCtx); err != nil { + if err := s.MakerOrderExecutor.GracefulCancel(ctx); err != nil { log.WithError(err).Errorf("graceful cancel %s order error", s.Symbol) } - if err := s.HedgeOrderExecutor.GracefulCancel(shutdownCtx); err != nil { + if err := s.HedgeOrderExecutor.GracefulCancel(ctx); err != nil { log.WithError(err).Errorf("graceful cancel %s order error", s.Symbol) } @@ -890,6 +887,13 @@ func (s *Strategy) cleanUpOpenOrders(ctx context.Context) error { return err } + if len(openOrders) == 0 { + return nil + } + + log.Infof("found existing open orders:") + types.OrderSlice(openOrders).Print() + if err := s.makerSession.Exchange.CancelOrders(ctx, openOrders...); err != nil { return err } From e86b1bb90feeadf8cafad865f967862165fd5646 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 13 Dec 2023 17:36:30 +0800 Subject: [PATCH 309/422] REFACTOR: make all common.Strategy from pointer to value --- pkg/strategy/atrpin/strategy.go | 3 +-- pkg/strategy/dca2/strategy.go | 3 +-- pkg/strategy/fixedmaker/strategy.go | 3 +-- pkg/strategy/liquiditymaker/strategy.go | 3 +-- pkg/strategy/random/strategy.go | 3 +-- pkg/strategy/rsicross/strategy.go | 5 ++--- pkg/strategy/scmaker/strategy.go | 3 +-- pkg/strategy/wall/strategy.go | 3 +-- pkg/strategy/xfixedmaker/strategy.go | 3 +-- 9 files changed, 10 insertions(+), 19 deletions(-) diff --git a/pkg/strategy/atrpin/strategy.go b/pkg/strategy/atrpin/strategy.go index cd817fa693..14653f4cba 100644 --- a/pkg/strategy/atrpin/strategy.go +++ b/pkg/strategy/atrpin/strategy.go @@ -20,7 +20,7 @@ func init() { } type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -62,7 +62,6 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) atr := session.Indicators(s.Symbol).ATR(s.Interval, s.Window) diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 597eb69da4..a91d3fd3b9 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -26,7 +26,7 @@ func init() { } type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -101,7 +101,6 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) instanceID := s.InstanceID() diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 48748370df..7b0f6fec92 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -23,7 +23,7 @@ func init() { // Fixed spread market making strategy type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -77,7 +77,6 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 9d90e8fedf..21af78dd40 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -32,7 +32,7 @@ func init() { // - place enough total liquidity amount on the order book, for example, 20k USDT value liquidity on both sell and buy // - ensure the spread by placing the orders from the mid price (or the last trade price) type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -81,7 +81,6 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.orderGenerator = &LiquidityOrderGenerator{ diff --git a/pkg/strategy/random/strategy.go b/pkg/strategy/random/strategy.go index 96b56cb5a6..40ed6026b9 100644 --- a/pkg/strategy/random/strategy.go +++ b/pkg/strategy/random/strategy.go @@ -23,7 +23,7 @@ func init() { } type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -67,7 +67,6 @@ func (s *Strategy) Validate() error { func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, s.ID(), s.InstanceID()) session.UserDataStream.OnStart(func() { diff --git a/pkg/strategy/rsicross/strategy.go b/pkg/strategy/rsicross/strategy.go index eb6c055eb2..0993201eb6 100644 --- a/pkg/strategy/rsicross/strategy.go +++ b/pkg/strategy/rsicross/strategy.go @@ -9,7 +9,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/indicator/v2" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" ) @@ -21,7 +21,7 @@ func init() { } type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -49,7 +49,6 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) fastRsi := session.Indicators(s.Symbol).RSI(types.IntervalWindow{Interval: s.Interval, Window: s.FastWindow}) diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index 79e3f89569..fffe9cbc23 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -34,7 +34,7 @@ func init() { // Strategy scmaker is a stable coin market maker type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -88,7 +88,6 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) s.book = types.NewStreamBook(s.Symbol) diff --git a/pkg/strategy/wall/strategy.go b/pkg/strategy/wall/strategy.go index 0967b241a8..83b825f111 100644 --- a/pkg/strategy/wall/strategy.go +++ b/pkg/strategy/wall/strategy.go @@ -30,7 +30,7 @@ func init() { } type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment Market types.Market @@ -235,7 +235,6 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) // initial required information diff --git a/pkg/strategy/xfixedmaker/strategy.go b/pkg/strategy/xfixedmaker/strategy.go index 2e6f927f14..be5ca3d3fa 100644 --- a/pkg/strategy/xfixedmaker/strategy.go +++ b/pkg/strategy/xfixedmaker/strategy.go @@ -23,7 +23,7 @@ func init() { // Fixed spread market making strategy type Strategy struct { - *common.Strategy + common.Strategy Environment *bbgo.Environment @@ -109,7 +109,6 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se } s.market = market - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, tradingSession, s.market, ID, s.InstanceID()) s.orderPriceRiskControl = NewOrderPriceRiskControl( From 84a43cfd9caf42da91213fc02be00ea429d5ea19 Mon Sep 17 00:00:00 2001 From: Shubham Singodiya Date: Wed, 13 Dec 2023 21:14:07 +0530 Subject: [PATCH 310/422] DOCS: Fix grammatical errors --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9ec4117ec3..f217d8153f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ You can use BBGO's trading unit and back-test unit to implement your own strateg ### Trading Unit Developers 🧑‍💻 -You can use BBGO's underlying common exchange API, currently, it supports 4+ major exchanges, so you don't have to repeat +You can use BBGO's underlying common exchange API; currently, it supports 4+ major exchanges, so you don't have to repeat the implementation. ## Features @@ -150,8 +150,8 @@ the implementation. - OKEx: - Kucoin: - This project is maintained and supported by a small group of team. If you would like to support this project, please - register on the exchanges using the provided links with referral codes above. + This project is maintained and supported by a small group of people. If you would like to support this project, please + Register on the exchanges using the provided links with the referral codes above. ## Installation From 84b34e0cf79b7943296f37d56ef63639769204f2 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 14 Dec 2023 18:05:02 +0800 Subject: [PATCH 311/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index f1b9d11eb7..9395889ffd 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index e61e5722d2..89e29cf466 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index 4ab2095b8f..518f415f4c 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 1d5e392faf..d01f3aec42 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 781473e7f7..834c37e0b6 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index b3131ea436..59fd787b5c 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index bd1a9b7b98..d7bc28e1fe 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index c565b09a93..736b31ad48 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index 5cd7a3e059..27f99af514 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index efa33d4e98..d4aff06ef8 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index 1631ba30f2..e43e914341 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index 283b7bca61..cc67692a3a 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index ee1cb005c1..8ef3e0137c 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index f1a84f9921..7f46a64ad6 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index 431a0fddf1..0cc1f4356a 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index 73e03cf776..fd40b0705d 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index d8332b8002..2337124081 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 01897691bf..2991bf1608 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index 069fe058ea..a458ee7d2e 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index f38771ed93..52c6de9283 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index e16aaea684..9386ba91d0 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 994636657d..722a2426f4 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 9889f099ed..add34532f9 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 55699bda45..9fe08e8886 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index f15732d310..8113abce49 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index 5537cc2ba5..b8ea309357 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index 502ae9f909..33a1c193e3 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index cc7d14dd1d..13cf5da5a9 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 7755ad0291..46d368940e 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 20-Nov-2023 +###### Auto generated by spf13/cobra on 14-Dec-2023 From 8690977b5c7aefdd5b71163c2cccb61f5a4ed18a Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 14 Dec 2023 18:05:02 +0800 Subject: [PATCH 312/422] bump version to v1.55.0 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index 491a8bbf2c..e1fc80f2b9 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.54.0-da3150e2-dev" +const Version = "v1.55.0-2c7e4292-dev" -const VersionGitRef = "da3150e2" +const VersionGitRef = "2c7e4292" diff --git a/pkg/version/version.go b/pkg/version/version.go index c93b147b6c..8d508cb447 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.54.0-da3150e2" +const Version = "v1.55.0-2c7e4292" -const VersionGitRef = "da3150e2" +const VersionGitRef = "2c7e4292" From 6d7ff54591f2978139165443e4d40d6e39975684 Mon Sep 17 00:00:00 2001 From: c9s Date: Thu, 14 Dec 2023 18:05:02 +0800 Subject: [PATCH 313/422] add v1.55.0 release note --- doc/release/v1.55.0.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/release/v1.55.0.md diff --git a/doc/release/v1.55.0.md b/doc/release/v1.55.0.md new file mode 100644 index 0000000000..abecf140dc --- /dev/null +++ b/doc/release/v1.55.0.md @@ -0,0 +1,28 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.54.0...main) + + - [#1429](https://github.com/c9s/bbgo/pull/1429): CHORE: [bybit] we don't need the fee rate in the public stream + - [#1452](https://github.com/c9s/bbgo/pull/1452): REFACTOR: make all common.Strategy from pointer to value + - [#1451](https://github.com/c9s/bbgo/pull/1451): CHORE: [xdepthmaker] improve shutdown process + - [#1450](https://github.com/c9s/bbgo/pull/1450): IMPROVE: [strategy] xdepthmaker final fine-tune + - [#1436](https://github.com/c9s/bbgo/pull/1436): FEATURE: cancel maker orders and open take profit order + - [#1449](https://github.com/c9s/bbgo/pull/1449): build(deps): bump @babel/traverse from 7.18.5 to 7.23.6 in /apps/frontend + - [#1448](https://github.com/c9s/bbgo/pull/1448): FIX: [core] solve memory leaks + - [#1447](https://github.com/c9s/bbgo/pull/1447): FIX: [bitget] fix price is zero when order not executed + - [#1446](https://github.com/c9s/bbgo/pull/1446): IMPROVE: [bitget] add more debug logs + - [#1445](https://github.com/c9s/bbgo/pull/1445): IMPROVE: [xdepthmaker] add more improvements + - [#1444](https://github.com/c9s/bbgo/pull/1444): FEATURE: upgrade to go 1.20 + - [#1443](https://github.com/c9s/bbgo/pull/1443): IMPROVE: [bitget] improve order type handling + - [#1441](https://github.com/c9s/bbgo/pull/1441): FIX: fix since time override + - [#1440](https://github.com/c9s/bbgo/pull/1440): FIX: [indicator] Possibly incorrect assignment + - [#1438](https://github.com/c9s/bbgo/pull/1438): STRATEGY: add xdepthmaker strategy + - [#1433](https://github.com/c9s/bbgo/pull/1433): FEATURE: prepare open maker orders function + - [#1428](https://github.com/c9s/bbgo/pull/1428): FEATURE: use max v3 new open orders api + - [#1439](https://github.com/c9s/bbgo/pull/1439): FIX: fix list closed orders api limit + - [#1432](https://github.com/c9s/bbgo/pull/1432): FIX: use original status for recover + - [#1431](https://github.com/c9s/bbgo/pull/1431): FIX: fix order status length + - [#1437](https://github.com/c9s/bbgo/pull/1437): FIX: deactivate exit when position in closing + - [#1430](https://github.com/c9s/bbgo/pull/1430): FIX: add executed quantity check when order status is partially filled + - [#1411](https://github.com/c9s/bbgo/pull/1411): FEATURE: new strategy dca2 perparation + - [#1405](https://github.com/c9s/bbgo/pull/1405): FIX: [grid2] use rest quote to place the last order when opening grid + - [#1421](https://github.com/c9s/bbgo/pull/1421): FEATURE: use new max v3 api to query closed orders by timestamp + - [#1427](https://github.com/c9s/bbgo/pull/1427): CHORE: add log rate limiter to stream event and use backoff retry on bybit From e7c3582334e69c858fa6d52310ab4042f21c3a64 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Dec 2023 19:19:06 +0800 Subject: [PATCH 314/422] fix: import tzdata package --- pkg/cmd/root.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 417e6196ce..b9fe59f53a 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -21,6 +21,8 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/util" + _ "time/tzdata" + _ "github.com/go-sql-driver/mysql" ) From 2a8124cf058f185e62653227eb95e4015572f208 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Dec 2023 19:20:01 +0800 Subject: [PATCH 315/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index 9395889ffd..df9b769cde 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index 89e29cf466..74a36e07ef 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index 518f415f4c..ca0f349436 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index d01f3aec42..926868f5ea 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 834c37e0b6..36be986e81 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 59fd787b5c..d0031a57c9 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index d7bc28e1fe..7df217fd49 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 736b31ad48..d16ae9ceb1 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index 27f99af514..1136a2ecba 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index d4aff06ef8..cd7fe66b5a 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index e43e914341..d5e51c0234 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index cc67692a3a..af0c2277ba 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 8ef3e0137c..97e8dfa0ab 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index 7f46a64ad6..38afcf7f9c 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index 0cc1f4356a..2677bf40d0 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index fd40b0705d..54f80db122 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index 2337124081..bd3145987b 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 2991bf1608..9f28558903 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index a458ee7d2e..d0a5b7d16e 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index 52c6de9283..ef22097563 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index 9386ba91d0..f981f44a06 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 722a2426f4..04dd8e60bd 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index add34532f9..650a86c029 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 9fe08e8886..fecd3f72b1 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index 8113abce49..b9fba9f6c7 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index b8ea309357..dfece68677 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index 33a1c193e3..3078c4c2f0 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index 13cf5da5a9..8dcda2e989 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 46d368940e..c41908e4c1 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 14-Dec-2023 +###### Auto generated by spf13/cobra on 15-Dec-2023 From 19636ae429945310d2467257538e8fbaaf4e6a14 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Dec 2023 19:20:01 +0800 Subject: [PATCH 316/422] bump version to v1.55.1 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index e1fc80f2b9..e99f9a6970 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.0-2c7e4292-dev" +const Version = "v1.55.1-e7c35823-dev" -const VersionGitRef = "2c7e4292" +const VersionGitRef = "e7c35823" diff --git a/pkg/version/version.go b/pkg/version/version.go index 8d508cb447..51fcdb304a 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.0-2c7e4292" +const Version = "v1.55.1-e7c35823" -const VersionGitRef = "2c7e4292" +const VersionGitRef = "e7c35823" From 04661652588296280daad161ca95e870c6d87077 Mon Sep 17 00:00:00 2001 From: c9s Date: Fri, 15 Dec 2023 19:20:01 +0800 Subject: [PATCH 317/422] add v1.55.1 release note --- doc/release/v1.55.1.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release/v1.55.1.md diff --git a/doc/release/v1.55.1.md b/doc/release/v1.55.1.md new file mode 100644 index 0000000000..b29324c7c1 --- /dev/null +++ b/doc/release/v1.55.1.md @@ -0,0 +1,6 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.55.0...main) + +## Fixes + +- import tzdata package + From 3e6d6e10b3f5326cff3e6c32468a417f19cf0d7c Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 12:09:03 +0800 Subject: [PATCH 318/422] all: move Initialize() call out, call it before the LoadState --- cmd/bbgo-webview/main.go | 5 +++++ pkg/bbgo/trader.go | 21 ++++++++++++--------- pkg/cmd/run.go | 4 ++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/cmd/bbgo-webview/main.go b/cmd/bbgo-webview/main.go index 7cc5060e2c..e863c2c611 100644 --- a/cmd/bbgo-webview/main.go +++ b/cmd/bbgo-webview/main.go @@ -109,6 +109,11 @@ func main() { return } + if err := trader.Initialize(ctx); err != nil { + log.WithError(err).Error("failed to initialize strategies") + return + } + if err := trader.LoadState(ctx); err != nil { log.WithError(err).Error("failed to load strategy states") return diff --git a/pkg/bbgo/trader.go b/pkg/bbgo/trader.go index 6c21f1a4d7..44be5c2a06 100644 --- a/pkg/bbgo/trader.go +++ b/pkg/bbgo/trader.go @@ -16,8 +16,8 @@ import ( ) // Strategy method calls: -// -> Defaults() (optional method) // -> Initialize() (optional method) +// -> Defaults() (optional method) // -> Validate() (optional method) // -> Run() (optional method) // -> Shutdown(shutdownCtx context.Context, wg *sync.WaitGroup) @@ -244,12 +244,6 @@ func (trader *Trader) injectFieldsAndSubscribe(ctx context.Context) error { } } - if initializer, ok := strategy.(StrategyInitializer); ok { - if err := initializer.Initialize(); err != nil { - panic(err) - } - } - if subscriber, ok := strategy.(ExchangeSessionSubscriber); ok { subscriber.Subscribe(session) } else { @@ -362,17 +356,26 @@ func (trader *Trader) Run(ctx context.Context) error { return trader.environment.Connect(ctx) } +func (trader *Trader) Initialize(ctx context.Context) error { + log.Infof("initializing strategies...") + return trader.IterateStrategies(func(strategy StrategyID) error { + if initializer, ok := strategy.(StrategyInitializer); ok { + return initializer.Initialize() + } + + return nil + }) +} + func (trader *Trader) LoadState(ctx context.Context) error { if trader.environment.BacktestService != nil { return nil } isolation := GetIsolationFromContext(ctx) - ps := isolation.persistenceServiceFacade.Get() log.Infof("loading strategies states...") - return trader.IterateStrategies(func(strategy StrategyID) error { id := dynamic.CallID(strategy) return loadPersistenceFields(strategy, id, ps) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index bc601e0dc2..92e1bbe8a2 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -163,6 +163,10 @@ func runConfig(basectx context.Context, cmd *cobra.Command, userConfig *bbgo.Con return err } + if err := trader.Initialize(tradingCtx); err != nil { + return err + } + if err := trader.LoadState(tradingCtx); err != nil { return err } From c5decf9bf804905a7176e9b458a803fcd672e17f Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 18 Dec 2023 12:17:49 +0800 Subject: [PATCH 319/422] pkg/exchange: support v2 get asset api --- .../bitget/bitgetapi/v2/client_test.go | 6 + .../v2/get_account_assets_request.go | 39 ++++ .../get_account_assets_request_requestgen.go | 195 ++++++++++++++++++ 3 files changed, 240 insertions(+) create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go create mode 100644 pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request_requestgen.go diff --git a/pkg/exchange/bitget/bitgetapi/v2/client_test.go b/pkg/exchange/bitget/bitgetapi/v2/client_test.go index f8eef7862b..3873b1479f 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/client_test.go +++ b/pkg/exchange/bitget/bitgetapi/v2/client_test.go @@ -101,4 +101,10 @@ func TestClient(t *testing.T) { assert.NoError(t, err) t.Logf("resp: %+v", resp) }) + + t.Run("GetAccountAssetsRequest", func(t *testing.T) { + resp, err := client.NewGetAccountAssetsRequest().AssetType(AssetTypeHoldOnly).Do(ctx) + assert.NoError(t, err) + t.Logf("resp: %+v", resp) + }) } diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go new file mode 100644 index 0000000000..d87468143e --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go @@ -0,0 +1,39 @@ +package bitgetapi + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type AccountAsset struct { + Coin string `json:"coin"` + Available fixedpoint.Value `json:"available"` + Frozen fixedpoint.Value `json:"frozen"` + Locked fixedpoint.Value `json:"locked"` + LimitAvailable fixedpoint.Value `json:"limitAvailable"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` +} + +type AssetType string + +const ( + AssetTypeHoldOnly AssetType = "hold_only" + AssetTypeHAll AssetType = "all" +) + +//go:generate GetRequest -url "/api/v2/spot/account/assets" -type GetAccountAssetsRequest -responseDataType []AccountAsset +type GetAccountAssetsRequest struct { + client requestgen.AuthenticatedAPIClient + + coin *string `param:"symbol,query"` + assetType AssetType `param:"limit,query"` +} + +func (c *Client) NewGetAccountAssetsRequest() *GetAccountAssetsRequest { + return &GetAccountAssetsRequest{client: c.Client} +} diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request_requestgen.go new file mode 100644 index 0000000000..945b25675f --- /dev/null +++ b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request_requestgen.go @@ -0,0 +1,195 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v2/spot/account/assets -type GetAccountAssetsRequest -responseDataType []AccountAsset"; DO NOT EDIT. + +package bitgetapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "net/url" + "reflect" + "regexp" +) + +func (c *GetAccountAssetsRequest) Coin(coin string) *GetAccountAssetsRequest { + c.coin = &coin + return c +} + +func (c *GetAccountAssetsRequest) AssetType(assetType AssetType) *GetAccountAssetsRequest { + c.assetType = assetType + return c +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (c *GetAccountAssetsRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check coin field -> json key symbol + if c.coin != nil { + coin := *c.coin + + // assign parameter of coin + params["symbol"] = coin + } else { + } + // check assetType field -> json key limit + assetType := c.assetType + + // TEMPLATE check-valid-values + switch assetType { + case AssetTypeHoldOnly, AssetTypeHAll: + params["limit"] = assetType + + default: + return nil, fmt.Errorf("limit value %v is invalid", assetType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of assetType + params["limit"] = assetType + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (c *GetAccountAssetsRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (c *GetAccountAssetsRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := c.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if c.isVarSlice(_v) { + c.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (c *GetAccountAssetsRequest) GetParametersJSON() ([]byte, error) { + params, err := c.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (c *GetAccountAssetsRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (c *GetAccountAssetsRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (c *GetAccountAssetsRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (c *GetAccountAssetsRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (c *GetAccountAssetsRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := c.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (c *GetAccountAssetsRequest) GetPath() string { + return "/api/v2/spot/account/assets" +} + +// Do generates the request object and send the request object to the API endpoint +func (c *GetAccountAssetsRequest) Do(ctx context.Context) ([]AccountAsset, error) { + + // no body params + var params interface{} + query, err := c.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = c.GetPath() + + req, err := c.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := c.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse bitgetapi.APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []AccountAsset + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} From f19ed7abe0269b5a514ab49da9cb4d141ee070d2 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 14:31:51 +0800 Subject: [PATCH 320/422] xdepthmaker: initialize s.CrossExchangeMarketMakingStrategy in Initialize() --- pkg/strategy/xdepthmaker/strategy.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index a553c98242..d947a3dc42 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -249,6 +249,16 @@ func (s *Strategy) InstanceID() string { return fmt.Sprintf("%s:%s", ID, s.Symbol) } +func (s *Strategy) Initialize() error { + if s.CrossExchangeMarketMakingStrategy == nil { + s.CrossExchangeMarketMakingStrategy = &CrossExchangeMarketMakingStrategy{} + } + + s.bidPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) + s.askPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) + return nil +} + func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { makerSession, hedgeSession, err := selectSessions2(sessions, s.MakerExchange, s.HedgeExchange) if err != nil { @@ -325,12 +335,6 @@ func (s *Strategy) Defaults() error { return nil } -func (s *Strategy) Initialize() error { - s.bidPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) - s.askPriceHeartBeat = types.NewPriceHeartBeat(priceUpdateTimeout) - return nil -} - func (s *Strategy) CrossRun( ctx context.Context, _ bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession, @@ -340,7 +344,6 @@ func (s *Strategy) CrossRun( return err } - s.CrossExchangeMarketMakingStrategy = &CrossExchangeMarketMakingStrategy{} if err := s.CrossExchangeMarketMakingStrategy.Initialize(ctx, s.Environment, makerSession, hedgeSession, s.Symbol, ID, s.InstanceID()); err != nil { return err } From 038d1807111fa0f174c1b882ea74c0cefd02edaf Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 14:44:33 +0800 Subject: [PATCH 321/422] bitget: check bitget websocket trade id and order status --- pkg/exchange/bitget/stream.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 4d8ec96a86..80f80a8a0b 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -432,13 +432,19 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { } continue } + // The bitget support only snapshot on orders channel, so we use snapshot as update to emit data. if m.actionType != ActionTypeSnapshot { continue } s.StandardStream.EmitOrderUpdate(globalOrder) - if globalOrder.Status == types.OrderStatusPartiallyFilled { + if order.TradeId == 0 { + continue + } + + switch globalOrder.Status { + case types.OrderStatusPartiallyFilled, types.OrderStatusFilled: trade, err := order.toGlobalTrade() if err != nil { if tradeLogLimiter.Allow() { @@ -446,6 +452,7 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { } continue } + s.StandardStream.EmitTradeUpdate(trade) } } From eda072327c415c847a9804147661db0439cb9a94 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 18 Dec 2023 14:48:13 +0800 Subject: [PATCH 322/422] FIX: move common.Strategy to Initialize --- pkg/strategy/atrpin/strategy.go | 7 ++++++- pkg/strategy/dca2/strategy.go | 3 ++- pkg/strategy/fixedmaker/strategy.go | 4 +++- pkg/strategy/liquiditymaker/strategy.go | 7 ++++++- pkg/strategy/random/strategy.go | 3 ++- pkg/strategy/rsicross/strategy.go | 7 ++++++- pkg/strategy/scmaker/strategy.go | 7 ++++++- pkg/strategy/wall/strategy.go | 7 ++++++- pkg/strategy/xfixedmaker/strategy.go | 4 +++- 9 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pkg/strategy/atrpin/strategy.go b/pkg/strategy/atrpin/strategy.go index 14653f4cba..836969e0e4 100644 --- a/pkg/strategy/atrpin/strategy.go +++ b/pkg/strategy/atrpin/strategy.go @@ -20,7 +20,7 @@ func init() { } type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -36,6 +36,11 @@ type Strategy struct { // bbgo.OpenPositionOptions } +func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} + return nil +} + func (s *Strategy) ID() string { return ID } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index a91d3fd3b9..0cf17fbf44 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -26,7 +26,7 @@ func init() { } type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -89,6 +89,7 @@ func (s *Strategy) Defaults() error { func (s *Strategy) Initialize() error { s.logger = log.WithFields(s.LogFields) + s.Strategy = &common.Strategy{} return nil } diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 7b0f6fec92..3585e4da1f 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -23,7 +23,7 @@ func init() { // Fixed spread market making strategy type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -45,7 +45,9 @@ func (s *Strategy) Defaults() error { } return nil } + func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} return nil } diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 21af78dd40..47ecec89c9 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -32,7 +32,7 @@ func init() { // - place enough total liquidity amount on the order book, for example, 20k USDT value liquidity on both sell and buy // - ensure the spread by placing the orders from the mid price (or the last trade price) type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -66,6 +66,11 @@ type Strategy struct { orderGenerator *LiquidityOrderGenerator } +func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} + return nil +} + func (s *Strategy) ID() string { return ID } diff --git a/pkg/strategy/random/strategy.go b/pkg/strategy/random/strategy.go index 40ed6026b9..a9ccd98882 100644 --- a/pkg/strategy/random/strategy.go +++ b/pkg/strategy/random/strategy.go @@ -23,7 +23,7 @@ func init() { } type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -42,6 +42,7 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} return nil } diff --git a/pkg/strategy/rsicross/strategy.go b/pkg/strategy/rsicross/strategy.go index 0993201eb6..0eb0ec5ecc 100644 --- a/pkg/strategy/rsicross/strategy.go +++ b/pkg/strategy/rsicross/strategy.go @@ -21,7 +21,7 @@ func init() { } type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -36,6 +36,11 @@ type Strategy struct { bbgo.OpenPositionOptions } +func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} + return nil +} + func (s *Strategy) ID() string { return ID } diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index fffe9cbc23..7ad5042c2e 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -34,7 +34,7 @@ func init() { // Strategy scmaker is a stable coin market maker type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -69,6 +69,11 @@ type Strategy struct { intensity *IntensityStream } +func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} + return nil +} + func (s *Strategy) ID() string { return ID } diff --git a/pkg/strategy/wall/strategy.go b/pkg/strategy/wall/strategy.go index 83b825f111..b097c9b427 100644 --- a/pkg/strategy/wall/strategy.go +++ b/pkg/strategy/wall/strategy.go @@ -30,7 +30,7 @@ func init() { } type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment Market types.Market @@ -64,6 +64,11 @@ type Strategy struct { activeWallOrders *bbgo.ActiveOrderBook } +func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} + return nil +} + func (s *Strategy) ID() string { return ID } diff --git a/pkg/strategy/xfixedmaker/strategy.go b/pkg/strategy/xfixedmaker/strategy.go index be5ca3d3fa..2e018f5718 100644 --- a/pkg/strategy/xfixedmaker/strategy.go +++ b/pkg/strategy/xfixedmaker/strategy.go @@ -23,7 +23,7 @@ func init() { // Fixed spread market making strategy type Strategy struct { - common.Strategy + *common.Strategy Environment *bbgo.Environment @@ -51,7 +51,9 @@ func (s *Strategy) Defaults() error { } return nil } + func (s *Strategy) Initialize() error { + s.Strategy = &common.Strategy{} return nil } From 92aa7652d58978633546aac40d53a8c8f40d14e3 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 15:49:20 +0800 Subject: [PATCH 323/422] bbgo: add recordPosition log --- pkg/bbgo/environment.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index e4c67de1a3..decd3c06ea 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -565,7 +565,7 @@ func (environ *Environment) RecordPosition(position *types.Position, trade types return } - // set profit info to position + // guard: set profit info to position if the strategy info is empty if profit != nil { if position.Strategy == "" && profit.Strategy != "" { position.Strategy = profit.Strategy @@ -576,10 +576,12 @@ func (environ *Environment) RecordPosition(position *types.Position, trade types } } + log.Infof("recordPosition: position = %s, trade = %+v, profit = %+v", position.Base.String(), trade, profit) if profit != nil { if err := environ.PositionService.Insert(position, trade, profit.Profit); err != nil { log.WithError(err).Errorf("can not insert position record") } + if err := environ.ProfitService.Insert(*profit); err != nil { log.WithError(err).Errorf("can not insert profit record: %+v", profit) } From 841229518a7e61ad6c29b7ba3af40959fd6ddb73 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 16:31:04 +0800 Subject: [PATCH 324/422] bitget: add more debug logs for orderEvent and tradeEvent --- pkg/exchange/bitget/debug.go | 19 +++++++++++++++++++ pkg/exchange/bitget/exchange.go | 17 ----------------- pkg/exchange/bitget/stream.go | 7 +++++++ 3 files changed, 26 insertions(+), 17 deletions(-) create mode 100644 pkg/exchange/bitget/debug.go diff --git a/pkg/exchange/bitget/debug.go b/pkg/exchange/bitget/debug.go new file mode 100644 index 0000000000..29a50b6ad1 --- /dev/null +++ b/pkg/exchange/bitget/debug.go @@ -0,0 +1,19 @@ +package bitget + +import "github.com/c9s/bbgo/pkg/util" + +type LogFunction func(msg string, args ...interface{}) + +var debugf LogFunction + +func getDebugFunction() LogFunction { + if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { + return log.Infof + } + + return func(msg string, args ...interface{}) {} +} + +func init() { + debugf = getDebugFunction() +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index dc0ca8e7ab..d10d7ce8e2 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -14,7 +14,6 @@ import ( "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" ) const ( @@ -64,22 +63,6 @@ var ( kLineRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) ) -type LogFunction func(msg string, args ...interface{}) - -var debugf LogFunction - -func getDebugFunction() LogFunction { - if v, ok := util.GetEnvVarBool("DEBUG_BITGET"); ok && v { - return log.Infof - } - - return func(msg string, args ...interface{}) {} -} - -func init() { - debugf = getDebugFunction() -} - type Exchange struct { key, secret, passphrase string diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 80f80a8a0b..2c99d265da 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -424,7 +424,11 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { return } + debugf("received OrderTradeEvent: %+v", m) + for _, order := range m.Orders { + debugf("received Order: %+v", order) + globalOrder, err := order.toGlobalOrder() if err != nil { if orderLogLimiter.Allow() { @@ -437,12 +441,15 @@ func (s *Stream) handleOrderTradeEvent(m OrderTradeEvent) { if m.actionType != ActionTypeSnapshot { continue } + s.StandardStream.EmitOrderUpdate(globalOrder) if order.TradeId == 0 { continue } + debugf("received Trade: %+v", order.Trade) + switch globalOrder.Status { case types.OrderStatusPartiallyFilled, types.OrderStatusFilled: trade, err := order.toGlobalTrade() From 63b26257f9ba3623b345b9bdb8820da5567d7432 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 18:00:40 +0800 Subject: [PATCH 325/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index df9b769cde..c7becef6ab 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index 74a36e07ef..de10c300e7 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index ca0f349436..cf74aac091 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 926868f5ea..0d73e38eeb 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index 36be986e81..f88eabf6ff 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index d0031a57c9..62e24a39cf 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index 7df217fd49..61765bec17 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index d16ae9ceb1..054282c434 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index 1136a2ecba..655b7b1db4 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index cd7fe66b5a..b4d8a4c86c 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index d5e51c0234..787cc3aad0 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index af0c2277ba..b3611eb70d 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 97e8dfa0ab..917edfa81f 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index 38afcf7f9c..b31e498fa9 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index 2677bf40d0..c13eb042a2 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index 54f80db122..ab8bbc930b 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index bd3145987b..5519dbe935 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 9f28558903..26e3ac5a0a 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index d0a5b7d16e..2e3dc3a6d7 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index ef22097563..2c79b7b303 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index f981f44a06..cf60397934 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index 04dd8e60bd..ae2792f44d 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index 650a86c029..cec954d4d1 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index fecd3f72b1..21a31271c9 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index b9fba9f6c7..b2410eb61d 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index dfece68677..34c55f3176 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index 3078c4c2f0..62e7583dfc 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index 8dcda2e989..ea920b7582 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index c41908e4c1..8a5961466f 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 15-Dec-2023 +###### Auto generated by spf13/cobra on 18-Dec-2023 From 3ac862d122536fbe470496065b3f209f8695ea7f Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 18:00:40 +0800 Subject: [PATCH 326/422] bump version to v1.55.2 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index e99f9a6970..f40ddf537c 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.1-e7c35823-dev" +const Version = "v1.55.2-98468feb-dev" -const VersionGitRef = "e7c35823" +const VersionGitRef = "98468feb" diff --git a/pkg/version/version.go b/pkg/version/version.go index 51fcdb304a..10caa2e32e 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.1-e7c35823" +const Version = "v1.55.2-98468feb" -const VersionGitRef = "e7c35823" +const VersionGitRef = "98468feb" From 1361fa519e4f9c51245aa53f17f243bf17c2f783 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 18:00:40 +0800 Subject: [PATCH 327/422] add v1.55.2 release note --- doc/release/v1.55.2.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release/v1.55.2.md diff --git a/doc/release/v1.55.2.md b/doc/release/v1.55.2.md new file mode 100644 index 0000000000..7b0d8f86be --- /dev/null +++ b/doc/release/v1.55.2.md @@ -0,0 +1,6 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.55.1...main) + + - [#1458](https://github.com/c9s/bbgo/pull/1458): FIX: [xdepthmaker] final fix + - [#1459](https://github.com/c9s/bbgo/pull/1459): FIX: move common.Strategy to Initialize + - [#1453](https://github.com/c9s/bbgo/pull/1453): DOC: Fix grammatical errors + - [#1455](https://github.com/c9s/bbgo/pull/1455): FIX: move Initialize() call out, call it before the LoadState From 3ba46f9b7e8c9d0527a576b8e2f36a2f568675c4 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 21:40:29 +0800 Subject: [PATCH 328/422] github: fix release workflow --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98def0eb02..8117f5b018 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,9 +18,9 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: "1.20" - name: Install Node uses: actions/setup-node@v2 with: From 671ce872c474023071d56989da8b4c9b6bb0fe4b Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:01:11 +0800 Subject: [PATCH 329/422] bbgo: fix and improve session UpdatePrice method --- pkg/bbgo/session.go | 38 ++++++++++++++++++++++++++++++++++---- pkg/types/currencies.go | 7 +++++++ pkg/types/market.go | 9 ++++++++- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 6778ce35b3..1c414f6bdd 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -654,7 +654,7 @@ func (session *ExchangeSession) Market(symbol string) (market types.Market, ok b return market, ok } -func (session *ExchangeSession) Markets() map[string]types.Market { +func (session *ExchangeSession) Markets() types.MarketMap { return session.markets } @@ -703,10 +703,30 @@ func (session *ExchangeSession) UpdatePrices(ctx context.Context, currencies []s // return nil // } + markets := session.Markets() + var symbols []string for _, c := range currencies { - symbols = append(symbols, c+fiat) // BTC/USDT - symbols = append(symbols, fiat+c) // USDT/TWD + var tries []string + // expand USD stable coin currencies + if types.IsUSDFiatCurrency(fiat) { + for _, usdFiat := range types.USDFiatCurrencies { + tries = append(tries, c+usdFiat, usdFiat+c) + } + } else { + tries = []string{c + fiat, fiat + c} + } + + for _, try := range tries { + if markets.Has(try) { + symbols = append(symbols, try) + break + } + } + } + + if len(symbols) == 0 { + return nil } tickers, err := session.Exchange.QueryTickers(ctx, symbols...) @@ -717,7 +737,17 @@ func (session *ExchangeSession) UpdatePrices(ctx context.Context, currencies []s var lastTime time.Time for k, v := range tickers { // for {Crypto}/USDT markets - session.lastPrices[k] = v.Last + // map things like BTCUSDT = {price} + if market, ok := markets[k]; ok { + if types.IsFiatCurrency(market.BaseCurrency) { + session.lastPrices[k] = v.Last.Div(fixedpoint.One) + } else { + session.lastPrices[k] = v.Last + } + } else { + session.lastPrices[k] = v.Last + } + if v.Time.After(lastTime) { lastTime = v.Time } diff --git a/pkg/types/currencies.go b/pkg/types/currencies.go index 4e24c3a830..3c5b539e21 100644 --- a/pkg/types/currencies.go +++ b/pkg/types/currencies.go @@ -24,8 +24,15 @@ var USD = wrapper{accounting.Accounting{Symbol: "$ ", Precision: 2}} var BTC = wrapper{accounting.Accounting{Symbol: "BTC ", Precision: 8}} var BNB = wrapper{accounting.Accounting{Symbol: "BNB ", Precision: 4}} +const ( + USDT = "USDT" + USDC = "USDC" + BUSD = "BUSD" +) + var FiatCurrencies = []string{"USDC", "USDT", "USD", "TWD", "EUR", "GBP", "BUSD"} +// USDFiatCurrencies lists the USD stable coins var USDFiatCurrencies = []string{"USDT", "USDC", "USD", "BUSD"} func IsUSDFiatCurrency(currency string) bool { diff --git a/pkg/types/market.go b/pkg/types/market.go index 8829fed73d..5ea42d6d72 100644 --- a/pkg/types/market.go +++ b/pkg/types/market.go @@ -90,7 +90,9 @@ func (m Market) TruncateQuoteQuantity(quantity fixedpoint.Value) fixedpoint.Valu // when side = buy, then available = quote balance // The balance will be truncated first in order to calculate the minimal notional and minimal quantity // The adjusted (truncated) order quantity will be returned -func (m Market) GreaterThanMinimalOrderQuantity(side SideType, price, available fixedpoint.Value) (fixedpoint.Value, bool) { +func (m Market) GreaterThanMinimalOrderQuantity( + side SideType, price, available fixedpoint.Value, +) (fixedpoint.Value, bool) { switch side { case SideTypeSell: available = m.TruncateQuantity(available) @@ -236,3 +238,8 @@ type MarketMap map[string]Market func (m MarketMap) Add(market Market) { m[market.Symbol] = market } + +func (m MarketMap) Has(symbol string) bool { + _, ok := m[symbol] + return ok +} From 882c1273b3678cf8b090f9f0ef00692997c04aa6 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:09:04 +0800 Subject: [PATCH 330/422] bbgo: pull out findPossibleMarketSymbols and add tests --- pkg/bbgo/environment.go | 2 +- pkg/bbgo/session.go | 42 ++++++++++++++++++++++---------------- pkg/bbgo/session_test.go | 44 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 pkg/bbgo/session_test.go diff --git a/pkg/bbgo/environment.go b/pkg/bbgo/environment.go index decd3c06ea..e5c7189e2e 100644 --- a/pkg/bbgo/environment.go +++ b/pkg/bbgo/environment.go @@ -1020,7 +1020,7 @@ func (session *ExchangeSession) getSessionSymbols(defaultSymbols ...string) ([]s return defaultSymbols, nil } - return session.FindPossibleSymbols() + return session.FindPossibleAssetSymbols() } func defaultSyncSinceTime() time.Time { diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 1c414f6bdd..4cfa57c404 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -704,25 +704,10 @@ func (session *ExchangeSession) UpdatePrices(ctx context.Context, currencies []s // } markets := session.Markets() - var symbols []string for _, c := range currencies { - var tries []string - // expand USD stable coin currencies - if types.IsUSDFiatCurrency(fiat) { - for _, usdFiat := range types.USDFiatCurrencies { - tries = append(tries, c+usdFiat, usdFiat+c) - } - } else { - tries = []string{c + fiat, fiat + c} - } - - for _, try := range tries { - if markets.Has(try) { - symbols = append(symbols, try) - break - } - } + possibleSymbols := findPossibleMarketSymbols(markets, c, fiat) + symbols = append(symbols, possibleSymbols...) } if len(symbols) == 0 { @@ -757,7 +742,7 @@ func (session *ExchangeSession) UpdatePrices(ctx context.Context, currencies []s return err } -func (session *ExchangeSession) FindPossibleSymbols() (symbols []string, err error) { +func (session *ExchangeSession) FindPossibleAssetSymbols() (symbols []string, err error) { // If the session is an isolated margin session, there will be only the isolated margin symbol if session.Margin && session.IsolatedMargin { return []string{ @@ -1041,3 +1026,24 @@ func (session *ExchangeSession) FormatOrders(orders []types.SubmitOrder) (format return formattedOrders, err } + +func findPossibleMarketSymbols(markets types.MarketMap, c, fiat string) (symbols []string) { + var tries []string + // expand USD stable coin currencies + if types.IsUSDFiatCurrency(fiat) { + for _, usdFiat := range types.USDFiatCurrencies { + tries = append(tries, c+usdFiat, usdFiat+c) + } + } else { + tries = []string{c + fiat, fiat + c} + } + + for _, try := range tries { + if markets.Has(try) { + symbols = append(symbols, try) + break + } + } + + return symbols +} diff --git a/pkg/bbgo/session_test.go b/pkg/bbgo/session_test.go new file mode 100644 index 0000000000..f4ab11f479 --- /dev/null +++ b/pkg/bbgo/session_test.go @@ -0,0 +1,44 @@ +package bbgo + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/types" +) + +func Test_findPossibleMarketSymbols(t *testing.T) { + t.Run("btcusdt", func(t *testing.T) { + markets := types.MarketMap{ + "BTCUSDT": types.Market{}, + "BTCUSDC": types.Market{}, + "BTCUSD": types.Market{}, + "BTCBUSD": types.Market{}, + } + symbols := findPossibleMarketSymbols(markets, "BTC", "USDT") + if assert.Len(t, symbols, 1) { + assert.Equal(t, "BTCUSDT", symbols[0]) + } + }) + + t.Run("btcusd only", func(t *testing.T) { + markets := types.MarketMap{ + "BTCUSD": types.Market{}, + } + symbols := findPossibleMarketSymbols(markets, "BTC", "USDT") + if assert.Len(t, symbols, 1) { + assert.Equal(t, "BTCUSD", symbols[0]) + } + }) + + t.Run("usd to stable coin", func(t *testing.T) { + markets := types.MarketMap{ + "BTCUSDT": types.Market{}, + } + symbols := findPossibleMarketSymbols(markets, "BTC", "USD") + if assert.Len(t, symbols, 1) { + assert.Equal(t, "BTCUSDT", symbols[0]) + } + }) +} From 2c9583cccb9343a08cb8fdf5bcb49758194a1e35 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:17:52 +0800 Subject: [PATCH 331/422] xdepthmaker: remove redundant notification --- pkg/strategy/xdepthmaker/strategy.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index d947a3dc42..e6d52cab64 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -35,10 +35,6 @@ func init() { bbgo.RegisterStrategy(ID, &Strategy{}) } -func notifyTrade(trade types.Trade, _, _ fixedpoint.Value) { - bbgo.Notify(trade) -} - type CrossExchangeMarketMakingStrategy struct { ctx, parent context.Context cancel context.CancelFunc @@ -133,11 +129,19 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // bbgo.Sync(ctx, s) }) + // global order store s.orderStore = core.NewOrderStore(s.Position.Symbol) s.orderStore.BindStream(hedgeSession.UserDataStream) s.orderStore.BindStream(makerSession.UserDataStream) + // global trade collector s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) + s.tradeCollector.OnTrade(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { + bbgo.Notify(trade) + }) + s.tradeCollector.OnPositionUpdate(func(position *types.Position) { + bbgo.Notify(position) + }) s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() @@ -169,7 +173,6 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( }) s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) s.tradeCollector.BindStream(s.makerSession.UserDataStream) - return nil } @@ -344,6 +347,8 @@ func (s *Strategy) CrossRun( return err } + log.Infof("makerSession: %s hedgeSession: %s", makerSession.Name, hedgeSession.Name) + if err := s.CrossExchangeMarketMakingStrategy.Initialize(ctx, s.Environment, makerSession, hedgeSession, s.Symbol, ID, s.InstanceID()); err != nil { return err } @@ -351,14 +356,6 @@ func (s *Strategy) CrossRun( s.pricingBook = types.NewStreamBook(s.Symbol) s.pricingBook.BindStream(s.hedgeSession.MarketDataStream) - if s.NotifyTrade { - s.tradeCollector.OnTrade(notifyTrade) - } - - s.tradeCollector.OnPositionUpdate(func(position *types.Position) { - bbgo.Notify(position) - }) - s.stopC = make(chan struct{}) if s.RecoverTrade { From 84085e09b56d1a8c9a38c9bfbbfacb648f6f1c5c Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:29:11 +0800 Subject: [PATCH 332/422] xdepthmaker: fix duplicated binding --- pkg/strategy/xdepthmaker/strategy.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index e6d52cab64..a2d05110e5 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -136,12 +136,6 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // global trade collector s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) - s.tradeCollector.OnTrade(func(trade types.Trade, profit fixedpoint.Value, netProfit fixedpoint.Value) { - bbgo.Notify(trade) - }) - s.tradeCollector.OnPositionUpdate(func(position *types.Position) { - bbgo.Notify(position) - }) s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() @@ -156,20 +150,6 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // TODO: make this atomic s.CoveredPosition = s.CoveredPosition.Add(c) } - - s.ProfitStats.AddTrade(trade) - - if profit.Compare(fixedpoint.Zero) == 0 { - s.Environ.RecordPosition(s.Position, trade, nil) - } else { - log.Infof("%s generated profit: %v", symbol, profit) - - p := s.Position.NewProfit(trade, profit, netProfit) - bbgo.Notify(&p) - s.ProfitStats.AddProfit(p) - - s.Environ.RecordPosition(s.Position, trade, &p) - } }) s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) s.tradeCollector.BindStream(s.makerSession.UserDataStream) From 47b12edc4d31bf2060331f59f7ab6b80482039b6 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:30:16 +0800 Subject: [PATCH 333/422] xdepthmaker: call bbgo.Sync on shutdown --- pkg/strategy/xdepthmaker/strategy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index a2d05110e5..912a44d802 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -462,6 +462,7 @@ func (s *Strategy) CrossRun( log.WithError(err).Errorf("graceful cancel %s order error", s.Symbol) } + bbgo.Sync(ctx, s) bbgo.Notify("%s: %s position", ID, s.Symbol, s.Position) }) From e8552140739ee09af7beda37acc25b9072d01ed9 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:42:20 +0800 Subject: [PATCH 334/422] bump version to v1.55.3 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index f40ddf537c..bb47e6c9e8 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.2-98468feb-dev" +const Version = "v1.55.3-083f6265-dev" -const VersionGitRef = "98468feb" +const VersionGitRef = "083f6265" diff --git a/pkg/version/version.go b/pkg/version/version.go index 10caa2e32e..afaa1e04aa 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.2-98468feb" +const Version = "v1.55.3-083f6265" -const VersionGitRef = "98468feb" +const VersionGitRef = "083f6265" From 66a90c50b336cc065ef38d103f652ad14ac2603d Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 18 Dec 2023 22:42:20 +0800 Subject: [PATCH 335/422] add v1.55.3 release note --- doc/release/v1.55.3.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release/v1.55.3.md diff --git a/doc/release/v1.55.3.md b/doc/release/v1.55.3.md new file mode 100644 index 0000000000..57073c266c --- /dev/null +++ b/doc/release/v1.55.3.md @@ -0,0 +1,4 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.55.2...main) + + - [#1461](https://github.com/c9s/bbgo/pull/1461): FIX: [xdepthmaker] fix double binding + - [#1460](https://github.com/c9s/bbgo/pull/1460): FIX: fix and improve session UpdatePrice method From ec4f43b1000dc475f2289f369ea73973d6356a1b Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 21:55:38 +0800 Subject: [PATCH 336/422] bollmaker: support custom quantity --- pkg/strategy/bollmaker/strategy.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index d8f32b8c25..05e7d3c2e1 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -58,6 +58,12 @@ type Strategy struct { bbgo.QuantityOrAmount + // BidQuantity is used for placing buy order, this will override the default quantity + BidQuantity fixedpoint.Value `json:"bidQuantity"` + + // AskQuantity is used for placing sell order, this will override the default quantity + AskQuantity fixedpoint.Value `json:"askQuantity"` + // TrendEMA is used for detecting the trend by a given EMA // you can define interval and window TrendEMA *bbgo.TrendEMA `json:"trendEMA"` @@ -251,8 +257,15 @@ func (s *Strategy) placeOrders(ctx context.Context, midPrice fixedpoint.Value, k s.Position, ) - sellQuantity := s.QuantityOrAmount.CalculateQuantity(askPrice) - buyQuantity := s.QuantityOrAmount.CalculateQuantity(bidPrice) + sellQuantity := s.AskQuantity + if sellQuantity.IsZero() { + sellQuantity = s.QuantityOrAmount.CalculateQuantity(askPrice) + } + + buyQuantity := s.BidQuantity + if buyQuantity.IsZero() { + buyQuantity = s.QuantityOrAmount.CalculateQuantity(bidPrice) + } sellOrder := types.SubmitOrder{ Symbol: s.Symbol, From 25c895bb09a3e349e5472a40f4d445beb7f45c85 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 6 Nov 2023 07:44:34 +0800 Subject: [PATCH 337/422] add emacross strategy --- pkg/strategy/emacross/strategy.go | 116 ++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 pkg/strategy/emacross/strategy.go diff --git a/pkg/strategy/emacross/strategy.go b/pkg/strategy/emacross/strategy.go new file mode 100644 index 0000000000..2778fb6871 --- /dev/null +++ b/pkg/strategy/emacross/strategy.go @@ -0,0 +1,116 @@ +package emacross + +import ( + "context" + "fmt" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" +) + +const ID = "emacross" + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + Interval types.Interval `json:"interval"` + SlowWindow int `json:"slowWindow"` + FastWindow int `json:"fastWindow"` + OpenBelow fixedpoint.Value `json:"openBelow"` + CloseAbove fixedpoint.Value `json:"closeAbove"` + + lastKLine types.KLine + + bbgo.OpenPositionOptions +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s:%s:%d-%d", ID, s.Symbol, s.Interval, s.FastWindow, s.SlowWindow) +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval5m}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) +} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + + session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, types.Interval5m, func(k types.KLine) { + s.lastKLine = k + })) + + fastEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: types.Interval5m, Window: 7}) + slowEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: types.Interval5m, Window: 14}) + cross := indicatorv2.Cross(fastEMA, slowEMA) + cross.OnUpdate(func(v float64) { + switch indicatorv2.CrossType(v) { + + case indicatorv2.CrossOver: + if err := s.Strategy.OrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Errorf("unable to cancel order") + } + + opts := s.OpenPositionOptions + opts.Long = true + if price, ok := session.LastPrice(s.Symbol); ok { + opts.Price = price + } + + opts.Tags = []string{"emaCrossOver"} + + _, err := s.Strategy.OrderExecutor.OpenPosition(ctx, opts) + if err != nil { + log.WithError(err).Errorf("unable to submit buy order") + } + case indicatorv2.CrossUnder: + err := s.Strategy.OrderExecutor.ClosePosition(ctx, fixedpoint.One) + if err != nil { + log.WithError(err).Errorf("unable to submit sell order") + } + } + }) + + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + }) + + return nil +} + +func logErr(err error, msgAndArgs ...interface{}) bool { + if err == nil { + return false + } + + if len(msgAndArgs) == 0 { + log.WithError(err).Error(err.Error()) + } else if len(msgAndArgs) == 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Error(msg) + } else if len(msgAndArgs) > 1 { + msg := msgAndArgs[0].(string) + log.WithError(err).Errorf(msg, msgAndArgs[1:]...) + } + + return true +} From 85e87e10b6e4a5b3b8a53e4bc8aaabd51a28eb52 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 6 Nov 2023 07:47:06 +0800 Subject: [PATCH 338/422] cmd: add emacross to builtin --- pkg/cmd/strategy/builtin.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index b2fbcf14f9..61c03788d3 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -12,6 +12,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/deposit2transfer" _ "github.com/c9s/bbgo/pkg/strategy/drift" _ "github.com/c9s/bbgo/pkg/strategy/elliottwave" + _ "github.com/c9s/bbgo/pkg/strategy/emacross" _ "github.com/c9s/bbgo/pkg/strategy/emastop" _ "github.com/c9s/bbgo/pkg/strategy/etf" _ "github.com/c9s/bbgo/pkg/strategy/ewoDgtrd" From 6abb320bce7f0e8481c7f397768b3c4475ba8715 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 7 Nov 2023 11:17:08 +0800 Subject: [PATCH 339/422] emacross: clean up and update config --- config/emacross.yaml | 37 +++++++++++++++++++++++++++++++ pkg/strategy/emacross/strategy.go | 16 +++++-------- 2 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 config/emacross.yaml diff --git a/config/emacross.yaml b/config/emacross.yaml new file mode 100644 index 0000000000..c1ce36a46d --- /dev/null +++ b/config/emacross.yaml @@ -0,0 +1,37 @@ +persistence: + json: + directory: var/data + redis: + host: 127.0.0.1 + port: 6379 + db: 0 + +sessions: + binance: + exchange: binance + envVarPrefix: binance + +exchangeStrategies: +- on: binance + emacross: + symbol: BTCUSDT + # interval: 5m + # fastWindow: 6 + # slowWindow: 18 + # quantity: 0.01 + leverage: 2 + +backtest: + startTime: "2022-01-01" + endTime: "2022-03-01" + symbols: + - BTCUSDT + sessions: [max,binance] + # syncSecKLines: true + accounts: + binance: + makerFeeRate: 0.0% + takerFeeRate: 0.075% + balances: + BTC: 0.0 + USDT: 10_000.0 diff --git a/pkg/strategy/emacross/strategy.go b/pkg/strategy/emacross/strategy.go index 2778fb6871..1ece94b76e 100644 --- a/pkg/strategy/emacross/strategy.go +++ b/pkg/strategy/emacross/strategy.go @@ -47,8 +47,7 @@ func (s *Strategy) InstanceID() string { } func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval5m}) - session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { @@ -59,8 +58,9 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.lastKLine = k })) - fastEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: types.Interval5m, Window: 7}) - slowEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: types.Interval5m, Window: 14}) + fastEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: s.Interval, Window: s.FastWindow}) + slowEMA := session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: s.Interval, Window: s.SlowWindow}) + cross := indicatorv2.Cross(fastEMA, slowEMA) cross.OnUpdate(func(v float64) { switch indicatorv2.CrossType(v) { @@ -79,14 +79,10 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. opts.Tags = []string{"emaCrossOver"} _, err := s.Strategy.OrderExecutor.OpenPosition(ctx, opts) - if err != nil { - log.WithError(err).Errorf("unable to submit buy order") - } + logErr(err, "unable to open position") case indicatorv2.CrossUnder: err := s.Strategy.OrderExecutor.ClosePosition(ctx, fixedpoint.One) - if err != nil { - log.WithError(err).Errorf("unable to submit sell order") - } + logErr(err, "unable to submit close position order") } }) From 3dd93b65dbeb09e5fb137b8affe7914a211f1fc4 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 21:58:50 +0800 Subject: [PATCH 340/422] emacross, scmaker: fix strategy initialization --- pkg/strategy/emacross/strategy.go | 8 +++++++- pkg/strategy/scmaker/strategy.go | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/emacross/strategy.go b/pkg/strategy/emacross/strategy.go index 1ece94b76e..6762a4419b 100644 --- a/pkg/strategy/emacross/strategy.go +++ b/pkg/strategy/emacross/strategy.go @@ -46,12 +46,18 @@ func (s *Strategy) InstanceID() string { return fmt.Sprintf("%s:%s:%s:%d-%d", ID, s.Symbol, s.Interval, s.FastWindow, s.SlowWindow) } +func (s *Strategy) Initialize() error { + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } + return nil +} + func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy = &common.Strategy{} s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) session.MarketDataStream.OnKLineClosed(types.KLineWith(s.Symbol, types.Interval5m, func(k types.KLine) { diff --git a/pkg/strategy/scmaker/strategy.go b/pkg/strategy/scmaker/strategy.go index 7ad5042c2e..5292ef6624 100644 --- a/pkg/strategy/scmaker/strategy.go +++ b/pkg/strategy/scmaker/strategy.go @@ -70,7 +70,9 @@ type Strategy struct { } func (s *Strategy) Initialize() error { - s.Strategy = &common.Strategy{} + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } return nil } From 4894a597569712ab691731627b7d10da7e57fb53 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 21:59:44 +0800 Subject: [PATCH 341/422] fixedmaker, liquiditymaker: update initialize method --- pkg/strategy/fixedmaker/strategy.go | 5 ++++- pkg/strategy/liquiditymaker/strategy.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index 3585e4da1f..af561de09f 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -47,7 +47,10 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { - s.Strategy = &common.Strategy{} + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } + return nil } diff --git a/pkg/strategy/liquiditymaker/strategy.go b/pkg/strategy/liquiditymaker/strategy.go index 47ecec89c9..187082d79a 100644 --- a/pkg/strategy/liquiditymaker/strategy.go +++ b/pkg/strategy/liquiditymaker/strategy.go @@ -67,7 +67,9 @@ type Strategy struct { } func (s *Strategy) Initialize() error { - s.Strategy = &common.Strategy{} + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } return nil } From 6a07af80d811204c483b1dd21e7604cfa502dc77 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 22:04:24 +0800 Subject: [PATCH 342/422] bollmaker: define EMACrossSetting --- pkg/strategy/bollmaker/strategy.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index 05e7d3c2e1..a6e3dd8ae0 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -6,12 +6,11 @@ import ( "math" "sync" - indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" - "github.com/c9s/bbgo/pkg/util" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -46,6 +45,12 @@ type BollingerSetting struct { BandWidth float64 `json:"bandWidth"` } +type EMACrossSetting struct { + Interval types.Interval `json:"interval"` + FastWindow int `json:"fastWindow"` + SlowWindow int `json:"slowWindow"` +} + type Strategy struct { Environment *bbgo.Environment StandardIndicatorSet *bbgo.StandardIndicatorSet @@ -121,6 +126,9 @@ type Strategy struct { // BuyBelowNeutralSMA if true, the market maker will only place buy order when the current price is below the neutral band SMA. BuyBelowNeutralSMA bool `json:"buyBelowNeutralSMA"` + // EMACrossSetting is used for defining ema cross signal to turn on/off buy + EMACrossSetting *EMACrossSetting `json:"emaCross"` + // NeutralBollinger is the smaller range of the bollinger band // If price is in this band, it usually means the price is oscillating. // If price goes out of this band, we tend to not place sell orders or buy orders @@ -150,19 +158,16 @@ type Strategy struct { ShadowProtection bool `json:"shadowProtection"` ShadowProtectionRatio fixedpoint.Value `json:"shadowProtectionRatio"` - session *bbgo.ExchangeSession - book *types.StreamOrderBook - ExitMethods bbgo.ExitMethodSet `json:"exits"` // persistence fields Position *types.Position `json:"position,omitempty" persistence:"position"` ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` + session *bbgo.ExchangeSession + book *types.StreamOrderBook orderExecutor *bbgo.GeneralOrderExecutor - groupID uint32 - // defaultBoll is the BOLLINGER indicator we used for predicting the price. defaultBoll *indicatorv2.BOLLStream @@ -274,7 +279,6 @@ func (s *Strategy) placeOrders(ctx context.Context, midPrice fixedpoint.Value, k Quantity: sellQuantity, Price: askPrice, Market: s.Market, - GroupID: s.groupID, } buyOrder := types.SubmitOrder{ Symbol: s.Symbol, @@ -283,7 +287,6 @@ func (s *Strategy) placeOrders(ctx context.Context, midPrice fixedpoint.Value, k Quantity: buyQuantity, Price: bidPrice, Market: s.Market, - GroupID: s.groupID, } var submitOrders []types.SubmitOrder @@ -511,7 +514,6 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se // calculate group id for orders instanceID := s.InstanceID() - s.groupID = util.FNV32(instanceID) // If position is nil, we need to allocate a new position for calculation if s.Position == nil { From 46329c3a243e845cfee2fff30a233d63b2cd5181 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 22:17:33 +0800 Subject: [PATCH 343/422] bollmaker: add ema cross signal to bollmaker strategy --- config/bollmaker.yaml | 8 ++++++- pkg/strategy/bollmaker/strategy.go | 34 ++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/config/bollmaker.yaml b/config/bollmaker.yaml index 0cc53894c0..d2794d94f1 100644 --- a/config/bollmaker.yaml +++ b/config/bollmaker.yaml @@ -17,7 +17,7 @@ backtest: # see here for more details # https://www.investopedia.com/terms/m/maximum-drawdown-mdd.asp startTime: "2022-05-01" - endTime: "2022-08-14" + endTime: "2023-11-01" sessions: - binance symbols: @@ -200,6 +200,12 @@ exchangeStrategies: # buyBelowNeutralSMA: when this set, it will only place buy order when the current price is below the SMA line. buyBelowNeutralSMA: true + emaCross: + enabled: true + interval: 1h + fastWindow: 3 + slowWindow: 12 + exits: # roiTakeProfit is used to force taking profit by percentage of the position ROI (currently the price change) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index a6e3dd8ae0..de0b92c135 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -46,9 +46,13 @@ type BollingerSetting struct { } type EMACrossSetting struct { + Enabled bool `json:"enabled"` Interval types.Interval `json:"interval"` FastWindow int `json:"fastWindow"` SlowWindow int `json:"slowWindow"` + + fastEMA, slowEMA *indicatorv2.EWMAStream + cross *indicatorv2.CrossStream } type Strategy struct { @@ -174,6 +178,8 @@ type Strategy struct { // neutralBoll is the neutral price section neutralBoll *indicatorv2.BOLLStream + shouldBuy bool + // StrategyController bbgo.StrategyController } @@ -280,6 +286,7 @@ func (s *Strategy) placeOrders(ctx context.Context, midPrice fixedpoint.Value, k Price: askPrice, Market: s.Market, } + buyOrder := types.SubmitOrder{ Symbol: s.Symbol, Side: types.SideTypeBuy, @@ -438,14 +445,20 @@ func (s *Strategy) placeOrders(ctx context.Context, midPrice fixedpoint.Value, k } } + if !s.shouldBuy { + log.Infof("shouldBuy is turned off, skip placing buy order") + canBuy = false + } + if canSell { submitOrders = append(submitOrders, sellOrder) } + if canBuy { submitOrders = append(submitOrders, buyOrder) } - // condition for lower the average cost + // condition for lowering the average cost /* if midPrice < s.Position.AverageCost.MulFloat64(1.0-s.MinProfitSpread.Float64()) && canBuy { submitOrders = append(submitOrders, buyOrder) @@ -481,9 +494,26 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se // StrategyController s.Status = types.StrategyStatusRunning + s.shouldBuy = true s.neutralBoll = session.Indicators(s.Symbol).BOLL(s.NeutralBollinger.IntervalWindow, s.NeutralBollinger.BandWidth) s.defaultBoll = session.Indicators(s.Symbol).BOLL(s.DefaultBollinger.IntervalWindow, s.DefaultBollinger.BandWidth) + if s.EMACrossSetting != nil && s.EMACrossSetting.Enabled { + s.EMACrossSetting.fastEMA = session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: s.Interval, Window: s.EMACrossSetting.FastWindow}) + s.EMACrossSetting.slowEMA = session.Indicators(s.Symbol).EWMA(types.IntervalWindow{Interval: s.Interval, Window: s.EMACrossSetting.SlowWindow}) + s.EMACrossSetting.cross = indicatorv2.Cross(s.EMACrossSetting.fastEMA, s.EMACrossSetting.slowEMA) + s.EMACrossSetting.cross.OnUpdate(func(v float64) { + switch indicatorv2.CrossType(v) { + case indicatorv2.CrossOver: + s.shouldBuy = true + case indicatorv2.CrossUnder: + s.shouldBuy = false + // TODO: can partially close position when necessary + // s.orderExecutor.ClosePosition(ctx) + } + }) + } + // Setup dynamic spread if s.DynamicSpread.IsEnabled() { if s.DynamicSpread.Interval == "" { @@ -565,7 +595,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se }) session.UserDataStream.OnStart(func() { - if s.UseTickerPrice { + if !bbgo.IsBackTesting && s.UseTickerPrice { ticker, err := s.session.Exchange.QueryTicker(ctx, s.Symbol) if err != nil { return From 127d4484fbef7b1446d5455a530030d2d0b672aa Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 19 Dec 2023 22:17:53 +0800 Subject: [PATCH 344/422] config: disable emaCross by default --- config/bollmaker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/bollmaker.yaml b/config/bollmaker.yaml index d2794d94f1..ac42aeed77 100644 --- a/config/bollmaker.yaml +++ b/config/bollmaker.yaml @@ -201,7 +201,7 @@ exchangeStrategies: buyBelowNeutralSMA: true emaCross: - enabled: true + enabled: false interval: 1h fastWindow: 3 slowWindow: 12 From 311ba3b2ac1d0389a059da9445ffae810e5a7154 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 12:09:19 +0800 Subject: [PATCH 345/422] bollmaker: fix ema cross subscription --- pkg/strategy/bollmaker/strategy.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/strategy/bollmaker/strategy.go b/pkg/strategy/bollmaker/strategy.go index de0b92c135..e372eea1f6 100644 --- a/pkg/strategy/bollmaker/strategy.go +++ b/pkg/strategy/bollmaker/strategy.go @@ -213,6 +213,10 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.TrendEMA.Interval}) } + if s.EMACrossSetting != nil && s.EMACrossSetting.Enabled { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.EMACrossSetting.Interval}) + } + s.ExitMethods.SetAndSubscribe(session, s) } From 382a78949c54525e3ef7cf18c1f83700bbb3b765 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 12:13:09 +0800 Subject: [PATCH 346/422] config: document emaCross in config --- config/bollmaker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/bollmaker.yaml b/config/bollmaker.yaml index ac42aeed77..5ecc673150 100644 --- a/config/bollmaker.yaml +++ b/config/bollmaker.yaml @@ -200,6 +200,9 @@ exchangeStrategies: # buyBelowNeutralSMA: when this set, it will only place buy order when the current price is below the SMA line. buyBelowNeutralSMA: true + # emaCross is used for turning buy on/off + # when short term EMA cross fast term EMA, turn on buy, + # otherwise, turn off buy emaCross: enabled: false interval: 1h From bfd9c8ac64692c1a4e3418b41813e1c89b85e9ae Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 27 Nov 2023 15:55:02 +0800 Subject: [PATCH 347/422] FEATURE: run state machine FEATURE: support recover FEATURE: add order into orderStore and recover position recover position/budget FEATURE: support recover budget --- config/dca2.yaml | 9 +- ...t_wallet_open_orders_request_requestgen.go | 1 - pkg/strategy/dca2/debug.go | 13 + pkg/strategy/dca2/open_position_test.go | 12 +- pkg/strategy/dca2/recover.go | 268 ++++++++++++++++++ pkg/strategy/dca2/recover_test.go | 236 +++++++++++++++ pkg/strategy/dca2/state.go | 163 +++++++++++ pkg/strategy/dca2/strategy.go | 89 +++++- 8 files changed, 766 insertions(+), 25 deletions(-) create mode 100644 pkg/strategy/dca2/recover.go create mode 100644 pkg/strategy/dca2/recover_test.go create mode 100644 pkg/strategy/dca2/state.go diff --git a/config/dca2.yaml b/config/dca2.yaml index 8c10453ca4..6cb8b6ca9a 100644 --- a/config/dca2.yaml +++ b/config/dca2.yaml @@ -23,8 +23,9 @@ exchangeStrategies: dca2: symbol: ETHUSDT short: false - budget: 5000 - maxOrderNum: 10 + budget: 200 + maxOrderNum: 5 priceDeviation: 1% - takeProfitRatio: 1% - coolDownInterval: 5m + takeProfitRatio: 0.2% + coolDownInterval: 3m + circuitBreakLossThreshold: -0.9 diff --git a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go index 51416ccda7..741f375a21 100644 --- a/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go +++ b/pkg/exchange/max/maxapi/v3/get_wallet_open_orders_request_requestgen.go @@ -72,7 +72,6 @@ func (g *GetWalletOpenOrdersRequest) GetParameters() (map[string]interface{}, er // assign parameter of timestamp // convert time.Time to milliseconds time stamp params["timestamp"] = strconv.FormatInt(timestamp.UnixNano()/int64(time.Millisecond), 10) - fmt.Println(params["timestamp"], timestamp) } else { } // check orderBy field -> json key order_by diff --git a/pkg/strategy/dca2/debug.go b/pkg/strategy/dca2/debug.go index 8e44bf916c..cf474080c6 100644 --- a/pkg/strategy/dca2/debug.go +++ b/pkg/strategy/dca2/debug.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/c9s/bbgo/pkg/types" + "github.com/sirupsen/logrus" ) func (s *Strategy) debugOrders(submitOrders []types.Order) { @@ -17,3 +18,15 @@ func (s *Strategy) debugOrders(submitOrders []types.Order) { s.logger.Info(sb.String()) } + +func debugRoundOrders(logger *logrus.Entry, roundName string, round Round) { + var sb strings.Builder + sb.WriteString("ROUND " + roundName + " [\n") + sb.WriteString(round.TakeProfitOrder.String() + "\n") + sb.WriteString("------------------------------------------------\n") + for i, order := range round.OpenPositionOrders { + sb.WriteString(fmt.Sprintf("%3d) ", i+1) + order.String() + "\n") + } + sb.WriteString("] END OF ROUND") + logger.Info(sb.String()) +} diff --git a/pkg/strategy/dca2/open_position_test.go b/pkg/strategy/dca2/open_position_test.go index 3d51bf6af0..a9fd33cf08 100644 --- a/pkg/strategy/dca2/open_position_test.go +++ b/pkg/strategy/dca2/open_position_test.go @@ -33,13 +33,11 @@ func newTestStrategy(va ...string) *Strategy { market := newTestMarket() s := &Strategy{ - logger: logrus.NewEntry(logrus.New()), - Symbol: symbol, - Market: market, - Short: false, - TakeProfitRatio: Number("10%"), - openPositionSide: types.SideTypeBuy, - takeProfitSide: types.SideTypeSell, + logger: logrus.NewEntry(logrus.New()), + Symbol: symbol, + Market: market, + Short: false, + TakeProfitRatio: Number("10%"), } return s } diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go new file mode 100644 index 0000000000..40c66ee9fd --- /dev/null +++ b/pkg/strategy/dca2/recover.go @@ -0,0 +1,268 @@ +package dca2 + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type queryAPI interface { + QueryOpenOrders(ctx context.Context, symbol string) ([]types.Order, error) + QueryClosedOrdersDesc(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) + QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([]types.Trade, error) +} + +func (s *Strategy) recover(ctx context.Context) error { + s.logger.Info("[DCA] recover") + queryService, ok := s.Session.Exchange.(queryAPI) + if !ok { + return fmt.Errorf("[DCA] exchange %s doesn't support queryAPI interface", s.Session.ExchangeName) + } + + openOrders, err := queryService.QueryOpenOrders(ctx, s.Symbol) + if err != nil { + return err + } + + closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Time{}, time.Now(), 0) + if err != nil { + return err + } + + currentRound, err := getCurrentRoundOrders(s.Short, openOrders, closedOrders, s.OrderGroupID) + if err != nil { + return err + } + debugRoundOrders(s.logger, "current", currentRound) + + // recover state + state, err := recoverState(ctx, s.Symbol, s.Short, int(s.MaxOrderNum), openOrders, currentRound, s.OrderExecutor.ActiveMakerOrders(), s.OrderExecutor.OrderStore(), s.OrderGroupID) + if err != nil { + return err + } + + // recover position + if err := recoverPosition(ctx, s.Position, queryService, currentRound); err != nil { + return err + } + + // recover budget + budget := recoverBudget(currentRound) + + // recover startTimeOfNextRound + startTimeOfNextRound := recoverStartTimeOfNextRound(ctx, currentRound, s.CoolDownInterval) + + s.state = state + if !budget.IsZero() { + s.Budget = budget + } + s.startTimeOfNextRound = startTimeOfNextRound + + return nil +} + +// recover state +func recoverState(ctx context.Context, symbol string, short bool, maxOrderNum int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { + numOpenOrders := len(openOrders) + // dca stop at take profit order stage + if currentRound.TakeProfitOrder.OrderID != 0 { + // check the open orders is take profit order or not + if numOpenOrders == 1 { + if openOrders[0].OrderID == currentRound.TakeProfitOrder.OrderID { + activeOrderBook.Add(openOrders[0]) + // current round's take-profit order still opened, wait to fill + return TakeProfitReady, nil + } else { + return None, fmt.Errorf("stop at taking profit stage, but the open order's OrderID is not the take-profit order's OrderID") + } + } + + if numOpenOrders == 0 { + // current round's take-profit order filled, wait to open next round + return WaitToOpenPosition, nil + } + + return None, fmt.Errorf("stop at taking profit stage, but the number of open orders is > 1") + } + + if len(currentRound.OpenPositionOrders) == 0 { + // new strategy + return WaitToOpenPosition, nil + } + + numOpenPositionOrders := len(currentRound.OpenPositionOrders) + if numOpenPositionOrders > maxOrderNum { + return None, fmt.Errorf("the number of open-position orders is > max order number") + } else if numOpenPositionOrders < maxOrderNum { + // failed to place some orders at open position stage + return None, fmt.Errorf("the number of open-position orders is < max order number") + } + + if numOpenOrders > numOpenPositionOrders { + return None, fmt.Errorf("the number of open orders is > the number of open-position orders") + } + + if numOpenOrders == numOpenPositionOrders { + activeOrderBook.Add(openOrders...) + orderStore.Add(openOrders...) + return OpenPositionReady, nil + } + + var openedCnt, filledCnt, cancelledCnt int64 + for _, order := range currentRound.OpenPositionOrders { + switch order.Status { + case types.OrderStatusNew, types.OrderStatusPartiallyFilled: + openedCnt++ + case types.OrderStatusFilled: + filledCnt++ + case types.OrderStatusCanceled: + cancelledCnt++ + default: + return None, fmt.Errorf("there is unexpected status %s of order %s", order.Status, order) + } + } + + if filledCnt > 0 && cancelledCnt == 0 { + activeOrderBook.Add(openOrders...) + orderStore.Add(openOrders...) + return OpenPositionOrderFilled, nil + } + + if openedCnt > 0 && filledCnt > 0 && cancelledCnt > 0 { + return OpenPositionOrdersCancelling, nil + } + + if openedCnt == 0 && filledCnt > 0 && cancelledCnt > 0 { + return OpenPositionOrdersCancelled, nil + } + + return None, fmt.Errorf("unexpected order status combination") +} + +func recoverPosition(ctx context.Context, position *types.Position, queryService queryAPI, currentRound Round) error { + if position == nil { + return nil + } + + var positionOrders []types.Order + position.Reset() + if currentRound.TakeProfitOrder.OrderID != 0 { + if !types.IsActiveOrder(currentRound.TakeProfitOrder) { + return nil + } + + positionOrders = append(positionOrders, currentRound.TakeProfitOrder) + } + + for _, order := range currentRound.OpenPositionOrders { + // no executed quantity order, no need to get trades + if order.ExecutedQuantity.IsZero() { + continue + } + + positionOrders = append(positionOrders, order) + } + + for _, positionOrder := range positionOrders { + trades, err := queryService.QueryOrderTrades(ctx, types.OrderQuery{ + Symbol: position.Symbol, + OrderID: strconv.FormatUint(positionOrder.OrderID, 10), + }) + + if err != nil { + return fmt.Errorf("failed to get trades of order (%d)", positionOrder.OrderID) + } + + position.AddTrades(trades) + } + + return nil +} + +func recoverBudget(currentRound Round) fixedpoint.Value { + if len(currentRound.OpenPositionOrders) == 0 { + return fixedpoint.Zero + } + + total := fixedpoint.Zero + for _, order := range currentRound.OpenPositionOrders { + total = total.Add(order.Quantity.Mul(order.Price)) + } + + if currentRound.TakeProfitOrder.OrderID != 0 && currentRound.TakeProfitOrder.Status == types.OrderStatusFilled { + total = total.Add(currentRound.TakeProfitOrder.Quantity.Mul(currentRound.TakeProfitOrder.Price)) + for _, order := range currentRound.OpenPositionOrders { + total = total.Sub(order.ExecutedQuantity.Mul(order.Price)) + } + } + + return total +} + +func recoverStartTimeOfNextRound(ctx context.Context, currentRound Round, coolDownInterval types.Duration) time.Time { + if currentRound.TakeProfitOrder.OrderID != 0 && currentRound.TakeProfitOrder.Status == types.OrderStatusFilled { + return currentRound.TakeProfitOrder.UpdateTime.Time().Add(coolDownInterval.Duration()) + } + + return time.Time{} +} + +type Round struct { + OpenPositionOrders []types.Order + TakeProfitOrder types.Order +} + +func getCurrentRoundOrders(short bool, openOrders, closedOrders []types.Order, groupID uint32) (Round, error) { + openPositionSide := types.SideTypeBuy + takeProfitSide := types.SideTypeSell + + if short { + openPositionSide = types.SideTypeSell + takeProfitSide = types.SideTypeBuy + } + + var allOrders []types.Order + allOrders = append(allOrders, openOrders...) + allOrders = append(allOrders, closedOrders...) + + sort.Slice(allOrders, func(i, j int) bool { + return allOrders[i].CreationTime.After(allOrders[j].CreationTime.Time()) + }) + + var currentRound Round + lastSide := takeProfitSide + for _, order := range allOrders { + // group id filter is used for debug when local running + /* + if order.GroupID != groupID { + continue + } + */ + + if order.Side == takeProfitSide && lastSide == openPositionSide { + break + } + + switch order.Side { + case openPositionSide: + currentRound.OpenPositionOrders = append(currentRound.OpenPositionOrders, order) + case takeProfitSide: + if currentRound.TakeProfitOrder.OrderID != 0 { + return currentRound, fmt.Errorf("there are two take-profit orders in one round, please check it") + } + currentRound.TakeProfitOrder = order + default: + } + + lastSide = order.Side + } + + return currentRound, nil +} diff --git a/pkg/strategy/dca2/recover_test.go b/pkg/strategy/dca2/recover_test.go new file mode 100644 index 0000000000..98550083ae --- /dev/null +++ b/pkg/strategy/dca2/recover_test.go @@ -0,0 +1,236 @@ +package dca2 + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/core" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func generateOrder(side types.SideType, status types.OrderStatus, createdAt time.Time) types.Order { + return types.Order{ + OrderID: rand.Uint64(), + SubmitOrder: types.SubmitOrder{ + Side: side, + }, + Status: status, + CreationTime: types.Time(createdAt), + } + +} + +func Test_GetCurrenctAndLastRoundOrders(t *testing.T) { + assert := assert.New(t) + + t.Run("case 1", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + } + + closedOrders := []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), + } + + currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) + + assert.NoError(err) + assert.NotEqual(0, currentRound.TakeProfitOrder.OrderID) + assert.Equal(5, len(currentRound.OpenPositionOrders)) + }) + + t.Run("case 2", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + } + + closedOrders := []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), + generateOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-6*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-7*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-8*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-9*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-10*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-11*time.Second)), + generateOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-12*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-13*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-14*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-15*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-16*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-17*time.Second)), + } + + currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) + + assert.NoError(err) + assert.NotEqual(0, currentRound.TakeProfitOrder.OrderID) + assert.Equal(5, len(currentRound.OpenPositionOrders)) + }) +} + +type MockQueryOrders struct { + OpenOrders []types.Order + ClosedOrders []types.Order +} + +func (m *MockQueryOrders) QueryOpenOrders(ctx context.Context, symbol string) ([]types.Order, error) { + return m.OpenOrders, nil +} + +func (m *MockQueryOrders) QueryClosedOrdersDesc(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) { + return m.ClosedOrders, nil +} + +func Test_RecoverState(t *testing.T) { + assert := assert.New(t) + symbol := "BTCUSDT" + + t.Run("new strategy", func(t *testing.T) { + openOrders := []types.Order{} + currentRound := Round{} + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + orderStore := core.NewOrderStore(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(WaitToOpenPosition, state) + }) + + t.Run("at open position stage and no filled order", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusPartiallyFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + } + currentRound := Round{ + OpenPositionOrders: openOrders, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(OpenPositionReady, state) + }) + + t.Run("at open position stage and there at least one filled order", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + } + currentRound := Round{ + OpenPositionOrders: []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + openOrders[0], + openOrders[1], + openOrders[2], + openOrders[3], + }, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(OpenPositionOrderFilled, state) + }) + + t.Run("open position stage finish, but stop at cancelling", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + } + currentRound := Round{ + OpenPositionOrders: []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + openOrders[0], + }, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(OpenPositionOrdersCancelling, state) + }) + + t.Run("open-position orders are cancelled", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{} + currentRound := Round{ + OpenPositionOrders: []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + }, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(OpenPositionOrdersCancelled, state) + }) + + t.Run("at take profit stage, and not filled yet", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{ + generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + } + currentRound := Round{ + TakeProfitOrder: openOrders[0], + OpenPositionOrders: []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + }, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(TakeProfitReady, state) + }) + + t.Run("at take profit stage, take-profit order filled", func(t *testing.T) { + now := time.Now() + openOrders := []types.Order{} + currentRound := Round{ + TakeProfitOrder: generateOrder(types.SideTypeSell, types.OrderStatusFilled, now), + OpenPositionOrders: []types.Order{ + generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + }, + } + orderStore := core.NewOrderStore(symbol) + activeOrderBook := bbgo.NewActiveOrderBook(symbol) + state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + assert.NoError(err) + assert.Equal(WaitToOpenPosition, state) + }) +} diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go new file mode 100644 index 0000000000..7a787fcda7 --- /dev/null +++ b/pkg/strategy/dca2/state.go @@ -0,0 +1,163 @@ +package dca2 + +import ( + "context" + "time" +) + +type State int64 + +const ( + None State = iota + WaitToOpenPosition + PositionOpening + OpenPositionReady + OpenPositionOrderFilled + OpenPositionOrdersCancelling + OpenPositionOrdersCancelled + TakeProfitReady +) + +func (s *Strategy) initializeNextStateC() bool { + s.mu.Lock() + defer s.mu.Unlock() + + isInitialize := false + if s.nextStateC == nil { + s.logger.Info("[DCA] initializing next state channel") + s.nextStateC = make(chan State, 1) + } else { + s.logger.Info("[DCA] nextStateC is already initialized") + isInitialize = true + } + + return isInitialize +} + +// runState +// WaitToOpenPosition -> after startTimeOfNextRound, place dca orders -> +// PositionOpening +// OpenPositionReady -> any dca maker order filled -> +// OpenPositionOrderFilled -> price hit the take profit ration, start cancelling -> +// OpenPositionOrdersCancelled -> place the takeProfit order -> +// TakeProfitReady -> the takeProfit order filled -> +func (s *Strategy) runState(ctx context.Context) { + s.logger.Info("[DCA] runState") + for { + select { + case <-ctx.Done(): + s.logger.Info("[DCA] runState DONE") + return + case nextState := <-s.nextStateC: + s.logger.Infof("[DCA] currenct state: %d, next state: %d", s.state, nextState) + switch s.state { + case WaitToOpenPosition: + s.runWaitToOpenPositionState(ctx, nextState) + case PositionOpening: + s.runPositionOpening(ctx, nextState) + case OpenPositionReady: + s.runOpenPositionReady(ctx, nextState) + case OpenPositionOrderFilled: + s.runOpenPositionOrderFilled(ctx, nextState) + case OpenPositionOrdersCancelling: + s.runOpenPositionOrdersCancelling(ctx, nextState) + case OpenPositionOrdersCancelled: + s.runOpenPositionOrdersCancelled(ctx, nextState) + case TakeProfitReady: + s.runTakeProfitReady(ctx, nextState) + } + } + } +} + +func (s *Strategy) runWaitToOpenPositionState(_ context.Context, next State) { + if next != None { + return + } + + s.logger.Info("[WaitToOpenPosition] check startTimeOfNextRound") + if time.Now().Before(s.startTimeOfNextRound) { + return + } + + s.state = PositionOpening + s.logger.Info("[WaitToOpenPosition] move to PositionOpening") +} + +func (s *Strategy) runPositionOpening(ctx context.Context, next State) { + if next != None { + return + } + + s.logger.Info("[PositionOpening] start placing open-position orders") + if err := s.placeOpenPositionOrders(ctx); err != nil { + s.logger.WithError(err).Error("failed to place dca orders, please check it.") + return + } + s.state = OpenPositionReady + s.logger.Info("[PositionOpening] move to OpenPositionReady") +} + +func (s *Strategy) runOpenPositionReady(_ context.Context, next State) { + if next != OpenPositionOrderFilled { + return + } + s.state = OpenPositionOrderFilled + s.logger.Info("[OpenPositionReady] move to OpenPositionOrderFilled") +} + +func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { + if next != OpenPositionOrdersCancelling { + return + } + s.state = OpenPositionOrdersCancelling + s.logger.Info("[OpenPositionOrderFilled] move to OpenPositionOrdersCancelling") +} + +func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { + if next != None { + return + } + + s.logger.Info("[OpenPositionOrdersCancelling] start cancelling open-position orders") + if err := s.cancelOpenPositionOrders(ctx); err != nil { + s.logger.WithError(err).Error("failed to cancel maker orders") + return + } + s.state = OpenPositionOrdersCancelled + s.logger.Info("[OpenPositionOrdersCancelling] move to OpenPositionOrdersCancelled") +} + +func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next State) { + if next != None { + return + } + s.logger.Info("[OpenPositionOrdersCancelled] start placing take-profit orders") + if err := s.placeTakeProfitOrders(ctx); err != nil { + s.logger.WithError(err).Error("failed to open take profit orders") + return + } + s.state = TakeProfitReady + s.logger.Info("[OpenPositionOrdersCancelled] move to TakeProfitReady") +} + +func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { + if next != WaitToOpenPosition { + return + } + + s.logger.Info("[TakeProfitReady] start reseting position and calculate budget for next round") + if s.Short { + s.Budget = s.Budget.Add(s.Position.Base) + } else { + s.Budget = s.Budget.Add(s.Position.Quote) + } + + // reset position + s.Position.Reset() + + // set the start time of the next round + s.startTimeOfNextRound = time.Now().Add(s.CoolDownInterval.Duration()) + s.state = WaitToOpenPosition + s.logger.Info("[TakeProfitReady] move to WaitToOpenPosition") +} diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 0cf17fbf44..2ed8e8c1cd 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -50,10 +50,10 @@ type Strategy struct { // private field mu sync.Mutex - openPositionSide types.SideType - takeProfitSide types.SideType takeProfitPrice fixedpoint.Value startTimeOfNextRound time.Time + nextStateC chan State + state State } func (s *Strategy) ID() string { @@ -105,33 +105,87 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) instanceID := s.InstanceID() - if s.Short { - s.openPositionSide = types.SideTypeSell - s.takeProfitSide = types.SideTypeBuy - } else { - s.openPositionSide = types.SideTypeBuy - s.takeProfitSide = types.SideTypeSell - } - if s.OrderGroupID == 0 { s.OrderGroupID = util.FNV32(instanceID) % math.MaxInt32 } + s.updateTakeProfitPrice() + // order executor s.OrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - s.logger.Infof("position: %s", s.Position.String()) + s.logger.Infof("[DCA] POSITION UPDATE: %s", s.Position.String()) bbgo.Sync(ctx, s) // update take profit price here + s.updateTakeProfitPrice() + }) + + s.OrderExecutor.ActiveMakerOrders().OnFilled(func(o types.Order) { + s.logger.Infof("[DCA] FILLED ORDER: %s", o.String()) + openPositionSide := types.SideTypeBuy + takeProfitSide := types.SideTypeSell + if s.Short { + openPositionSide = types.SideTypeSell + takeProfitSide = types.SideTypeBuy + } + + switch o.Side { + case openPositionSide: + s.nextStateC <- OpenPositionOrderFilled + case takeProfitSide: + s.nextStateC <- WaitToOpenPosition + default: + s.logger.Infof("[DCA] unsupported side (%s) of order: %s", o.Side, o) + } }) session.MarketDataStream.OnKLine(func(kline types.KLine) { + s.logger.Infof("[DCA] %s", s.Strategy.Position.String()) + s.logger.Infof("[DCA] tkae-profit price: %s", s.takeProfitPrice) // check price here + // because we subscribe 1m kline, it will close every 1 min + // we use it as ticker to maker WaitToOpenPosition -> OpenPositionReady + select { + case s.nextStateC <- None: + default: + s.logger.Info("[DCA] nextStateC is full or not initialized") + } + + if s.state != OpenPositionOrderFilled { + return + } + + compRes := kline.Close.Compare(s.takeProfitPrice) + // price doesn't hit the take profit price + if (s.Short && compRes > 0) || (!s.Short && compRes < 0) { + return + } + + s.nextStateC <- OpenPositionOrdersCancelling }) session.UserDataStream.OnAuth(func() { - s.logger.Info("user data stream authenticated, start the process") - // decide state here + s.logger.Info("[DCA] user data stream authenticated") + time.AfterFunc(3*time.Second, func() { + if isInitialize := s.initializeNextStateC(); !isInitialize { + // recover + if err := s.recover(ctx); err != nil { + s.logger.WithError(err).Error("[DCA] something wrong when state recovering") + return + } + + s.logger.Infof("[DCA] recovered state: %d", s.state) + s.logger.Infof("[DCA] recovered position %s", s.Position.String()) + s.logger.Infof("[DCA] recovered budget %s", s.Budget) + s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) + + // store persistence + bbgo.Sync(ctx, s) + + // start running state machine + s.runState(ctx) + } + }) }) balances, err := session.Exchange.QueryAccountBalances(ctx) @@ -146,3 +200,12 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. return nil } + +func (s *Strategy) updateTakeProfitPrice() { + takeProfitRatio := s.TakeProfitRatio + if s.Short { + takeProfitRatio = takeProfitRatio.Neg() + } + s.takeProfitPrice = s.Market.TruncatePrice(s.Position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) + s.logger.Infof("[DCA] cost: %s, ratio: %s, price: %s", s.Position.AverageCost, takeProfitRatio, s.takeProfitPrice) +} From da02c926be82f47f43e292561ed14a0a893e9fe9 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:21:34 +0800 Subject: [PATCH 348/422] fix profit stats and position --- pkg/strategy/rebalance/multi_market_strategy.go | 4 ++-- pkg/strategy/rebalance/strategy.go | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/rebalance/multi_market_strategy.go b/pkg/strategy/rebalance/multi_market_strategy.go index e7d17dd0b0..e59eff801c 100644 --- a/pkg/strategy/rebalance/multi_market_strategy.go +++ b/pkg/strategy/rebalance/multi_market_strategy.go @@ -11,8 +11,8 @@ type MultiMarketStrategy struct { Environ *bbgo.Environment Session *bbgo.ExchangeSession - PositionMap PositionMap `persistence:"positionMap"` - ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"` + PositionMap PositionMap `persistence:"position_map"` + ProfitStatsMap ProfitStatsMap `persistence:"profit_stats_map"` OrderExecutorMap GeneralOrderExecutorMap parent, ctx context.Context diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index 40c0fa8211..0a1358727c 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -54,6 +54,10 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { + if s.MultiMarketStrategy == nil { + s.MultiMarketStrategy = &MultiMarketStrategy{} + } + for currency := range s.TargetWeights { if currency == s.QuoteCurrency { continue @@ -105,7 +109,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.markets[symbol] = market } - s.MultiMarketStrategy = &MultiMarketStrategy{} s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) s.activeOrderBook = bbgo.NewActiveOrderBook("") From 762a09042a47110765c510f27384290823975737 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:26:34 +0800 Subject: [PATCH 349/422] graceful cancel orders --- pkg/strategy/rebalance/strategy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index 0a1358727c..af7834fb34 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -137,7 +137,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. func (s *Strategy) rebalance(ctx context.Context) { // cancel active orders before rebalance - if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.activeOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { log.WithError(err).Errorf("failed to cancel orders") } From 7b121b10bee067a2a4d4dda7afccc52a4691b67b Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:35:43 +0800 Subject: [PATCH 350/422] rebalance on order filled --- pkg/strategy/rebalance/strategy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index af7834fb34..cfeb9844d8 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -113,6 +113,9 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.activeOrderBook = bbgo.NewActiveOrderBook("") s.activeOrderBook.BindStream(session.UserDataStream) + s.activeOrderBook.OnFilled(func(order types.Order) { + s.rebalance(ctx) + }) session.UserDataStream.OnStart(func() { if s.OnStart { From eb36ed6926aa9f10d83860dae378a4ef490a6e9b Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 21:54:32 +0800 Subject: [PATCH 351/422] xdepthmaker: remove the shared trade collector and order store, add mutex for covered position --- pkg/strategy/xdepthmaker/strategy.go | 59 +++++++++++----------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 912a44d802..ad2e9a37f2 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -12,7 +12,6 @@ import ( "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/bbgo" - "github.com/c9s/bbgo/pkg/core" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -48,14 +47,9 @@ type CrossExchangeMarketMakingStrategy struct { Position *types.Position `json:"position,omitempty" persistence:"position"` ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` CoveredPosition fixedpoint.Value `json:"coveredPosition,omitempty" persistence:"covered_position"` + mu sync.Mutex MakerOrderExecutor, HedgeOrderExecutor *bbgo.GeneralOrderExecutor - - // orderStore is a shared order store between the maker session and the hedge session - orderStore *core.OrderStore - - // tradeCollector is a shared trade collector between the maker session and the hedge session - tradeCollector *core.TradeCollector } func (s *CrossExchangeMarketMakingStrategy) Initialize( @@ -129,14 +123,7 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // bbgo.Sync(ctx, s) }) - // global order store - s.orderStore = core.NewOrderStore(s.Position.Symbol) - s.orderStore.BindStream(hedgeSession.UserDataStream) - s.orderStore.BindStream(makerSession.UserDataStream) - - // global trade collector - s.tradeCollector = core.NewTradeCollector(symbol, s.Position, s.orderStore) - s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { + coveredFunc := func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() // sync covered position @@ -148,11 +135,13 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // 2) short position -> increase short position if trade.Exchange == s.hedgeSession.ExchangeName { // TODO: make this atomic + s.mu.Lock() s.CoveredPosition = s.CoveredPosition.Add(c) + s.mu.Unlock() } - }) - s.tradeCollector.BindStream(s.hedgeSession.UserDataStream) - s.tradeCollector.BindStream(s.makerSession.UserDataStream) + } + s.MakerOrderExecutor.TradeCollector().OnTrade(coveredFunc) + s.HedgeOrderExecutor.TradeCollector().OnTrade(coveredFunc) return nil } @@ -339,11 +328,7 @@ func (s *Strategy) CrossRun( s.stopC = make(chan struct{}) if s.RecoverTrade { - s.tradeCollector.OnRecover(func(trade types.Trade) { - bbgo.Notify("Recovered trade", trade) - }) - - go s.runTradeRecover(ctx) + // go s.runTradeRecover(ctx) } s.authedC = make(chan struct{}, 2) @@ -373,7 +358,7 @@ func (s *Strategy) CrossRun( defer fullReplenishTicker.Stop() // clean up the previous open orders - if err := s.cleanUpOpenOrders(ctx); err != nil { + if err := s.cleanUpOpenOrders(ctx, s.makerSession); err != nil { log.WithError(err).Errorf("error cleaning up open orders") } @@ -426,10 +411,10 @@ func (s *Strategy) CrossRun( // // For negative position: // uncover position = -5 - -3 (covered position) = -2 - s.tradeCollector.Process() + s.HedgeOrderExecutor.TradeCollector().Process() + s.MakerOrderExecutor.TradeCollector().Process() position := s.Position.GetBase() - uncoverPosition := position.Sub(s.CoveredPosition) absPos := uncoverPosition.Abs() if absPos.Compare(s.hedgeMarket.MinQuantity) > 0 { @@ -550,7 +535,7 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { log.Infof("submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) bbgo.Notify("Submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) - createdOrders, err := s.HedgeOrderExecutor.SubmitOrders(ctx, types.SubmitOrder{ + _, err := s.HedgeOrderExecutor.SubmitOrders(ctx, types.SubmitOrder{ Market: s.hedgeMarket, Symbol: s.Symbol, Type: types.OrderTypeMarket, @@ -564,14 +549,16 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { return } - s.orderStore.Add(createdOrders...) - // if the hedge is on sell side, then we should add positive position switch side { case types.SideTypeSell: + s.mu.Lock() s.CoveredPosition = s.CoveredPosition.Add(quantity) + s.mu.Unlock() case types.SideTypeBuy: + s.mu.Lock() s.CoveredPosition = s.CoveredPosition.Add(quantity.Neg()) + s.mu.Unlock() } } @@ -597,11 +584,11 @@ func (s *Strategy) runTradeRecover(ctx context.Context) { if s.RecoverTrade { startTime := time.Now().Add(-tradeScanInterval).Add(-tradeScanOverlapBufferPeriod) - if err := s.tradeCollector.Recover(ctx, s.hedgeSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { + if err := s.HedgeOrderExecutor.TradeCollector().Recover(ctx, s.hedgeSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { log.WithError(err).Errorf("query trades error") } - if err := s.tradeCollector.Recover(ctx, s.makerSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { + if err := s.MakerOrderExecutor.TradeCollector().Recover(ctx, s.makerSession.Exchange.(types.ExchangeTradeHistoryService), s.Symbol, startTime); err != nil { log.WithError(err).Errorf("query trades error") } } @@ -853,17 +840,15 @@ func (s *Strategy) updateQuote(ctx context.Context, maxLayer int) { return } - createdOrders, err := s.MakerOrderExecutor.SubmitOrders(ctx, submitOrders...) + _, err = s.MakerOrderExecutor.SubmitOrders(ctx, submitOrders...) if err != nil { log.WithError(err).Errorf("order error: %s", err.Error()) return } - - s.orderStore.Add(createdOrders...) } -func (s *Strategy) cleanUpOpenOrders(ctx context.Context) error { - openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, s.makerSession.Exchange, s.Symbol) +func (s *Strategy) cleanUpOpenOrders(ctx context.Context, session *bbgo.ExchangeSession) error { + openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, session.Exchange, s.Symbol) if err != nil { return err } @@ -875,7 +860,7 @@ func (s *Strategy) cleanUpOpenOrders(ctx context.Context) error { log.Infof("found existing open orders:") types.OrderSlice(openOrders).Print() - if err := s.makerSession.Exchange.CancelOrders(ctx, openOrders...); err != nil { + if err := session.Exchange.CancelOrders(ctx, openOrders...); err != nil { return err } From 58321e8aa529295750bd5a62de45f7a5bd4190cb Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 22:20:40 +0800 Subject: [PATCH 352/422] xdepthmaker: update instance id format --- pkg/strategy/xdepthmaker/strategy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index ad2e9a37f2..4ff76b85df 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -218,7 +218,7 @@ func (s *Strategy) ID() string { } func (s *Strategy) InstanceID() string { - return fmt.Sprintf("%s:%s", ID, s.Symbol) + return fmt.Sprintf("%s:%s:%s-%s", ID, s.Symbol, s.MakerExchange, s.HedgeExchange) } func (s *Strategy) Initialize() error { From 3ba16215901b74480b762068ba4e8274f3aa4055 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 22:28:20 +0800 Subject: [PATCH 353/422] xdepthmaker: simplify covered handler registration --- pkg/strategy/xdepthmaker/strategy.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/strategy/xdepthmaker/strategy.go b/pkg/strategy/xdepthmaker/strategy.go index 4ff76b85df..26c24ebea1 100644 --- a/pkg/strategy/xdepthmaker/strategy.go +++ b/pkg/strategy/xdepthmaker/strategy.go @@ -123,7 +123,7 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // bbgo.Sync(ctx, s) }) - coveredFunc := func(trade types.Trade, profit, netProfit fixedpoint.Value) { + s.HedgeOrderExecutor.TradeCollector().OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { c := trade.PositionChange() // sync covered position @@ -133,15 +133,12 @@ func (s *CrossExchangeMarketMakingStrategy) Initialize( // buy trade -> positive delta -> // 1) short position -> reduce short position // 2) short position -> increase short position - if trade.Exchange == s.hedgeSession.ExchangeName { - // TODO: make this atomic - s.mu.Lock() - s.CoveredPosition = s.CoveredPosition.Add(c) - s.mu.Unlock() - } - } - s.MakerOrderExecutor.TradeCollector().OnTrade(coveredFunc) - s.HedgeOrderExecutor.TradeCollector().OnTrade(coveredFunc) + + // TODO: make this atomic + s.mu.Lock() + s.CoveredPosition = s.CoveredPosition.Add(c) + s.mu.Unlock() + }) return nil } From b46b82d41417289f0a99e401c3932c1ffef0827a Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 22:47:28 +0800 Subject: [PATCH 354/422] update command doc files --- doc/commands/bbgo.md | 2 +- doc/commands/bbgo_account.md | 2 +- doc/commands/bbgo_backtest.md | 2 +- doc/commands/bbgo_balances.md | 2 +- doc/commands/bbgo_build.md | 2 +- doc/commands/bbgo_cancel-order.md | 2 +- doc/commands/bbgo_deposits.md | 2 +- doc/commands/bbgo_execute-order.md | 2 +- doc/commands/bbgo_get-order.md | 2 +- doc/commands/bbgo_hoptimize.md | 2 +- doc/commands/bbgo_kline.md | 2 +- doc/commands/bbgo_list-orders.md | 2 +- doc/commands/bbgo_margin.md | 2 +- doc/commands/bbgo_margin_interests.md | 2 +- doc/commands/bbgo_margin_loans.md | 2 +- doc/commands/bbgo_margin_repays.md | 2 +- doc/commands/bbgo_market.md | 2 +- doc/commands/bbgo_optimize.md | 2 +- doc/commands/bbgo_orderbook.md | 2 +- doc/commands/bbgo_orderupdate.md | 2 +- doc/commands/bbgo_pnl.md | 2 +- doc/commands/bbgo_run.md | 2 +- doc/commands/bbgo_submit-order.md | 2 +- doc/commands/bbgo_sync.md | 2 +- doc/commands/bbgo_trades.md | 2 +- doc/commands/bbgo_tradeupdate.md | 2 +- doc/commands/bbgo_transfer-history.md | 2 +- doc/commands/bbgo_userdatastream.md | 2 +- doc/commands/bbgo_version.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/commands/bbgo.md b/doc/commands/bbgo.md index c7becef6ab..ec0c55243f 100644 --- a/doc/commands/bbgo.md +++ b/doc/commands/bbgo.md @@ -58,4 +58,4 @@ bbgo [flags] * [bbgo userdatastream](bbgo_userdatastream.md) - Listen to session events (orderUpdate, tradeUpdate, balanceUpdate, balanceSnapshot) * [bbgo version](bbgo_version.md) - show version name -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_account.md b/doc/commands/bbgo_account.md index de10c300e7..44f938d3b9 100644 --- a/doc/commands/bbgo_account.md +++ b/doc/commands/bbgo_account.md @@ -41,4 +41,4 @@ bbgo account [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_backtest.md b/doc/commands/bbgo_backtest.md index cf74aac091..3aa1490f5a 100644 --- a/doc/commands/bbgo_backtest.md +++ b/doc/commands/bbgo_backtest.md @@ -50,4 +50,4 @@ bbgo backtest [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_balances.md b/doc/commands/bbgo_balances.md index 0d73e38eeb..e18e22124e 100644 --- a/doc/commands/bbgo_balances.md +++ b/doc/commands/bbgo_balances.md @@ -40,4 +40,4 @@ bbgo balances [--session SESSION] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_build.md b/doc/commands/bbgo_build.md index f88eabf6ff..057abe164e 100644 --- a/doc/commands/bbgo_build.md +++ b/doc/commands/bbgo_build.md @@ -39,4 +39,4 @@ bbgo build [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_cancel-order.md b/doc/commands/bbgo_cancel-order.md index 62e24a39cf..42e8cf642a 100644 --- a/doc/commands/bbgo_cancel-order.md +++ b/doc/commands/bbgo_cancel-order.md @@ -49,4 +49,4 @@ bbgo cancel-order [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_deposits.md b/doc/commands/bbgo_deposits.md index 61765bec17..f9e9e2c0aa 100644 --- a/doc/commands/bbgo_deposits.md +++ b/doc/commands/bbgo_deposits.md @@ -41,4 +41,4 @@ bbgo deposits [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_execute-order.md b/doc/commands/bbgo_execute-order.md index 054282c434..9cd8df10f6 100644 --- a/doc/commands/bbgo_execute-order.md +++ b/doc/commands/bbgo_execute-order.md @@ -48,4 +48,4 @@ bbgo execute-order --session SESSION --symbol SYMBOL --side SIDE --target-quanti * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_get-order.md b/doc/commands/bbgo_get-order.md index 655b7b1db4..3112c23f63 100644 --- a/doc/commands/bbgo_get-order.md +++ b/doc/commands/bbgo_get-order.md @@ -42,4 +42,4 @@ bbgo get-order --session SESSION --order-id ORDER_ID [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_hoptimize.md b/doc/commands/bbgo_hoptimize.md index b4d8a4c86c..58c0990af3 100644 --- a/doc/commands/bbgo_hoptimize.md +++ b/doc/commands/bbgo_hoptimize.md @@ -45,4 +45,4 @@ bbgo hoptimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_kline.md b/doc/commands/bbgo_kline.md index 787cc3aad0..5ea8cfb0b4 100644 --- a/doc/commands/bbgo_kline.md +++ b/doc/commands/bbgo_kline.md @@ -42,4 +42,4 @@ bbgo kline [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_list-orders.md b/doc/commands/bbgo_list-orders.md index b3611eb70d..bc0bcfc3c5 100644 --- a/doc/commands/bbgo_list-orders.md +++ b/doc/commands/bbgo_list-orders.md @@ -41,4 +41,4 @@ bbgo list-orders open|closed --session SESSION --symbol SYMBOL [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_margin.md b/doc/commands/bbgo_margin.md index 917edfa81f..7e8803a6c0 100644 --- a/doc/commands/bbgo_margin.md +++ b/doc/commands/bbgo_margin.md @@ -38,4 +38,4 @@ margin related history * [bbgo margin loans](bbgo_margin_loans.md) - query loans history * [bbgo margin repays](bbgo_margin_repays.md) - query repay history -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_margin_interests.md b/doc/commands/bbgo_margin_interests.md index b31e498fa9..5d033609a8 100644 --- a/doc/commands/bbgo_margin_interests.md +++ b/doc/commands/bbgo_margin_interests.md @@ -41,4 +41,4 @@ bbgo margin interests --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_margin_loans.md b/doc/commands/bbgo_margin_loans.md index c13eb042a2..0fe163777b 100644 --- a/doc/commands/bbgo_margin_loans.md +++ b/doc/commands/bbgo_margin_loans.md @@ -41,4 +41,4 @@ bbgo margin loans --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_margin_repays.md b/doc/commands/bbgo_margin_repays.md index ab8bbc930b..f068d5a6ad 100644 --- a/doc/commands/bbgo_margin_repays.md +++ b/doc/commands/bbgo_margin_repays.md @@ -41,4 +41,4 @@ bbgo margin repays --session=SESSION_NAME --asset=ASSET [flags] * [bbgo margin](bbgo_margin.md) - margin related history -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_market.md b/doc/commands/bbgo_market.md index 5519dbe935..5240a9ec8d 100644 --- a/doc/commands/bbgo_market.md +++ b/doc/commands/bbgo_market.md @@ -40,4 +40,4 @@ bbgo market [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_optimize.md b/doc/commands/bbgo_optimize.md index 26e3ac5a0a..d885ddde14 100644 --- a/doc/commands/bbgo_optimize.md +++ b/doc/commands/bbgo_optimize.md @@ -44,4 +44,4 @@ bbgo optimize [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_orderbook.md b/doc/commands/bbgo_orderbook.md index 2e3dc3a6d7..e1a4ad178c 100644 --- a/doc/commands/bbgo_orderbook.md +++ b/doc/commands/bbgo_orderbook.md @@ -42,4 +42,4 @@ bbgo orderbook --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_orderupdate.md b/doc/commands/bbgo_orderupdate.md index 2c79b7b303..b7f5dd0028 100644 --- a/doc/commands/bbgo_orderupdate.md +++ b/doc/commands/bbgo_orderupdate.md @@ -40,4 +40,4 @@ bbgo orderupdate [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_pnl.md b/doc/commands/bbgo_pnl.md index cf60397934..90a574670a 100644 --- a/doc/commands/bbgo_pnl.md +++ b/doc/commands/bbgo_pnl.md @@ -49,4 +49,4 @@ bbgo pnl [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_run.md b/doc/commands/bbgo_run.md index ae2792f44d..0ab6b1cd08 100644 --- a/doc/commands/bbgo_run.md +++ b/doc/commands/bbgo_run.md @@ -51,4 +51,4 @@ bbgo run [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_submit-order.md b/doc/commands/bbgo_submit-order.md index cec954d4d1..0151e8bd9b 100644 --- a/doc/commands/bbgo_submit-order.md +++ b/doc/commands/bbgo_submit-order.md @@ -46,4 +46,4 @@ bbgo submit-order --session SESSION --symbol SYMBOL --side SIDE --quantity QUANT * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_sync.md b/doc/commands/bbgo_sync.md index 21a31271c9..33952990ca 100644 --- a/doc/commands/bbgo_sync.md +++ b/doc/commands/bbgo_sync.md @@ -42,4 +42,4 @@ bbgo sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/d * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_trades.md b/doc/commands/bbgo_trades.md index b2410eb61d..15f0a166e3 100644 --- a/doc/commands/bbgo_trades.md +++ b/doc/commands/bbgo_trades.md @@ -42,4 +42,4 @@ bbgo trades --session=[exchange_name] --symbol=[pair_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_tradeupdate.md b/doc/commands/bbgo_tradeupdate.md index 34c55f3176..ddf20b63dc 100644 --- a/doc/commands/bbgo_tradeupdate.md +++ b/doc/commands/bbgo_tradeupdate.md @@ -40,4 +40,4 @@ bbgo tradeupdate --session=[exchange_name] [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_transfer-history.md b/doc/commands/bbgo_transfer-history.md index 62e7583dfc..f91b9cc0c7 100644 --- a/doc/commands/bbgo_transfer-history.md +++ b/doc/commands/bbgo_transfer-history.md @@ -42,4 +42,4 @@ bbgo transfer-history [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_userdatastream.md b/doc/commands/bbgo_userdatastream.md index ea920b7582..3a3f979997 100644 --- a/doc/commands/bbgo_userdatastream.md +++ b/doc/commands/bbgo_userdatastream.md @@ -40,4 +40,4 @@ bbgo userdatastream [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 diff --git a/doc/commands/bbgo_version.md b/doc/commands/bbgo_version.md index 8a5961466f..851b880bb5 100644 --- a/doc/commands/bbgo_version.md +++ b/doc/commands/bbgo_version.md @@ -39,4 +39,4 @@ bbgo version [flags] * [bbgo](bbgo.md) - bbgo is a crypto trading bot -###### Auto generated by spf13/cobra on 18-Dec-2023 +###### Auto generated by spf13/cobra on 20-Dec-2023 From f29238788665d84d3dcea65800e68c312f3e51fc Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 22:47:29 +0800 Subject: [PATCH 355/422] bump version to v1.55.4 --- pkg/version/dev.go | 4 ++-- pkg/version/version.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/version/dev.go b/pkg/version/dev.go index bb47e6c9e8..5ccc4f5c1a 100644 --- a/pkg/version/dev.go +++ b/pkg/version/dev.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.3-083f6265-dev" +const Version = "v1.55.4-26f7d869-dev" -const VersionGitRef = "083f6265" +const VersionGitRef = "26f7d869" diff --git a/pkg/version/version.go b/pkg/version/version.go index afaa1e04aa..324fe11985 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -3,6 +3,6 @@ package version -const Version = "v1.55.3-083f6265" +const Version = "v1.55.4-26f7d869" -const VersionGitRef = "083f6265" +const VersionGitRef = "26f7d869" From 00c9f2ed71d5760c85da7019817a818a9689da23 Mon Sep 17 00:00:00 2001 From: c9s Date: Wed, 20 Dec 2023 22:47:29 +0800 Subject: [PATCH 356/422] add v1.55.4 release note --- doc/release/v1.55.4.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release/v1.55.4.md diff --git a/doc/release/v1.55.4.md b/doc/release/v1.55.4.md new file mode 100644 index 0000000000..fdaf5a8316 --- /dev/null +++ b/doc/release/v1.55.4.md @@ -0,0 +1,5 @@ +[Full Changelog](https://github.com/c9s/bbgo/compare/v1.55.3...main) + + - [#1466](https://github.com/c9s/bbgo/pull/1466): FIX: [xdepthmaker] remove shared trade collector and fix hedge + - [#1393](https://github.com/c9s/bbgo/pull/1393): STRATEGY: add emacross strategy + - [#1462](https://github.com/c9s/bbgo/pull/1462): FEATURE: [bollmaker] support custom quantity From 9870ea0d6c579c10615ceafa662a72e59425f839 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 12:22:48 +0800 Subject: [PATCH 357/422] improve/db: add futures kilne tables --- .../20231221121432_add_futures_klines.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 pkg/migrations/mysql/20231221121432_add_futures_klines.go diff --git a/pkg/migrations/mysql/20231221121432_add_futures_klines.go b/pkg/migrations/mysql/20231221121432_add_futures_klines.go new file mode 100644 index 0000000000..350cc7f31e --- /dev/null +++ b/pkg/migrations/mysql/20231221121432_add_futures_klines.go @@ -0,0 +1,64 @@ +package mysql + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upAddBybitKlines, downAddBybitKlines) + +} + +func upAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_futures_klines` LIKE `binance_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `bybit_futures_klines` LIKE `bybit_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_futures_klines` LIKE `okex_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_futures_klines` LIKE `max_klines`;") + if err != nil { + return err + } + + return err +} + +func downAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `binance_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `bybit_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `okex_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `max_futures_klines`;") + if err != nil { + return err + } + + return err +} From d5cbcc3fb2b8f4a2fcf069a65b3a090fc61b485f Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 12:50:38 +0800 Subject: [PATCH 358/422] improve/db: add futures kilne sqlite tables --- .../20231221121432_add_futures_klines.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 pkg/migrations/sqlite3/20231221121432_add_futures_klines.go diff --git a/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go b/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go new file mode 100644 index 0000000000..8554e49d65 --- /dev/null +++ b/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go @@ -0,0 +1,64 @@ +package sqlite3 + +import ( + "context" + + "github.com/c9s/rockhopper" +) + +func init() { + AddMigration(upAddFuturesKlines, downAddFuturesKlines) + +} + +func upAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is applied. + + _, err = tx.ExecContext(ctx, "CREATE TABLE `bybit_futures_klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0,\n `quote_volume` DECIMAL NOT NULL DEFAULT 0.0,\n `taker_buy_base_volume` DECIMAL NOT NULL DEFAULT 0.0,\n `taker_buy_quote_volume` DECIMAL NOT NULL DEFAULT 0.0\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `okex_futures_klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `binance_futures_klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0\n);") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "CREATE TABLE `max_futures_klines`\n(\n `gid` INTEGER PRIMARY KEY AUTOINCREMENT,\n `exchange` VARCHAR(10) NOT NULL,\n `start_time` DATETIME(3) NOT NULL,\n `end_time` DATETIME(3) NOT NULL,\n `interval` VARCHAR(3) NOT NULL,\n `symbol` VARCHAR(7) NOT NULL,\n `open` DECIMAL(16, 8) NOT NULL,\n `high` DECIMAL(16, 8) NOT NULL,\n `low` DECIMAL(16, 8) NOT NULL,\n `close` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `volume` DECIMAL(16, 8) NOT NULL DEFAULT 0.0,\n `closed` BOOLEAN NOT NULL DEFAULT TRUE,\n `last_trade_id` INT NOT NULL DEFAULT 0,\n `num_trades` INT NOT NULL DEFAULT 0\n);") + if err != nil { + return err + } + + return err +} + +func downAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { + // This code is executed when the migration is rolled back. + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `bybit_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `okex_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `binance_futures_klines`;") + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `max_futures_klines`;") + if err != nil { + return err + } + + return err +} From 5b0b5428fb94676b5e4e2f15529a5cda7bf949c4 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 15:47:24 +0800 Subject: [PATCH 359/422] improve/db: query futures kilne if session 'futures' is true when sync --- pkg/service/backtest.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/service/backtest.go b/pkg/service/backtest.go index 5a609ff8f0..28f9aaf202 100644 --- a/pkg/service/backtest.go +++ b/pkg/service/backtest.go @@ -25,13 +25,26 @@ type BacktestService struct { func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error { log.Infof("synchronizing %s klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) - // TODO: use isFutures here - _, _, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(exchange) + _, isFutures, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(exchange) + // override symbol if isolatedSymbol is not empty if isIsolated && len(isolatedSymbol) > 0 { symbol = isolatedSymbol } + if isFutures { + futuresExchange, ok := exchange.(types.FuturesExchange) + if !ok { + return fmt.Errorf("exchange %s does not support futures", exchange.Name()) + } + + if isIsolated { + futuresExchange.UseIsolatedFutures(symbol) + } else { + futuresExchange.UseFutures() + } + } + if s.DB.DriverName() == "sqlite3" { _, _ = s.DB.Exec("PRAGMA journal_mode = WAL") _, _ = s.DB.Exec("PRAGMA synchronous = NORMAL") From 8ecba4378c70909a45356cad5903735b1728797e Mon Sep 17 00:00:00 2001 From: narumi Date: Wed, 8 Nov 2023 17:59:37 +0800 Subject: [PATCH 360/422] inventory skew --- pkg/strategy/fixedmaker/inventory_skew.go | 43 ++++++++++++ .../fixedmaker/inventory_skew_test.go | 69 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 pkg/strategy/fixedmaker/inventory_skew.go create mode 100644 pkg/strategy/fixedmaker/inventory_skew_test.go diff --git a/pkg/strategy/fixedmaker/inventory_skew.go b/pkg/strategy/fixedmaker/inventory_skew.go new file mode 100644 index 0000000000..fbbc03c60d --- /dev/null +++ b/pkg/strategy/fixedmaker/inventory_skew.go @@ -0,0 +1,43 @@ +package fixedmaker + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" +) + +var ( + zero = fixedpoint.Zero + two = fixedpoint.NewFromFloat(2.0) +) + +type InventorySkewBidAskRatios struct { + bidRatio fixedpoint.Value + askRatio fixedpoint.Value +} + +// https://hummingbot.org/strategy-configs/inventory-skew/ +// https://github.com/hummingbot/hummingbot/blob/31fc61d5e71b2c15732142d30983f3ea2be4d466/hummingbot/strategy/pure_market_making/inventory_skew_calculator.pyx +type InventorySkew struct { + InventoryRangeMultiplier fixedpoint.Value `json:"inventoryRangeMultiplier"` + TargetBaseRatio fixedpoint.Value `json:"targetBaseRatio"` +} + +func (s *InventorySkew) CalculateBidAskRatios(quantity fixedpoint.Value, price fixedpoint.Value, baseBalance fixedpoint.Value, quoteBalance fixedpoint.Value) *InventorySkewBidAskRatios { + baseValue := baseBalance.Mul(price) + totalValue := baseValue.Add(quoteBalance) + + inventoryRange := s.InventoryRangeMultiplier.Mul(quantity.Mul(two)).Mul(price) + leftLimit := s.TargetBaseRatio.Mul(totalValue).Sub(inventoryRange) + rightLimit := s.TargetBaseRatio.Mul(totalValue).Add(inventoryRange) + + bidAdjustment := interp(baseValue, leftLimit, rightLimit, two, zero).Clamp(zero, two) + askAdjustment := interp(baseValue, leftLimit, rightLimit, zero, two).Clamp(zero, two) + + return &InventorySkewBidAskRatios{ + bidRatio: bidAdjustment, + askRatio: askAdjustment, + } +} + +func interp(x, x0, x1, y0, y1 fixedpoint.Value) fixedpoint.Value { + return y0.Add(x.Sub(x0).Mul(y1.Sub(y0)).Div(x1.Sub(x0))) +} diff --git a/pkg/strategy/fixedmaker/inventory_skew_test.go b/pkg/strategy/fixedmaker/inventory_skew_test.go new file mode 100644 index 0000000000..8c73139973 --- /dev/null +++ b/pkg/strategy/fixedmaker/inventory_skew_test.go @@ -0,0 +1,69 @@ +package fixedmaker + +import ( + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/stretchr/testify/assert" +) + +func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { + cases := []struct { + quantity fixedpoint.Value + price fixedpoint.Value + baseBalance fixedpoint.Value + quoteBalance fixedpoint.Value + want *InventorySkewBidAskRatios + }{ + { + quantity: fixedpoint.NewFromFloat(1.0), + price: fixedpoint.NewFromFloat(1000), + baseBalance: fixedpoint.NewFromFloat(1.0), + quoteBalance: fixedpoint.NewFromFloat(1000), + want: &InventorySkewBidAskRatios{ + bidRatio: fixedpoint.NewFromFloat(1.0), + askRatio: fixedpoint.NewFromFloat(1.0), + }, + }, + { + quantity: fixedpoint.NewFromFloat(1.0), + price: fixedpoint.NewFromFloat(1000), + baseBalance: fixedpoint.NewFromFloat(1.0), + quoteBalance: fixedpoint.NewFromFloat(1200), + want: &InventorySkewBidAskRatios{ + bidRatio: fixedpoint.NewFromFloat(1.5), + askRatio: fixedpoint.NewFromFloat(0.5), + }, + }, + { + quantity: fixedpoint.NewFromFloat(1.0), + price: fixedpoint.NewFromFloat(1000), + baseBalance: fixedpoint.NewFromFloat(0.0), + quoteBalance: fixedpoint.NewFromFloat(10000), + want: &InventorySkewBidAskRatios{ + bidRatio: fixedpoint.NewFromFloat(2.0), + askRatio: fixedpoint.NewFromFloat(0.0), + }, + }, + { + quantity: fixedpoint.NewFromFloat(1.0), + price: fixedpoint.NewFromFloat(1000), + baseBalance: fixedpoint.NewFromFloat(2.0), + quoteBalance: fixedpoint.NewFromFloat(0.0), + want: &InventorySkewBidAskRatios{ + bidRatio: fixedpoint.NewFromFloat(0.0), + askRatio: fixedpoint.NewFromFloat(2.0), + }, + }, + } + + for _, c := range cases { + s := &InventorySkew{ + InventoryRangeMultiplier: fixedpoint.NewFromFloat(0.1), + TargetBaseRatio: fixedpoint.NewFromFloat(0.5), + } + got := s.CalculateBidAskRatios(c.quantity, c.price, c.baseBalance, c.quoteBalance) + assert.Equal(t, c.want.bidRatio.Float64(), got.bidRatio.Float64()) + assert.Equal(t, c.want.askRatio.Float64(), got.askRatio.Float64()) + } +} From 6809efa6960fe004524c80e806a8ca1d637565d0 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 16:19:32 +0800 Subject: [PATCH 361/422] improve/db: save futures kilne to futures table --- pkg/service/backtest.go | 50 ++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/pkg/service/backtest.go b/pkg/service/backtest.go index 28f9aaf202..1af6e370a5 100644 --- a/pkg/service/backtest.go +++ b/pkg/service/backtest.go @@ -19,12 +19,11 @@ import ( ) type BacktestService struct { - DB *sqlx.DB + DB *sqlx.DB + Futures bool } func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error { - log.Infof("synchronizing %s klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) - _, isFutures, isIsolated, isolatedSymbol := exchange2.GetSessionAttributes(exchange) // override symbol if isolatedSymbol is not empty @@ -32,7 +31,8 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type symbol = isolatedSymbol } - if isFutures { + s.Futures = isFutures + if s.Futures { futuresExchange, ok := exchange.(types.FuturesExchange) if !ok { return fmt.Errorf("exchange %s does not support futures", exchange.Name()) @@ -43,6 +43,9 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type } else { futuresExchange.UseFutures() } + log.Infof("synchronizing %s futures klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) + } else { + log.Infof("synchronizing %s klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) } if s.DB.DriverName() == "sqlite3" { @@ -54,7 +57,7 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type tasks := []SyncTask{ { Type: types.KLine{}, - Select: SelectLastKLines(exchange.Name(), symbol, interval, startTime, endTime, 100), + Select: s.SelectLastKLines(exchange.Name(), symbol, interval, startTime, endTime, 100), Time: func(obj interface{}) time.Time { return obj.(types.KLine).StartTime.Time() }, @@ -147,7 +150,7 @@ func (s *BacktestService) SyncFresh(ctx context.Context, exchange types.Exchange func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) { log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval) - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex) // make the SQL syntax IDE friendly, so that it can analyze it. sql := fmt.Sprintf("SELECT * FROM `%s` WHERE `symbol` = :symbol AND `interval` = :interval ORDER BY end_time "+orderBy+" LIMIT "+strconv.Itoa(limit), tableName) @@ -176,7 +179,7 @@ func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, inter // QueryKLinesForward is used for querying klines to back-testing func (s *BacktestService) QueryKLinesForward(exchange types.ExchangeName, symbol string, interval types.Interval, startTime time.Time, limit int) ([]types.KLine, error) { - tableName := targetKlineTable(exchange) + tableName := s.targetKlineTable(exchange) sql := "SELECT * FROM `binance_klines` WHERE `end_time` >= :start_time AND `symbol` = :symbol AND `interval` = :interval and exchange = :exchange ORDER BY end_time ASC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -195,7 +198,7 @@ func (s *BacktestService) QueryKLinesForward(exchange types.ExchangeName, symbol } func (s *BacktestService) QueryKLinesBackward(exchange types.ExchangeName, symbol string, interval types.Interval, endTime time.Time, limit int) ([]types.KLine, error) { - tableName := targetKlineTable(exchange) + tableName := s.targetKlineTable(exchange) sql := "SELECT * FROM `binance_klines` WHERE `end_time` <= :end_time and exchange = :exchange AND `symbol` = :symbol AND `interval` = :interval ORDER BY end_time DESC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -220,7 +223,7 @@ func (s *BacktestService) QueryKLinesCh(since, until time.Time, exchange types.E return returnError(errors.Errorf("symbols is empty when querying kline, plesae check your strategy setting. ")) } - tableName := targetKlineTable(exchange.Name()) + tableName := s.targetKlineTable(exchange.Name()) var query string // need to sort by start_time desc in order to let matching engine process 1m first @@ -313,8 +316,13 @@ func (s *BacktestService) scanRows(rows *sqlx.Rows) (klines []types.KLine, err e return klines, rows.Err() } -func targetKlineTable(exchangeName types.ExchangeName) string { - return strings.ToLower(exchangeName.String()) + "_klines" +func (s *BacktestService) targetKlineTable(exchangeName types.ExchangeName) string { + tableName := strings.ToLower(exchangeName.String()) + if s.Futures { + return tableName + "_futures_klines" + } else { + return tableName + "_klines" + } } var errExchangeFieldIsUnset = errors.New("kline.Exchange field should not be empty") @@ -324,7 +332,7 @@ func (s *BacktestService) Insert(kline types.KLine) error { return errExchangeFieldIsUnset } - tableName := targetKlineTable(kline.Exchange) + tableName := s.targetKlineTable(kline.Exchange) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ "VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume)", tableName) @@ -339,7 +347,7 @@ func (s *BacktestService) BatchInsert(kline []types.KLine) error { return nil } - tableName := targetKlineTable(kline[0].Exchange) + tableName := s.targetKlineTable(kline[0].Exchange) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ " VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume); ", tableName) @@ -434,7 +442,7 @@ func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, sy // FindMissingTimeRanges returns the missing time ranges, the start/end time represents the existing data time points. // So when sending kline query to the exchange API, we need to add one second to the start time and minus one second to the end time. func (s *BacktestService) FindMissingTimeRanges(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) ([]TimeRange, error) { - query := SelectKLineTimePoints(ex.Name(), symbol, interval, since, until) + query := s.SelectKLineTimePoints(ex.Name(), symbol, interval, since, until) sql, args, err := query.ToSql() if err != nil { return nil, err @@ -477,7 +485,7 @@ func (s *BacktestService) FindMissingTimeRanges(ctx context.Context, ex types.Ex } func (s *BacktestService) QueryExistingDataRange(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, tArgs ...time.Time) (start, end *types.Time, err error) { - sel := SelectKLineTimeRange(ex.Name(), symbol, interval, tArgs...) + sel := s.SelectKLineTimeRange(ex.Name(), symbol, interval, tArgs...) sql, args, err := sel.ToSql() if err != nil { return nil, nil, err @@ -502,7 +510,7 @@ func (s *BacktestService) QueryExistingDataRange(ctx context.Context, ex types.E return &t1, &t2, nil } -func SelectKLineTimePoints(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { +func (s *BacktestService) SelectKLineTimePoints(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { conditions := sq.And{ sq.Eq{"symbol": symbol}, sq.Eq{"`interval`": interval.String()}, @@ -514,7 +522,7 @@ func SelectKLineTimePoints(ex types.ExchangeName, symbol string, interval types. conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex) return sq.Select("start_time"). From(tableName). @@ -523,7 +531,7 @@ func SelectKLineTimePoints(ex types.ExchangeName, symbol string, interval types. } // SelectKLineTimeRange returns the existing klines time range (since < kline.start_time < until) -func SelectKLineTimeRange(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { +func (s *BacktestService) SelectKLineTimeRange(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { conditions := sq.And{ sq.Eq{"symbol": symbol}, sq.Eq{"`interval`": interval.String()}, @@ -538,7 +546,7 @@ func SelectKLineTimeRange(ex types.ExchangeName, symbol string, interval types.I conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex) return sq.Select("MIN(start_time) AS t1, MAX(start_time) AS t2"). From(tableName). @@ -546,8 +554,8 @@ func SelectKLineTimeRange(ex types.ExchangeName, symbol string, interval types.I } // TODO: add is_futures column since the klines data is different -func SelectLastKLines(ex types.ExchangeName, symbol string, interval types.Interval, startTime, endTime time.Time, limit uint64) sq.SelectBuilder { - tableName := targetKlineTable(ex) +func (s *BacktestService) SelectLastKLines(ex types.ExchangeName, symbol string, interval types.Interval, startTime, endTime time.Time, limit uint64) sq.SelectBuilder { + tableName := s.targetKlineTable(ex) return sq.Select("*"). From(tableName). Where(sq.And{ From f160ea856fc2c0a03b3245936424e894137a7983 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Thu, 21 Dec 2023 16:29:46 +0800 Subject: [PATCH 362/422] apply inventory-skew to fixedmaker --- config/fixedmaker.yaml | 11 +++++++-- pkg/strategy/fixedmaker/inventory_skew.go | 13 +++++++++++ pkg/strategy/fixedmaker/strategy.go | 27 ++++++++++++++++++++--- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/config/fixedmaker.yaml b/config/fixedmaker.yaml index 63d3ee058d..d66a426f73 100644 --- a/config/fixedmaker.yaml +++ b/config/fixedmaker.yaml @@ -16,12 +16,19 @@ exchangeStrategies: - on: max fixedmaker: symbol: USDCUSDT - interval: 5m + interval: 1m halfSpread: 0.05% quantity: 15 orderType: LIMIT_MAKER - dryRun: false + dryRun: true positionHardLimit: 1500 maxPositionQuantity: 1500 circuitBreakLossThreshold: -0.15 + circuitBreakEMA: + interval: 1m + window: 14 + + inventorySkew: + inventoryRangeMultiplier: 1.0 + targetBaseRatio: 0.5 diff --git a/pkg/strategy/fixedmaker/inventory_skew.go b/pkg/strategy/fixedmaker/inventory_skew.go index fbbc03c60d..0dafe474ab 100644 --- a/pkg/strategy/fixedmaker/inventory_skew.go +++ b/pkg/strategy/fixedmaker/inventory_skew.go @@ -1,6 +1,8 @@ package fixedmaker import ( + "fmt" + "github.com/c9s/bbgo/pkg/fixedpoint" ) @@ -21,6 +23,17 @@ type InventorySkew struct { TargetBaseRatio fixedpoint.Value `json:"targetBaseRatio"` } +func (s *InventorySkew) Validate() error { + if s.InventoryRangeMultiplier.Float64() < 0 { + return fmt.Errorf("inventoryRangeMultiplier should be positive") + } + + if s.TargetBaseRatio.Float64() < 0 { + return fmt.Errorf("targetBaseRatio should be positive") + } + return nil +} + func (s *InventorySkew) CalculateBidAskRatios(quantity fixedpoint.Value, price fixedpoint.Value, baseBalance fixedpoint.Value, quoteBalance fixedpoint.Value) *InventorySkewBidAskRatios { baseValue := baseBalance.Mul(price) totalValue := baseValue.Add(quoteBalance) diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index af561de09f..f08f6c4b44 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -35,6 +35,8 @@ type Strategy struct { OrderType types.OrderType `json:"orderType"` DryRun bool `json:"dryRun"` + InventorySkew InventorySkew `json:"inventorySkew"` + activeOrderBook *bbgo.ActiveOrderBook } @@ -70,6 +72,10 @@ func (s *Strategy) Validate() error { if s.HalfSpread.Float64() <= 0 { return fmt.Errorf("halfSpread should be positive") } + + if err := s.InventorySkew.Validate(); err != nil { + return err + } return nil } @@ -123,7 +129,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } func (s *Strategy) cancelOrders(ctx context.Context) { - if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.activeOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { log.WithError(err).Errorf("failed to cancel orders") } } @@ -180,6 +186,21 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err buyPrice := midPrice.Mul(fixedpoint.One.Sub(s.HalfSpread)).Round(s.Market.PricePrecision, fixedpoint.Down) log.Infof("sell price: %s, buy price: %s", sellPrice.String(), buyPrice.String()) + buyQuantity := s.Quantity + sellQuantity := s.Quantity + if !s.InventorySkew.InventoryRangeMultiplier.IsZero() { + ratios := s.InventorySkew.CalculateBidAskRatios( + s.Quantity, + midPrice, + baseBalance.Total(), + quoteBalance.Total(), + ) + log.Infof("bid ratio: %s, ask ratio: %s", ratios.bidRatio.String(), ratios.askRatio.String()) + buyQuantity = s.Quantity.Mul(ratios.bidRatio) + sellQuantity = s.Quantity.Mul(ratios.askRatio) + log.Infof("buy quantity: %s, sell quantity: %s", buyQuantity.String(), sellQuantity.String()) + } + // check balance and generate orders amount := s.Quantity.Mul(buyPrice) if quoteBalance.Available.Compare(amount) > 0 { @@ -188,7 +209,7 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err Side: types.SideTypeBuy, Type: s.OrderType, Price: buyPrice, - Quantity: s.Quantity, + Quantity: buyQuantity, }) } else { log.Infof("not enough quote balance to buy, available: %s, amount: %s", quoteBalance.Available, amount) @@ -200,7 +221,7 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err Side: types.SideTypeSell, Type: s.OrderType, Price: sellPrice, - Quantity: s.Quantity, + Quantity: sellQuantity, }) } else { log.Infof("not enough base balance to sell, available: %s, quantity: %s", baseBalance.Available, s.Quantity) From 7f0a4a9953c655b2b5fc1d8c26f77cf29aee0168 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Thu, 21 Dec 2023 16:34:05 +0800 Subject: [PATCH 363/422] apply inventory-skew to xfixedmaker --- config/xfixedmaker.yaml | 9 +++-- pkg/strategy/fixedmaker/inventory_skew.go | 8 ++--- .../fixedmaker/inventory_skew_test.go | 20 +++++------ pkg/strategy/fixedmaker/strategy.go | 6 ++-- pkg/strategy/xfixedmaker/strategy.go | 33 +++++++++++++++---- 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/config/xfixedmaker.yaml b/config/xfixedmaker.yaml index bfee2e7711..b1ebaf01fb 100644 --- a/config/xfixedmaker.yaml +++ b/config/xfixedmaker.yaml @@ -6,6 +6,7 @@ sessions: binance: exchange: binance envVarPrefix: binance + publicOnly: true backtest: startTime: "2023-01-01" @@ -25,11 +26,11 @@ crossExchangeStrategies: - xfixedmaker: tradingExchange: max symbol: BTCUSDT - interval: 5m + interval: 1m halfSpread: 0.01% quantity: 0.005 orderType: LIMIT_MAKER - dryRun: false + dryRun: true referenceExchange: binance referencePriceEMA: @@ -44,3 +45,7 @@ crossExchangeStrategies: circuitBreakEMA: interval: 1m window: 14 + + inventorySkew: + inventoryRangeMultiplier: 1.0 + targetBaseRatio: 0.5 diff --git a/pkg/strategy/fixedmaker/inventory_skew.go b/pkg/strategy/fixedmaker/inventory_skew.go index 0dafe474ab..dae68e7c45 100644 --- a/pkg/strategy/fixedmaker/inventory_skew.go +++ b/pkg/strategy/fixedmaker/inventory_skew.go @@ -12,8 +12,8 @@ var ( ) type InventorySkewBidAskRatios struct { - bidRatio fixedpoint.Value - askRatio fixedpoint.Value + BidRatio fixedpoint.Value + AskRatio fixedpoint.Value } // https://hummingbot.org/strategy-configs/inventory-skew/ @@ -46,8 +46,8 @@ func (s *InventorySkew) CalculateBidAskRatios(quantity fixedpoint.Value, price f askAdjustment := interp(baseValue, leftLimit, rightLimit, zero, two).Clamp(zero, two) return &InventorySkewBidAskRatios{ - bidRatio: bidAdjustment, - askRatio: askAdjustment, + BidRatio: bidAdjustment, + AskRatio: askAdjustment, } } diff --git a/pkg/strategy/fixedmaker/inventory_skew_test.go b/pkg/strategy/fixedmaker/inventory_skew_test.go index 8c73139973..cd40931a1f 100644 --- a/pkg/strategy/fixedmaker/inventory_skew_test.go +++ b/pkg/strategy/fixedmaker/inventory_skew_test.go @@ -21,8 +21,8 @@ func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { baseBalance: fixedpoint.NewFromFloat(1.0), quoteBalance: fixedpoint.NewFromFloat(1000), want: &InventorySkewBidAskRatios{ - bidRatio: fixedpoint.NewFromFloat(1.0), - askRatio: fixedpoint.NewFromFloat(1.0), + BidRatio: fixedpoint.NewFromFloat(1.0), + AskRatio: fixedpoint.NewFromFloat(1.0), }, }, { @@ -31,8 +31,8 @@ func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { baseBalance: fixedpoint.NewFromFloat(1.0), quoteBalance: fixedpoint.NewFromFloat(1200), want: &InventorySkewBidAskRatios{ - bidRatio: fixedpoint.NewFromFloat(1.5), - askRatio: fixedpoint.NewFromFloat(0.5), + BidRatio: fixedpoint.NewFromFloat(1.5), + AskRatio: fixedpoint.NewFromFloat(0.5), }, }, { @@ -41,8 +41,8 @@ func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { baseBalance: fixedpoint.NewFromFloat(0.0), quoteBalance: fixedpoint.NewFromFloat(10000), want: &InventorySkewBidAskRatios{ - bidRatio: fixedpoint.NewFromFloat(2.0), - askRatio: fixedpoint.NewFromFloat(0.0), + BidRatio: fixedpoint.NewFromFloat(2.0), + AskRatio: fixedpoint.NewFromFloat(0.0), }, }, { @@ -51,8 +51,8 @@ func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { baseBalance: fixedpoint.NewFromFloat(2.0), quoteBalance: fixedpoint.NewFromFloat(0.0), want: &InventorySkewBidAskRatios{ - bidRatio: fixedpoint.NewFromFloat(0.0), - askRatio: fixedpoint.NewFromFloat(2.0), + BidRatio: fixedpoint.NewFromFloat(0.0), + AskRatio: fixedpoint.NewFromFloat(2.0), }, }, } @@ -63,7 +63,7 @@ func Test_InventorySkew_CalculateBidAskRatios(t *testing.T) { TargetBaseRatio: fixedpoint.NewFromFloat(0.5), } got := s.CalculateBidAskRatios(c.quantity, c.price, c.baseBalance, c.quoteBalance) - assert.Equal(t, c.want.bidRatio.Float64(), got.bidRatio.Float64()) - assert.Equal(t, c.want.askRatio.Float64(), got.askRatio.Float64()) + assert.Equal(t, c.want.BidRatio.Float64(), got.BidRatio.Float64()) + assert.Equal(t, c.want.AskRatio.Float64(), got.AskRatio.Float64()) } } diff --git a/pkg/strategy/fixedmaker/strategy.go b/pkg/strategy/fixedmaker/strategy.go index f08f6c4b44..8046636a83 100644 --- a/pkg/strategy/fixedmaker/strategy.go +++ b/pkg/strategy/fixedmaker/strategy.go @@ -195,9 +195,9 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err baseBalance.Total(), quoteBalance.Total(), ) - log.Infof("bid ratio: %s, ask ratio: %s", ratios.bidRatio.String(), ratios.askRatio.String()) - buyQuantity = s.Quantity.Mul(ratios.bidRatio) - sellQuantity = s.Quantity.Mul(ratios.askRatio) + log.Infof("bid ratio: %s, ask ratio: %s", ratios.BidRatio.String(), ratios.AskRatio.String()) + buyQuantity = s.Quantity.Mul(ratios.BidRatio) + sellQuantity = s.Quantity.Mul(ratios.AskRatio) log.Infof("buy quantity: %s, sell quantity: %s", buyQuantity.String(), sellQuantity.String()) } diff --git a/pkg/strategy/xfixedmaker/strategy.go b/pkg/strategy/xfixedmaker/strategy.go index 2e018f5718..45d1bd78a6 100644 --- a/pkg/strategy/xfixedmaker/strategy.go +++ b/pkg/strategy/xfixedmaker/strategy.go @@ -10,6 +10,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/strategy/fixedmaker" "github.com/c9s/bbgo/pkg/types" ) @@ -35,9 +36,10 @@ type Strategy struct { OrderType types.OrderType `json:"orderType"` DryRun bool `json:"dryRun"` - ReferenceExchange string `json:"referenceExchange"` - ReferencePriceEMA types.IntervalWindow `json:"referencePriceEMA"` - OrderPriceLossThreshold fixedpoint.Value `json:"orderPriceLossThreshold"` + ReferenceExchange string `json:"referenceExchange"` + ReferencePriceEMA types.IntervalWindow `json:"referencePriceEMA"` + OrderPriceLossThreshold fixedpoint.Value `json:"orderPriceLossThreshold"` + InventorySkew fixedmaker.InventorySkew `json:"inventorySkew"` market types.Market activeOrderBook *bbgo.ActiveOrderBook @@ -73,6 +75,10 @@ func (s *Strategy) Validate() error { if s.HalfSpread.Float64() <= 0 { return fmt.Errorf("halfSpread should be positive") } + + if err := s.InventorySkew.Validate(); err != nil { + return err + } return nil } @@ -155,7 +161,7 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se } func (s *Strategy) cancelOrders(ctx context.Context) { - if err := s.Session.Exchange.CancelOrders(ctx, s.activeOrderBook.Orders()...); err != nil { + if err := s.activeOrderBook.GracefulCancel(ctx, s.Session.Exchange); err != nil { log.WithError(err).Errorf("failed to cancel orders") } } @@ -212,6 +218,21 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err buyPrice := midPrice.Mul(fixedpoint.One.Sub(s.HalfSpread)).Round(s.market.PricePrecision, fixedpoint.Down) log.Infof("sell price: %s, buy price: %s", sellPrice.String(), buyPrice.String()) + buyQuantity := s.Quantity + sellQuantity := s.Quantity + if !s.InventorySkew.InventoryRangeMultiplier.IsZero() { + ratios := s.InventorySkew.CalculateBidAskRatios( + s.Quantity, + midPrice, + baseBalance.Total(), + quoteBalance.Total(), + ) + log.Infof("bid ratio: %s, ask ratio: %s", ratios.BidRatio.String(), ratios.AskRatio.String()) + buyQuantity = s.Quantity.Mul(ratios.BidRatio) + sellQuantity = s.Quantity.Mul(ratios.AskRatio) + log.Infof("buy quantity: %s, sell quantity: %s", buyQuantity.String(), sellQuantity.String()) + } + // check balance and generate orders amount := s.Quantity.Mul(buyPrice) if quoteBalance.Available.Compare(amount) > 0 { @@ -221,7 +242,7 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err Side: types.SideTypeBuy, Type: s.OrderType, Price: buyPrice, - Quantity: s.Quantity, + Quantity: buyQuantity, }) } else { @@ -238,7 +259,7 @@ func (s *Strategy) generateOrders(ctx context.Context) ([]types.SubmitOrder, err Side: types.SideTypeSell, Type: s.OrderType, Price: sellPrice, - Quantity: s.Quantity, + Quantity: sellQuantity, }) } else { log.Infof("ref price risk control triggered, not placing sell order") From c82cbbc1723fe82a6153f1b208e64cd2c9295618 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 16:52:52 +0800 Subject: [PATCH 364/422] fix/futures-kline-sync: typo --- pkg/migrations/mysql/20231221121432_add_futures_klines.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/migrations/mysql/20231221121432_add_futures_klines.go b/pkg/migrations/mysql/20231221121432_add_futures_klines.go index 350cc7f31e..d30558dfca 100644 --- a/pkg/migrations/mysql/20231221121432_add_futures_klines.go +++ b/pkg/migrations/mysql/20231221121432_add_futures_klines.go @@ -7,7 +7,7 @@ import ( ) func init() { - AddMigration(upAddBybitKlines, downAddBybitKlines) + AddMigration(upAddFuturesKlines, downAddFuturesKlines) } From 66718e0d37fc0aa2ed9bb57ec9d49e5c2d70d8bc Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Thu, 21 Dec 2023 18:19:28 +0800 Subject: [PATCH 365/422] improve/backtest-sync: set exchange to use futures --- pkg/backtest/exchange.go | 4 +-- pkg/cmd/backtest.go | 10 ++++++ pkg/service/backtest.go | 72 +++++++++++++++++----------------------- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index 28a6b49a4e..c459a893a6 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -256,11 +256,11 @@ func (e *Exchange) QueryKLines( ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions, ) ([]types.KLine, error) { if options.EndTime != nil { - return e.srv.QueryKLinesBackward(e.sourceName, symbol, interval, *options.EndTime, 1000) + return e.srv.QueryKLinesBackward(e, symbol, interval, *options.EndTime, 1000) } if options.StartTime != nil { - return e.srv.QueryKLinesForward(e.sourceName, symbol, interval, *options.StartTime, 1000) + return e.srv.QueryKLinesForward(e, symbol, interval, *options.StartTime, 1000) } return nil, errors.New("endTime or startTime can not be nil") diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 979412e9bc..6cd97e1093 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -192,6 +192,16 @@ var BacktestCmd = &cobra.Command{ return err } sourceExchanges[exName] = publicExchange + + // Set exchange to use futures + if userConfig.Sessions[exName.String()].Futures { + futuresExchange, ok := publicExchange.(types.FuturesExchange) + if !ok { + return fmt.Errorf("exchange %s does not support futures", publicExchange.Name()) + } + + futuresExchange.UseFutures() + } } var syncFromTime time.Time diff --git a/pkg/service/backtest.go b/pkg/service/backtest.go index 1af6e370a5..929c505041 100644 --- a/pkg/service/backtest.go +++ b/pkg/service/backtest.go @@ -19,8 +19,7 @@ import ( ) type BacktestService struct { - DB *sqlx.DB - Futures bool + DB *sqlx.DB } func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error { @@ -31,18 +30,7 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type symbol = isolatedSymbol } - s.Futures = isFutures - if s.Futures { - futuresExchange, ok := exchange.(types.FuturesExchange) - if !ok { - return fmt.Errorf("exchange %s does not support futures", exchange.Name()) - } - - if isIsolated { - futuresExchange.UseIsolatedFutures(symbol) - } else { - futuresExchange.UseFutures() - } + if isFutures { log.Infof("synchronizing %s futures klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) } else { log.Infof("synchronizing %s klines with interval %s: %s <=> %s", exchange.Name(), interval, startTime, endTime) @@ -57,7 +45,7 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type tasks := []SyncTask{ { Type: types.KLine{}, - Select: s.SelectLastKLines(exchange.Name(), symbol, interval, startTime, endTime, 100), + Select: s.SelectLastKLines(exchange, symbol, interval, startTime, endTime, 100), Time: func(obj interface{}) time.Time { return obj.(types.KLine).StartTime.Time() }, @@ -86,11 +74,11 @@ func (s *BacktestService) SyncKLineByInterval(ctx context.Context, exchange type BatchInsertBuffer: 1000, BatchInsert: func(obj interface{}) error { kLines := obj.([]types.KLine) - return s.BatchInsert(kLines) + return s.BatchInsert(kLines, exchange) }, Insert: func(obj interface{}) error { kline := obj.(types.KLine) - return s.Insert(kline) + return s.Insert(kline, exchange) }, LogInsert: log.GetLevel() == log.DebugLevel, }, @@ -147,10 +135,10 @@ func (s *BacktestService) SyncFresh(ctx context.Context, exchange types.Exchange } // QueryKLine queries the klines from the database -func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) { +func (s *BacktestService) QueryKLine(ex types.Exchange, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) { log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval) - tableName := s.targetKlineTable(ex) + tableName := targetKlineTable(ex) // make the SQL syntax IDE friendly, so that it can analyze it. sql := fmt.Sprintf("SELECT * FROM `%s` WHERE `symbol` = :symbol AND `interval` = :interval ORDER BY end_time "+orderBy+" LIMIT "+strconv.Itoa(limit), tableName) @@ -178,8 +166,8 @@ func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, inter } // QueryKLinesForward is used for querying klines to back-testing -func (s *BacktestService) QueryKLinesForward(exchange types.ExchangeName, symbol string, interval types.Interval, startTime time.Time, limit int) ([]types.KLine, error) { - tableName := s.targetKlineTable(exchange) +func (s *BacktestService) QueryKLinesForward(exchange types.Exchange, symbol string, interval types.Interval, startTime time.Time, limit int) ([]types.KLine, error) { + tableName := targetKlineTable(exchange) sql := "SELECT * FROM `binance_klines` WHERE `end_time` >= :start_time AND `symbol` = :symbol AND `interval` = :interval and exchange = :exchange ORDER BY end_time ASC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -188,7 +176,7 @@ func (s *BacktestService) QueryKLinesForward(exchange types.ExchangeName, symbol "limit": limit, "symbol": symbol, "interval": interval, - "exchange": exchange.String(), + "exchange": exchange.Name().String(), }) if err != nil { return nil, err @@ -197,8 +185,8 @@ func (s *BacktestService) QueryKLinesForward(exchange types.ExchangeName, symbol return s.scanRows(rows) } -func (s *BacktestService) QueryKLinesBackward(exchange types.ExchangeName, symbol string, interval types.Interval, endTime time.Time, limit int) ([]types.KLine, error) { - tableName := s.targetKlineTable(exchange) +func (s *BacktestService) QueryKLinesBackward(exchange types.Exchange, symbol string, interval types.Interval, endTime time.Time, limit int) ([]types.KLine, error) { + tableName := targetKlineTable(exchange) sql := "SELECT * FROM `binance_klines` WHERE `end_time` <= :end_time and exchange = :exchange AND `symbol` = :symbol AND `interval` = :interval ORDER BY end_time DESC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -209,7 +197,7 @@ func (s *BacktestService) QueryKLinesBackward(exchange types.ExchangeName, symbo "end_time": endTime, "symbol": symbol, "interval": interval, - "exchange": exchange.String(), + "exchange": exchange.Name().String(), }) if err != nil { return nil, err @@ -223,7 +211,7 @@ func (s *BacktestService) QueryKLinesCh(since, until time.Time, exchange types.E return returnError(errors.Errorf("symbols is empty when querying kline, plesae check your strategy setting. ")) } - tableName := s.targetKlineTable(exchange.Name()) + tableName := targetKlineTable(exchange) var query string // need to sort by start_time desc in order to let matching engine process 1m first @@ -316,9 +304,11 @@ func (s *BacktestService) scanRows(rows *sqlx.Rows) (klines []types.KLine, err e return klines, rows.Err() } -func (s *BacktestService) targetKlineTable(exchangeName types.ExchangeName) string { - tableName := strings.ToLower(exchangeName.String()) - if s.Futures { +func targetKlineTable(exchange types.Exchange) string { + _, isFutures, _, _ := exchange2.GetSessionAttributes(exchange) + + tableName := strings.ToLower(exchange.Name().String()) + if isFutures { return tableName + "_futures_klines" } else { return tableName + "_klines" @@ -327,12 +317,12 @@ func (s *BacktestService) targetKlineTable(exchangeName types.ExchangeName) stri var errExchangeFieldIsUnset = errors.New("kline.Exchange field should not be empty") -func (s *BacktestService) Insert(kline types.KLine) error { +func (s *BacktestService) Insert(kline types.KLine, ex types.Exchange) error { if len(kline.Exchange) == 0 { return errExchangeFieldIsUnset } - tableName := s.targetKlineTable(kline.Exchange) + tableName := targetKlineTable(ex) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ "VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume)", tableName) @@ -342,12 +332,12 @@ func (s *BacktestService) Insert(kline types.KLine) error { } // BatchInsert Note: all kline should be same exchange, or it will cause issue. -func (s *BacktestService) BatchInsert(kline []types.KLine) error { +func (s *BacktestService) BatchInsert(kline []types.KLine, ex types.Exchange) error { if len(kline) == 0 { return nil } - tableName := s.targetKlineTable(kline[0].Exchange) + tableName := targetKlineTable(ex) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ " VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume); ", tableName) @@ -442,7 +432,7 @@ func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, sy // FindMissingTimeRanges returns the missing time ranges, the start/end time represents the existing data time points. // So when sending kline query to the exchange API, we need to add one second to the start time and minus one second to the end time. func (s *BacktestService) FindMissingTimeRanges(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) ([]TimeRange, error) { - query := s.SelectKLineTimePoints(ex.Name(), symbol, interval, since, until) + query := s.SelectKLineTimePoints(ex, symbol, interval, since, until) sql, args, err := query.ToSql() if err != nil { return nil, err @@ -485,7 +475,7 @@ func (s *BacktestService) FindMissingTimeRanges(ctx context.Context, ex types.Ex } func (s *BacktestService) QueryExistingDataRange(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, tArgs ...time.Time) (start, end *types.Time, err error) { - sel := s.SelectKLineTimeRange(ex.Name(), symbol, interval, tArgs...) + sel := s.SelectKLineTimeRange(ex, symbol, interval, tArgs...) sql, args, err := sel.ToSql() if err != nil { return nil, nil, err @@ -510,7 +500,7 @@ func (s *BacktestService) QueryExistingDataRange(ctx context.Context, ex types.E return &t1, &t2, nil } -func (s *BacktestService) SelectKLineTimePoints(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { +func (s *BacktestService) SelectKLineTimePoints(ex types.Exchange, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { conditions := sq.And{ sq.Eq{"symbol": symbol}, sq.Eq{"`interval`": interval.String()}, @@ -522,7 +512,7 @@ func (s *BacktestService) SelectKLineTimePoints(ex types.ExchangeName, symbol st conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := s.targetKlineTable(ex) + tableName := targetKlineTable(ex) return sq.Select("start_time"). From(tableName). @@ -531,7 +521,7 @@ func (s *BacktestService) SelectKLineTimePoints(ex types.ExchangeName, symbol st } // SelectKLineTimeRange returns the existing klines time range (since < kline.start_time < until) -func (s *BacktestService) SelectKLineTimeRange(ex types.ExchangeName, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { +func (s *BacktestService) SelectKLineTimeRange(ex types.Exchange, symbol string, interval types.Interval, args ...time.Time) sq.SelectBuilder { conditions := sq.And{ sq.Eq{"symbol": symbol}, sq.Eq{"`interval`": interval.String()}, @@ -546,7 +536,7 @@ func (s *BacktestService) SelectKLineTimeRange(ex types.ExchangeName, symbol str conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := s.targetKlineTable(ex) + tableName := targetKlineTable(ex) return sq.Select("MIN(start_time) AS t1, MAX(start_time) AS t2"). From(tableName). @@ -554,8 +544,8 @@ func (s *BacktestService) SelectKLineTimeRange(ex types.ExchangeName, symbol str } // TODO: add is_futures column since the klines data is different -func (s *BacktestService) SelectLastKLines(ex types.ExchangeName, symbol string, interval types.Interval, startTime, endTime time.Time, limit uint64) sq.SelectBuilder { - tableName := s.targetKlineTable(ex) +func (s *BacktestService) SelectLastKLines(ex types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time, limit uint64) sq.SelectBuilder { + tableName := targetKlineTable(ex) return sq.Select("*"). From(tableName). Where(sq.And{ From 0ac720c4cb314ce5f266b0f308b84328fde23bd6 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Fri, 22 Dec 2023 11:55:11 +0800 Subject: [PATCH 366/422] improve/backtest: backtest with futures klines --- pkg/backtest/exchange.go | 11 +++++++++-- pkg/cmd/backtest.go | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index c459a893a6..183b41c39a 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -30,6 +30,7 @@ package backtest import ( "context" "fmt" + exchange2 "github.com/c9s/bbgo/pkg/exchange" "strconv" "sync" "time" @@ -381,7 +382,13 @@ func (e *Exchange) SubscribeMarketData( intervals = append(intervals, interval) } - log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) + _, isFutures, _, _ := exchange2.GetSessionAttributes(e.publicExchange) + if isFutures { + log.Infof("querying futures klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) + } else { + log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) + } + if len(symbols) == 0 { log.Warnf("empty symbols, will not query kline data from the database") @@ -390,7 +397,7 @@ func (e *Exchange) SubscribeMarketData( return c, nil } - klineC, errC := e.srv.QueryKLinesCh(startTime, endTime, e, symbols, intervals) + klineC, errC := e.srv.QueryKLinesCh(startTime, endTime, e.publicExchange, symbols, intervals) go func() { if err := <-errC; err != nil { log.WithError(err).Error("backtest data feed error") diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index 6cd97e1093..f83dfae53d 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -279,6 +279,7 @@ var BacktestCmd = &cobra.Command{ exchangeFromConfig := userConfig.Sessions[name.String()] if exchangeFromConfig != nil { session.UseHeikinAshi = exchangeFromConfig.UseHeikinAshi + session.Futures = exchangeFromConfig.Futures } } From d2f946e3491a6b9ea6c315c55dc67f0a6f506ef4 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Fri, 22 Dec 2023 12:00:14 +0800 Subject: [PATCH 367/422] improve/migration: indices for sqlite --- .../sqlite3/20231221121432_add_futures_klines.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go b/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go index 8554e49d65..18225a93a5 100644 --- a/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go +++ b/pkg/migrations/sqlite3/20231221121432_add_futures_klines.go @@ -34,12 +34,22 @@ func upAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err err return err } + _, err = tx.ExecContext(ctx, "CREATE INDEX `bybit_futures_klines_end_time_symbol_interval` ON `bybit_futures_klines` (`end_time`, `symbol`, `interval`);\nCREATE INDEX `okex_futures_klines_end_time_symbol_interval` ON `okex_futures_klines` (`end_time`, `symbol`, `interval`);\nCREATE INDEX `binance_futures_klines_end_time_symbol_interval` ON `binance_futures_klines` (`end_time`, `symbol`, `interval`);\nCREATE INDEX `max_futures_klines_end_time_symbol_interval` ON `max_futures_klines` (`end_time`, `symbol`, `interval`);") + if err != nil { + return err + } + return err } func downAddFuturesKlines(ctx context.Context, tx rockhopper.SQLExecutor) (err error) { // This code is executed when the migration is rolled back. + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS `bybit_futures_klines_end_time_symbol_interval`;\nDROP INDEX IF EXISTS `okex_futures_klines_end_time_symbol_interval`;\nDROP INDEX IF EXISTS `binance_futures_klines_end_time_symbol_interval`;\nDROP INDEX IF EXISTS `max_futures_klines_end_time_symbol_interval`;\n") + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, "DROP TABLE IF EXISTS `bybit_futures_klines`;") if err != nil { return err From b30b02385884a438b8028804b48c8350e8289af4 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Fri, 22 Dec 2023 15:27:31 +0800 Subject: [PATCH 368/422] FEATURE: check every cuerrent state and next state is valid --- pkg/strategy/dca2/recover.go | 8 ++-- pkg/strategy/dca2/state.go | 71 +++++++++++++++++++++++++++-------- pkg/strategy/dca2/strategy.go | 20 +++------- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index 40c66ee9fd..7795d9908f 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -240,11 +240,9 @@ func getCurrentRoundOrders(short bool, openOrders, closedOrders []types.Order, g lastSide := takeProfitSide for _, order := range allOrders { // group id filter is used for debug when local running - /* - if order.GroupID != groupID { - continue - } - */ + if order.GroupID != groupID { + continue + } if order.Side == takeProfitSide && lastSide == openPositionSide { break diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 7a787fcda7..2cc5a332d9 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -34,6 +34,14 @@ func (s *Strategy) initializeNextStateC() bool { return isInitialize } +func (s *Strategy) emitNextState(nextState State) { + select { + case s.nextStateC <- nextState: + default: + s.logger.Info("[DCA] nextStateC is full or not initialized") + } +} + // runState // WaitToOpenPosition -> after startTimeOfNextRound, place dca orders -> // PositionOpening @@ -43,11 +51,16 @@ func (s *Strategy) initializeNextStateC() bool { // TakeProfitReady -> the takeProfit order filled -> func (s *Strategy) runState(ctx context.Context) { s.logger.Info("[DCA] runState") + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { select { case <-ctx.Done(): s.logger.Info("[DCA] runState DONE") return + case <-ticker.C: + s.triggerNextState() case nextState := <-s.nextStateC: s.logger.Infof("[DCA] currenct state: %d, next state: %d", s.state, nextState) switch s.state { @@ -70,32 +83,51 @@ func (s *Strategy) runState(ctx context.Context) { } } +func (s *Strategy) triggerNextState() { + switch s.state { + case WaitToOpenPosition: + s.nextStateC <- PositionOpening + case PositionOpening: + s.nextStateC <- OpenPositionReady + case OpenPositionReady: + // trigger from order filled event + case OpenPositionOrderFilled: + // trigger from kline event + case OpenPositionOrdersCancelling: + s.nextStateC <- OpenPositionOrdersCancelled + case OpenPositionOrdersCancelled: + s.nextStateC <- TakeProfitReady + case TakeProfitReady: + // trigger from order filled event + } +} + func (s *Strategy) runWaitToOpenPositionState(_ context.Context, next State) { - if next != None { + if next != PositionOpening { return } - s.logger.Info("[WaitToOpenPosition] check startTimeOfNextRound") + s.logger.Info("[State] WaitToOpenPosition - check startTimeOfNextRound") if time.Now().Before(s.startTimeOfNextRound) { return } s.state = PositionOpening - s.logger.Info("[WaitToOpenPosition] move to PositionOpening") + s.logger.Info("[State] WaitToOpenPosition -> PositionOpening") } func (s *Strategy) runPositionOpening(ctx context.Context, next State) { - if next != None { + if next != OpenPositionReady { return } - s.logger.Info("[PositionOpening] start placing open-position orders") + s.logger.Info("[State] PositionOpening - start placing open-position orders") if err := s.placeOpenPositionOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to place dca orders, please check it.") return } s.state = OpenPositionReady - s.logger.Info("[PositionOpening] move to OpenPositionReady") + s.logger.Info("[State] PositionOpening -> OpenPositionReady") } func (s *Strategy) runOpenPositionReady(_ context.Context, next State) { @@ -103,7 +135,7 @@ func (s *Strategy) runOpenPositionReady(_ context.Context, next State) { return } s.state = OpenPositionOrderFilled - s.logger.Info("[OpenPositionReady] move to OpenPositionOrderFilled") + s.logger.Info("[State] OpenPositionReady -> OpenPositionOrderFilled") } func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { @@ -111,34 +143,41 @@ func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { return } s.state = OpenPositionOrdersCancelling - s.logger.Info("[OpenPositionOrderFilled] move to OpenPositionOrdersCancelling") + s.logger.Info("[State] OpenPositionOrderFilled -> OpenPositionOrdersCancelling") + + // after open position cancelling, immediately trigger open position cancelled to cancel the other orders + s.nextStateC <- OpenPositionOrdersCancelled } func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { - if next != None { + if next != OpenPositionOrdersCancelled { return } - s.logger.Info("[OpenPositionOrdersCancelling] start cancelling open-position orders") + s.logger.Info("[State] OpenPositionOrdersCancelling - start cancelling open-position orders") if err := s.cancelOpenPositionOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to cancel maker orders") return } s.state = OpenPositionOrdersCancelled - s.logger.Info("[OpenPositionOrdersCancelling] move to OpenPositionOrdersCancelled") + s.logger.Info("[State] OpenPositionOrdersCancelling -> OpenPositionOrdersCancelled") + + // after open position cancelled, immediately trigger take profit ready to open take-profit order + s.nextStateC <- TakeProfitReady } func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next State) { - if next != None { + if next != TakeProfitReady { return } - s.logger.Info("[OpenPositionOrdersCancelled] start placing take-profit orders") + + s.logger.Info("[State] OpenPositionOrdersCancelled - start placing take-profit orders") if err := s.placeTakeProfitOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to open take profit orders") return } s.state = TakeProfitReady - s.logger.Info("[OpenPositionOrdersCancelled] move to TakeProfitReady") + s.logger.Info("[State] OpenPositionOrdersCancelled -> TakeProfitReady") } func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { @@ -146,7 +185,7 @@ func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { return } - s.logger.Info("[TakeProfitReady] start reseting position and calculate budget for next round") + s.logger.Info("[State] TakeProfitReady - start reseting position and calculate budget for next round") if s.Short { s.Budget = s.Budget.Add(s.Position.Base) } else { @@ -159,5 +198,5 @@ func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { // set the start time of the next round s.startTimeOfNextRound = time.Now().Add(s.CoolDownInterval.Duration()) s.state = WaitToOpenPosition - s.logger.Info("[TakeProfitReady] move to WaitToOpenPosition") + s.logger.Info("[State] TakeProfitReady -> WaitToOpenPosition") } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 2ed8e8c1cd..92dfa8db73 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -109,8 +109,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.OrderGroupID = util.FNV32(instanceID) % math.MaxInt32 } - s.updateTakeProfitPrice() - // order executor s.OrderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { s.logger.Infof("[DCA] POSITION UPDATE: %s", s.Position.String()) @@ -131,26 +129,16 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. switch o.Side { case openPositionSide: - s.nextStateC <- OpenPositionOrderFilled + s.emitNextState(OpenPositionOrderFilled) case takeProfitSide: - s.nextStateC <- WaitToOpenPosition + s.emitNextState(WaitToOpenPosition) default: s.logger.Infof("[DCA] unsupported side (%s) of order: %s", o.Side, o) } }) session.MarketDataStream.OnKLine(func(kline types.KLine) { - s.logger.Infof("[DCA] %s", s.Strategy.Position.String()) - s.logger.Infof("[DCA] tkae-profit price: %s", s.takeProfitPrice) // check price here - // because we subscribe 1m kline, it will close every 1 min - // we use it as ticker to maker WaitToOpenPosition -> OpenPositionReady - select { - case s.nextStateC <- None: - default: - s.logger.Info("[DCA] nextStateC is full or not initialized") - } - if s.state != OpenPositionOrderFilled { return } @@ -161,7 +149,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. return } - s.nextStateC <- OpenPositionOrdersCancelling + s.emitNextState(OpenPositionOrdersCancelling) }) session.UserDataStream.OnAuth(func() { @@ -179,6 +167,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Infof("[DCA] recovered budget %s", s.Budget) s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) + s.updateTakeProfitPrice() + // store persistence bbgo.Sync(ctx, s) From 88780054170cbf54e3af17b4e99c194bcf6af50b Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Dec 2023 10:53:18 +0800 Subject: [PATCH 369/422] add DisableMarketDataStore option --- pkg/bbgo/config.go | 5 ++++- pkg/bbgo/session.go | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/bbgo/config.go b/pkg/bbgo/config.go index f892f2138f..1121717b90 100644 --- a/pkg/bbgo/config.go +++ b/pkg/bbgo/config.go @@ -335,7 +335,10 @@ type EnvironmentConfig struct { DisableStartupBalanceQuery bool `json:"disableStartupBalanceQuery"` DisableSessionTradeBuffer bool `json:"disableSessionTradeBuffer"` - MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` + + DisableMarketDataStore bool `json:"disableMarketDataStore"` + + MaxSessionTradeBufferSize int `json:"maxSessionTradeBufferSize"` } type Config struct { diff --git a/pkg/bbgo/session.go b/pkg/bbgo/session.go index 4cfa57c404..b61c6f5ed2 100644 --- a/pkg/bbgo/session.go +++ b/pkg/bbgo/session.go @@ -405,6 +405,7 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ return fmt.Errorf("market %s is not defined", symbol) } + disableMarketDataStore := environ.environmentConfig != nil && environ.environmentConfig.DisableMarketDataStore disableSessionTradeBuffer := environ.environmentConfig != nil && environ.environmentConfig.DisableSessionTradeBuffer maxSessionTradeBufferSize := 0 if environ.environmentConfig != nil && environ.environmentConfig.MaxSessionTradeBufferSize > 0 { @@ -441,13 +442,13 @@ func (session *ExchangeSession) initSymbol(ctx context.Context, environ *Environ orderStore.BindStream(session.UserDataStream) session.orderStores[symbol] = orderStore - if _, ok := session.marketDataStores[symbol]; !ok { - marketDataStore := NewMarketDataStore(symbol) - marketDataStore.BindStream(session.MarketDataStream) - session.marketDataStores[symbol] = marketDataStore + marketDataStore := NewMarketDataStore(symbol) + if !disableMarketDataStore { + if _, ok := session.marketDataStores[symbol]; !ok { + marketDataStore.BindStream(session.MarketDataStream) + } } - - marketDataStore := session.marketDataStores[symbol] + session.marketDataStores[symbol] = marketDataStore if _, ok := session.standardIndicatorSets[symbol]; !ok { standardIndicatorSet := NewStandardIndicatorSet(symbol, session.MarketDataStream, marketDataStore) From 4d17d7e0499c19d041333fcc84ccdeec748df2a9 Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 26 Dec 2023 10:56:08 +0800 Subject: [PATCH 370/422] grid2: subscribe 1m kline only when one of the trigger price is set --- pkg/strategy/grid2/strategy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index f3c79803cf..209b82feb7 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -264,7 +264,9 @@ func (s *Strategy) Initialize() error { } func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) + if !s.TriggerPrice.IsZero() || !s.StopLossPrice.IsZero() || !s.TakeProfitPrice.IsZero() { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: types.Interval1m}) + } if s.AutoRange != nil { interval := s.AutoRange.Interval() From 5592d93c13a949bf5a4296af01f56b6aca2fe8a2 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sat, 23 Dec 2023 16:26:28 +0800 Subject: [PATCH 371/422] add cron schedule to xnav --- config/xnav.yaml | 1 + pkg/strategy/xnav/strategy.go | 65 ++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/config/xnav.yaml b/config/xnav.yaml index 2e59eef842..7b1078d3a8 100644 --- a/config/xnav.yaml +++ b/config/xnav.yaml @@ -30,6 +30,7 @@ crossExchangeStrategies: - xnav: interval: 1h + # schedule: "0 * * * *" # every hour reportOnStart: true ignoreDusts: true diff --git a/pkg/strategy/xnav/strategy.go b/pkg/strategy/xnav/strategy.go index 91e3bc22fe..a517dbfca3 100644 --- a/pkg/strategy/xnav/strategy.go +++ b/pkg/strategy/xnav/strategy.go @@ -2,19 +2,19 @@ package xnav import ( "context" + "fmt" "sync" "time" + "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/util" "github.com/c9s/bbgo/pkg/util/templateutil" - "github.com/pkg/errors" + "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/slack-go/slack" - - "github.com/c9s/bbgo/pkg/bbgo" - "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util" ) const ID = "xnav" @@ -59,16 +59,30 @@ type Strategy struct { *bbgo.Environment Interval types.Interval `json:"interval"` + Schedule string `json:"schedule"` ReportOnStart bool `json:"reportOnStart"` IgnoreDusts bool `json:"ignoreDusts"` State *State `persistence:"state"` + + cron *cron.Cron } func (s *Strategy) ID() string { return ID } +func (s *Strategy) Initialize() error { + return nil +} + +func (s *Strategy) Validate() error { + if s.Interval == "" && s.Schedule == "" { + return fmt.Errorf("interval or schedule is required") + } + return nil +} + var Ten = fixedpoint.NewFromInt(10) func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) {} @@ -138,10 +152,6 @@ func (s *Strategy) recordNetAssetValue(ctx context.Context, sessions map[string] } func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession) error { - if s.Interval == "" { - return errors.New("interval can not be empty") - } - if s.State == nil { s.State = &State{} s.State.Reset() @@ -161,25 +171,32 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se log.Warnf("xnav does not support backtesting") } - // TODO: if interval is supported, we can use kline as the ticker - if _, ok := types.SupportedIntervals[s.Interval]; ok { - - } + if s.Interval != "" { + go func() { + ticker := time.NewTicker(util.MillisecondsJitter(s.Interval.Duration(), 1000)) + defer ticker.Stop() - go func() { - ticker := time.NewTicker(util.MillisecondsJitter(s.Interval.Duration(), 1000)) - defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return - for { - select { - case <-ctx.Done(): - return - - case <-ticker.C: - s.recordNetAssetValue(ctx, sessions) + case <-ticker.C: + s.recordNetAssetValue(ctx, sessions) + } } + }() + + } else if s.Schedule != "" { + s.cron = cron.New() + _, err := s.cron.AddFunc(s.Schedule, func() { + s.recordNetAssetValue(ctx, sessions) + }) + if err != nil { + return err } - }() + s.cron.Start() + } return nil } From 687df81784a5f0fbe37f98b333984fd8b3c36518 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Fri, 22 Dec 2023 16:40:30 +0800 Subject: [PATCH 372/422] add autobuy strategy --- config/autobuy.yaml | 22 ++++ pkg/cmd/strategy/builtin.go | 1 + pkg/strategy/autobuy/strategy.go | 169 +++++++++++++++++++++++++++++++ pkg/strategy/autobuy/types.go | 10 ++ 4 files changed, 202 insertions(+) create mode 100644 config/autobuy.yaml create mode 100644 pkg/strategy/autobuy/strategy.go create mode 100644 pkg/strategy/autobuy/types.go diff --git a/config/autobuy.yaml b/config/autobuy.yaml new file mode 100644 index 0000000000..7b174ea331 --- /dev/null +++ b/config/autobuy.yaml @@ -0,0 +1,22 @@ +--- +exchangeStrategies: + - on: max + # automaticaly buy coins when the balance is lower than the threshold + autobuy: + symbol: MAXTWD + schedule: "@every 1s" + threshold: 200 + # price type: buy, sell, last or mid + priceType: buy + + # order quantity or amount + # quantity: 100 + amount: 800 + + # skip if the price is higher than the upper band + bollinger: + interval: 1m + window: 21 + bandWidth: 2.0 + + dryRun: true diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 61c03788d3..5fac8306f3 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -5,6 +5,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/atrpin" _ "github.com/c9s/bbgo/pkg/strategy/audacitymaker" _ "github.com/c9s/bbgo/pkg/strategy/autoborrow" + _ "github.com/c9s/bbgo/pkg/strategy/autobuy" _ "github.com/c9s/bbgo/pkg/strategy/bollgrid" _ "github.com/c9s/bbgo/pkg/strategy/bollmaker" _ "github.com/c9s/bbgo/pkg/strategy/convert" diff --git a/pkg/strategy/autobuy/strategy.go b/pkg/strategy/autobuy/strategy.go new file mode 100644 index 0000000000..dbb44a88d6 --- /dev/null +++ b/pkg/strategy/autobuy/strategy.go @@ -0,0 +1,169 @@ +package autobuy + +import ( + "context" + "fmt" + "sync" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" + indicatorv2 "github.com/c9s/bbgo/pkg/indicator/v2" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/types" + "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" +) + +const ID = "autobuy" + +var log = logrus.WithField("strategy", ID) + +var two = fixedpoint.NewFromInt(2) + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Market types.Market + + Symbol string `json:"symbol"` + Schedule string `json:"schedule"` + Threshold fixedpoint.Value `json:"threshold"` + PriceType PriceType `json:"priceType"` + Bollinger *types.BollingerSetting `json:"bollinger"` + DryRun bool `json:"dryRun"` + + bbgo.QuantityOrAmount + + boll *indicatorv2.BOLLStream + cron *cron.Cron +} + +func (s *Strategy) Initialize() error { + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } + return nil +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + +func (s *Strategy) Validate() error { + if err := s.QuantityOrAmount.Validate(); err != nil { + return err + } + return nil +} + +func (s *Strategy) Defaults() error { + if s.PriceType == "" { + s.PriceType = PriceTypeBuy + } + return nil +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Bollinger.Interval}) +} + +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + + s.boll = session.Indicators(s.Symbol).BOLL(s.Bollinger.IntervalWindow, s.Bollinger.BandWidth) + + s.OrderExecutor.ActiveMakerOrders().OnFilled(func(order types.Order) { + s.autobuy(ctx) + }) + + // the shutdown handler, you can cancel all orders + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + s.cancelOrders(ctx) + }) + + s.cron = cron.New() + s.cron.AddFunc(s.Schedule, func() { + s.autobuy(ctx) + }) + s.cron.Start() + + return nil +} + +func (s *Strategy) cancelOrders(ctx context.Context) { + if err := s.OrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Errorf("failed to cancel orders") + } +} + +func (s *Strategy) autobuy(ctx context.Context) { + s.cancelOrders(ctx) + + balance, ok := s.Session.GetAccount().Balance(s.Market.BaseCurrency) + if !ok { + log.Errorf("%s balance not found", s.Market.BaseCurrency) + return + } + log.Infof("balance: %s", balance.String()) + + ticker, err := s.Session.Exchange.QueryTicker(ctx, s.Symbol) + if err != nil { + log.WithError(err).Errorf("failed to query ticker") + return + } + + var price fixedpoint.Value + switch s.PriceType { + case PriceTypeLast: + price = ticker.Last + case PriceTypeBuy: + price = ticker.Buy + case PriceTypeSell: + price = ticker.Sell + case PriceTypeMid: + price = ticker.Buy.Add(ticker.Sell).Div(two) + default: + price = ticker.Buy + } + + if price.Float64() > s.boll.UpBand.Last(0) { + log.Infof("price %s is higher than upper band %f, skip", price.String(), s.boll.UpBand.Last(0)) + return + } + + if balance.Available.Compare(s.Threshold) > 0 { + log.Infof("balance %s is higher than threshold %s", balance.Available.String(), s.Threshold.String()) + return + } + log.Infof("balance %s is lower than threshold %s", balance.Available.String(), s.Threshold.String()) + + quantity := s.CalculateQuantity(price) + submitOrder := types.SubmitOrder{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeLimitMaker, + Quantity: quantity, + Price: price, + } + + if s.DryRun { + log.Infof("dry run, skip") + return + } + + log.Infof("submitting order: %s", submitOrder.String()) + _, err = s.OrderExecutor.SubmitOrders(ctx, submitOrder) + if err != nil { + log.WithError(err).Errorf("submit order error") + } +} diff --git a/pkg/strategy/autobuy/types.go b/pkg/strategy/autobuy/types.go new file mode 100644 index 0000000000..0e3347ae05 --- /dev/null +++ b/pkg/strategy/autobuy/types.go @@ -0,0 +1,10 @@ +package autobuy + +type PriceType string + +const ( + PriceTypeLast = PriceType("last") + PriceTypeBuy = PriceType("buy") + PriceTypeSell = PriceType("sell") + PriceTypeMid = PriceType("mid") +) From 59b1bb68cb1ee9262d3a8c74a28163ab80d886ea Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 27 Dec 2023 11:41:29 +0800 Subject: [PATCH 373/422] use stateTransition --- pkg/strategy/dca2/recover.go | 44 +++++---- pkg/strategy/dca2/recover_test.go | 157 +++++++++++++++--------------- pkg/strategy/dca2/state.go | 67 ++++++------- pkg/types/sort.go | 8 ++ 4 files changed, 139 insertions(+), 137 deletions(-) diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index 7795d9908f..5be876a6df 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -3,7 +3,6 @@ package dca2 import ( "context" "fmt" - "sort" "strconv" "time" @@ -13,15 +12,19 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -type queryAPI interface { - QueryOpenOrders(ctx context.Context, symbol string) ([]types.Order, error) +type descendingClosedOrderQueryService interface { QueryClosedOrdersDesc(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) ([]types.Order, error) - QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([]types.Trade, error) +} + +type RecoverApiQueryService interface { + types.ExchangeOrderQueryService + types.ExchangeTradeService + descendingClosedOrderQueryService } func (s *Strategy) recover(ctx context.Context) error { s.logger.Info("[DCA] recover") - queryService, ok := s.Session.Exchange.(queryAPI) + queryService, ok := s.Session.Exchange.(RecoverApiQueryService) if !ok { return fmt.Errorf("[DCA] exchange %s doesn't support queryAPI interface", s.Session.ExchangeName) } @@ -70,9 +73,19 @@ func (s *Strategy) recover(ctx context.Context) error { // recover state func recoverState(ctx context.Context, symbol string, short bool, maxOrderNum int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { + if len(currentRound.OpenPositionOrders) == 0 { + // new strategy + return WaitToOpenPosition, nil + } + numOpenOrders := len(openOrders) // dca stop at take profit order stage if currentRound.TakeProfitOrder.OrderID != 0 { + if numOpenOrders == 0 { + // current round's take-profit order filled, wait to open next round + return WaitToOpenPosition, nil + } + // check the open orders is take profit order or not if numOpenOrders == 1 { if openOrders[0].OrderID == currentRound.TakeProfitOrder.OrderID { @@ -84,24 +97,17 @@ func recoverState(ctx context.Context, symbol string, short bool, maxOrderNum in } } - if numOpenOrders == 0 { - // current round's take-profit order filled, wait to open next round - return WaitToOpenPosition, nil - } - return None, fmt.Errorf("stop at taking profit stage, but the number of open orders is > 1") } - if len(currentRound.OpenPositionOrders) == 0 { - // new strategy - return WaitToOpenPosition, nil - } - numOpenPositionOrders := len(currentRound.OpenPositionOrders) if numOpenPositionOrders > maxOrderNum { return None, fmt.Errorf("the number of open-position orders is > max order number") } else if numOpenPositionOrders < maxOrderNum { - // failed to place some orders at open position stage + // The number of open-position orders should be the same as maxOrderNum + // If not, it may be the following possible cause + // 1. This strategy at position opening, so it may not place all orders we want successfully + // 2. There are some errors when placing open-position orders. e.g. cannot lock fund..... return None, fmt.Errorf("the number of open-position orders is < max order number") } @@ -146,7 +152,7 @@ func recoverState(ctx context.Context, symbol string, short bool, maxOrderNum in return None, fmt.Errorf("unexpected order status combination") } -func recoverPosition(ctx context.Context, position *types.Position, queryService queryAPI, currentRound Round) error { +func recoverPosition(ctx context.Context, position *types.Position, queryService RecoverApiQueryService, currentRound Round) error { if position == nil { return nil } @@ -232,9 +238,7 @@ func getCurrentRoundOrders(short bool, openOrders, closedOrders []types.Order, g allOrders = append(allOrders, openOrders...) allOrders = append(allOrders, closedOrders...) - sort.Slice(allOrders, func(i, j int) bool { - return allOrders[i].CreationTime.After(allOrders[j].CreationTime.Time()) - }) + types.SortOrdersDescending(allOrders) var currentRound Round lastSide := takeProfitSide diff --git a/pkg/strategy/dca2/recover_test.go b/pkg/strategy/dca2/recover_test.go index 98550083ae..0b7a501d23 100644 --- a/pkg/strategy/dca2/recover_test.go +++ b/pkg/strategy/dca2/recover_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func generateOrder(side types.SideType, status types.OrderStatus, createdAt time.Time) types.Order { +func generateTestOrder(side types.SideType, status types.OrderStatus, createdAt time.Time) types.Order { return types.Order{ OrderID: rand.Uint64(), SubmitOrder: types.SubmitOrder{ @@ -25,60 +25,58 @@ func generateOrder(side types.SideType, status types.OrderStatus, createdAt time } func Test_GetCurrenctAndLastRoundOrders(t *testing.T) { - assert := assert.New(t) - t.Run("case 1", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + generateTestOrder(types.SideTypeSell, types.OrderStatusNew, now), } closedOrders := []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), } currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) - assert.NoError(err) - assert.NotEqual(0, currentRound.TakeProfitOrder.OrderID) - assert.Equal(5, len(currentRound.OpenPositionOrders)) + assert.NoError(t, err) + assert.NotEqual(t, 0, currentRound.TakeProfitOrder.OrderID) + assert.Equal(t, 5, len(currentRound.OpenPositionOrders)) }) t.Run("case 2", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + generateTestOrder(types.SideTypeSell, types.OrderStatusNew, now), } closedOrders := []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), - generateOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-6*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-7*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-8*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-9*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-10*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-11*time.Second)), - generateOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-12*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-13*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-14*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-15*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-16*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-17*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-6*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-7*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-8*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-9*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-10*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-11*time.Second)), + generateTestOrder(types.SideTypeSell, types.OrderStatusFilled, now.Add(-12*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-13*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-14*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-15*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-16*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-17*time.Second)), } currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) - assert.NoError(err) - assert.NotEqual(0, currentRound.TakeProfitOrder.OrderID) - assert.Equal(5, len(currentRound.OpenPositionOrders)) + assert.NoError(t, err) + assert.NotEqual(t, 0, currentRound.TakeProfitOrder.OrderID) + assert.Equal(t, 5, len(currentRound.OpenPositionOrders)) }) } @@ -96,7 +94,6 @@ func (m *MockQueryOrders) QueryClosedOrdersDesc(ctx context.Context, symbol stri } func Test_RecoverState(t *testing.T) { - assert := assert.New(t) symbol := "BTCUSDT" t.Run("new strategy", func(t *testing.T) { @@ -105,18 +102,18 @@ func Test_RecoverState(t *testing.T) { activeOrderBook := bbgo.NewActiveOrderBook(symbol) orderStore := core.NewOrderStore(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(WaitToOpenPosition, state) + assert.NoError(t, err) + assert.Equal(t, WaitToOpenPosition, state) }) t.Run("at open position stage and no filled order", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusPartiallyFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusPartiallyFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), } currentRound := Round{ OpenPositionOrders: openOrders, @@ -124,21 +121,21 @@ func Test_RecoverState(t *testing.T) { orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(OpenPositionReady, state) + assert.NoError(t, err) + assert.Equal(t, OpenPositionReady, state) }) t.Run("at open position stage and there at least one filled order", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), } currentRound := Round{ OpenPositionOrders: []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), openOrders[0], openOrders[1], openOrders[2], @@ -148,29 +145,29 @@ func Test_RecoverState(t *testing.T) { orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(OpenPositionOrderFilled, state) + assert.NoError(t, err) + assert.Equal(t, OpenPositionOrderFilled, state) }) t.Run("open position stage finish, but stop at cancelling", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusNew, now.Add(-5*time.Second)), } currentRound := Round{ OpenPositionOrders: []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), openOrders[0], }, } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(OpenPositionOrdersCancelling, state) + assert.NoError(t, err) + assert.Equal(t, OpenPositionOrdersCancelling, state) }) t.Run("open-position orders are cancelled", func(t *testing.T) { @@ -178,59 +175,59 @@ func Test_RecoverState(t *testing.T) { openOrders := []types.Order{} currentRound := Round{ OpenPositionOrders: []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), }, } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(OpenPositionOrdersCancelled, state) + assert.NoError(t, err) + assert.Equal(t, OpenPositionOrdersCancelled, state) }) t.Run("at take profit stage, and not filled yet", func(t *testing.T) { now := time.Now() openOrders := []types.Order{ - generateOrder(types.SideTypeSell, types.OrderStatusNew, now), + generateTestOrder(types.SideTypeSell, types.OrderStatusNew, now), } currentRound := Round{ TakeProfitOrder: openOrders[0], OpenPositionOrders: []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), }, } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(TakeProfitReady, state) + assert.NoError(t, err) + assert.Equal(t, TakeProfitReady, state) }) t.Run("at take profit stage, take-profit order filled", func(t *testing.T) { now := time.Now() openOrders := []types.Order{} currentRound := Round{ - TakeProfitOrder: generateOrder(types.SideTypeSell, types.OrderStatusFilled, now), + TakeProfitOrder: generateTestOrder(types.SideTypeSell, types.OrderStatusFilled, now), OpenPositionOrders: []types.Order{ - generateOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), - generateOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-1*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-2*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-3*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-4*time.Second)), + generateTestOrder(types.SideTypeBuy, types.OrderStatusCanceled, now.Add(-5*time.Second)), }, } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) - assert.NoError(err) - assert.Equal(WaitToOpenPosition, state) + assert.NoError(t, err) + assert.Equal(t, WaitToOpenPosition, state) }) } diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 2cc5a332d9..f852aa100f 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -18,6 +18,16 @@ const ( TakeProfitReady ) +var stateTransition map[State]State = map[State]State{ + WaitToOpenPosition: PositionOpening, + PositionOpening: OpenPositionReady, + OpenPositionReady: OpenPositionOrderFilled, + OpenPositionOrderFilled: OpenPositionOrdersCancelling, + OpenPositionOrdersCancelling: OpenPositionOrdersCancelled, + OpenPositionOrdersCancelled: TakeProfitReady, + TakeProfitReady: WaitToOpenPosition, +} + func (s *Strategy) initializeNextStateC() bool { s.mu.Lock() defer s.mu.Unlock() @@ -63,6 +73,19 @@ func (s *Strategy) runState(ctx context.Context) { s.triggerNextState() case nextState := <-s.nextStateC: s.logger.Infof("[DCA] currenct state: %d, next state: %d", s.state, nextState) + + // check the next state is valid + validNextState, exist := stateTransition[s.state] + if !exist { + s.logger.Warnf("[DCA] %d not in stateTransition", s.state) + continue + } + + if nextState != validNextState { + s.logger.Warnf("[DCA] %d is not valid next state of curreny state %d", nextState, s.state) + } + + // move to next state switch s.state { case WaitToOpenPosition: s.runWaitToOpenPositionState(ctx, nextState) @@ -85,28 +108,20 @@ func (s *Strategy) runState(ctx context.Context) { func (s *Strategy) triggerNextState() { switch s.state { - case WaitToOpenPosition: - s.nextStateC <- PositionOpening - case PositionOpening: - s.nextStateC <- OpenPositionReady case OpenPositionReady: - // trigger from order filled event + // only trigger from order filled event case OpenPositionOrderFilled: - // trigger from kline event - case OpenPositionOrdersCancelling: - s.nextStateC <- OpenPositionOrdersCancelled - case OpenPositionOrdersCancelled: - s.nextStateC <- TakeProfitReady + // only trigger from kline event case TakeProfitReady: - // trigger from order filled event + // only trigger from order filled event + default: + if nextState, ok := stateTransition[s.state]; ok { + s.nextStateC <- nextState + } } } func (s *Strategy) runWaitToOpenPositionState(_ context.Context, next State) { - if next != PositionOpening { - return - } - s.logger.Info("[State] WaitToOpenPosition - check startTimeOfNextRound") if time.Now().Before(s.startTimeOfNextRound) { return @@ -117,10 +132,6 @@ func (s *Strategy) runWaitToOpenPositionState(_ context.Context, next State) { } func (s *Strategy) runPositionOpening(ctx context.Context, next State) { - if next != OpenPositionReady { - return - } - s.logger.Info("[State] PositionOpening - start placing open-position orders") if err := s.placeOpenPositionOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to place dca orders, please check it.") @@ -131,17 +142,11 @@ func (s *Strategy) runPositionOpening(ctx context.Context, next State) { } func (s *Strategy) runOpenPositionReady(_ context.Context, next State) { - if next != OpenPositionOrderFilled { - return - } s.state = OpenPositionOrderFilled s.logger.Info("[State] OpenPositionReady -> OpenPositionOrderFilled") } func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { - if next != OpenPositionOrdersCancelling { - return - } s.state = OpenPositionOrdersCancelling s.logger.Info("[State] OpenPositionOrderFilled -> OpenPositionOrdersCancelling") @@ -150,10 +155,6 @@ func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { } func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { - if next != OpenPositionOrdersCancelled { - return - } - s.logger.Info("[State] OpenPositionOrdersCancelling - start cancelling open-position orders") if err := s.cancelOpenPositionOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to cancel maker orders") @@ -167,10 +168,6 @@ func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next Sta } func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next State) { - if next != TakeProfitReady { - return - } - s.logger.Info("[State] OpenPositionOrdersCancelled - start placing take-profit orders") if err := s.placeTakeProfitOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to open take profit orders") @@ -181,10 +178,6 @@ func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next Stat } func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { - if next != WaitToOpenPosition { - return - } - s.logger.Info("[State] TakeProfitReady - start reseting position and calculate budget for next round") if s.Short { s.Budget = s.Budget.Add(s.Position.Base) diff --git a/pkg/types/sort.go b/pkg/types/sort.go index 31840cc5b3..adaa95681d 100644 --- a/pkg/types/sort.go +++ b/pkg/types/sort.go @@ -20,6 +20,14 @@ func SortOrdersAscending(orders []Order) []Order { return orders } +// SortOrdersDescending sorts by creation time descending-ly +func SortOrdersDescending(orders []Order) []Order { + sort.Slice(orders, func(i, j int) bool { + return orders[i].CreationTime.Time().After(orders[j].CreationTime.Time()) + }) + return orders +} + // SortOrdersByPrice sorts by creation time ascending-ly func SortOrdersByPrice(orders []Order, descending bool) []Order { var f func(i, j int) bool From 030c6c1ca51c65d88e76cce73f36ecba577d514c Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Thu, 28 Dec 2023 17:25:07 +0800 Subject: [PATCH 374/422] fix instance id --- .../rebalance/multi_market_strategy.go | 34 ++++++++++++++++--- pkg/strategy/rebalance/order_executor_map.go | 12 ++----- pkg/strategy/rebalance/position_map.go | 22 ------------ pkg/strategy/rebalance/profit_stats_map.go | 19 ----------- pkg/strategy/rebalance/strategy.go | 10 +++--- 5 files changed, 36 insertions(+), 61 deletions(-) delete mode 100644 pkg/strategy/rebalance/position_map.go delete mode 100644 pkg/strategy/rebalance/profit_stats_map.go diff --git a/pkg/strategy/rebalance/multi_market_strategy.go b/pkg/strategy/rebalance/multi_market_strategy.go index e59eff801c..ce0d67955b 100644 --- a/pkg/strategy/rebalance/multi_market_strategy.go +++ b/pkg/strategy/rebalance/multi_market_strategy.go @@ -7,6 +7,9 @@ import ( "github.com/c9s/bbgo/pkg/types" ) +type PositionMap map[string]*types.Position +type ProfitStatsMap map[string]*types.ProfitStats + type MultiMarketStrategy struct { Environ *bbgo.Environment Session *bbgo.ExchangeSession @@ -19,26 +22,47 @@ type MultiMarketStrategy struct { cancel context.CancelFunc } -func (s *MultiMarketStrategy) Initialize(ctx context.Context, environ *bbgo.Environment, session *bbgo.ExchangeSession, markets map[string]types.Market, strategyID string) { +func (s *MultiMarketStrategy) Initialize(ctx context.Context, environ *bbgo.Environment, session *bbgo.ExchangeSession, markets map[string]types.Market, strategyID string, instanceID string) { s.parent = ctx s.ctx, s.cancel = context.WithCancel(ctx) s.Environ = environ s.Session = session + // initialize position map if s.PositionMap == nil { + log.Infof("creating position map") s.PositionMap = make(PositionMap) } - s.PositionMap.CreatePositions(markets) + for symbol, market := range markets { + if _, ok := s.PositionMap[symbol]; ok { + continue + } + + log.Infof("creating position for symbol %s", symbol) + position := types.NewPositionFromMarket(market) + position.Strategy = ID + position.StrategyInstanceID = instanceID + s.PositionMap[symbol] = position + } + // initialize profit stats map if s.ProfitStatsMap == nil { + log.Infof("creating profit stats map") s.ProfitStatsMap = make(ProfitStatsMap) } - s.ProfitStatsMap.CreateProfitStats(markets) + for symbol, market := range markets { + if _, ok := s.ProfitStatsMap[symbol]; ok { + continue + } + + log.Infof("creating profit stats for symbol %s", symbol) + s.ProfitStatsMap[symbol] = types.NewProfitStats(market) + } - s.OrderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap) + // initialize order executor map + s.OrderExecutorMap = NewGeneralOrderExecutorMap(session, strategyID, instanceID, s.PositionMap) s.OrderExecutorMap.BindEnvironment(environ) s.OrderExecutorMap.BindProfitStats(s.ProfitStatsMap) - s.OrderExecutorMap.Sync(ctx, s) s.OrderExecutorMap.Bind() } diff --git a/pkg/strategy/rebalance/order_executor_map.go b/pkg/strategy/rebalance/order_executor_map.go index a4d0d386f9..ed9ceaa04b 100644 --- a/pkg/strategy/rebalance/order_executor_map.go +++ b/pkg/strategy/rebalance/order_executor_map.go @@ -10,12 +10,12 @@ import ( type GeneralOrderExecutorMap map[string]*bbgo.GeneralOrderExecutor -func NewGeneralOrderExecutorMap(session *bbgo.ExchangeSession, positionMap PositionMap) GeneralOrderExecutorMap { +func NewGeneralOrderExecutorMap(session *bbgo.ExchangeSession, strategyID string, instanceID string, positionMap PositionMap) GeneralOrderExecutorMap { m := make(GeneralOrderExecutorMap) for symbol, position := range positionMap { log.Infof("creating order executor for symbol %s", symbol) - orderExecutor := bbgo.NewGeneralOrderExecutor(session, symbol, ID, instanceID(symbol), position) + orderExecutor := bbgo.NewGeneralOrderExecutor(session, symbol, strategyID, instanceID, position) m[symbol] = orderExecutor } @@ -41,14 +41,6 @@ func (m GeneralOrderExecutorMap) Bind() { } } -func (m GeneralOrderExecutorMap) Sync(ctx context.Context, obj interface{}) { - for _, orderExecutor := range m { - orderExecutor.TradeCollector().OnPositionUpdate(func(position *types.Position) { - bbgo.Sync(ctx, obj) - }) - } -} - func (m GeneralOrderExecutorMap) SubmitOrders(ctx context.Context, submitOrders ...types.SubmitOrder) (types.OrderSlice, error) { var allCreatedOrders types.OrderSlice for _, submitOrder := range submitOrders { diff --git a/pkg/strategy/rebalance/position_map.go b/pkg/strategy/rebalance/position_map.go deleted file mode 100644 index 73bdda4996..0000000000 --- a/pkg/strategy/rebalance/position_map.go +++ /dev/null @@ -1,22 +0,0 @@ -package rebalance - -import ( - "github.com/c9s/bbgo/pkg/types" -) - -type PositionMap map[string]*types.Position - -func (m PositionMap) CreatePositions(markets map[string]types.Market) PositionMap { - for symbol, market := range markets { - if _, ok := m[symbol]; ok { - continue - } - - log.Infof("creating position for symbol %s", symbol) - position := types.NewPositionFromMarket(market) - position.Strategy = ID - position.StrategyInstanceID = instanceID(symbol) - m[symbol] = position - } - return m -} diff --git a/pkg/strategy/rebalance/profit_stats_map.go b/pkg/strategy/rebalance/profit_stats_map.go deleted file mode 100644 index 29e427a6e8..0000000000 --- a/pkg/strategy/rebalance/profit_stats_map.go +++ /dev/null @@ -1,19 +0,0 @@ -package rebalance - -import ( - "github.com/c9s/bbgo/pkg/types" -) - -type ProfitStatsMap map[string]*types.ProfitStats - -func (m ProfitStatsMap) CreateProfitStats(markets map[string]types.Market) ProfitStatsMap { - for symbol, market := range markets { - if _, ok := m[symbol]; ok { - continue - } - - log.Infof("creating profit stats for symbol %s", symbol) - m[symbol] = types.NewProfitStats(market) - } - return m -} diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index cfeb9844d8..33ea2ef8be 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -22,10 +22,6 @@ func init() { bbgo.RegisterStrategy(ID, &Strategy{}) } -func instanceID(symbol string) string { - return fmt.Sprintf("%s:%s", ID, symbol) -} - type Strategy struct { *MultiMarketStrategy @@ -72,6 +68,10 @@ func (s *Strategy) ID() string { return ID } +func (s *Strategy) InstanceID() string { + return ID +} + func (s *Strategy) Validate() error { if len(s.TargetWeights) == 0 { return fmt.Errorf("targetWeights should not be empty") @@ -109,7 +109,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.markets[symbol] = market } - s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) + s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID, s.InstanceID()) s.activeOrderBook = bbgo.NewActiveOrderBook("") s.activeOrderBook.BindStream(session.UserDataStream) From 57282c30d2a4db14cf459aa8aea5dd58a7567084 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Fri, 22 Dec 2023 15:50:48 +0800 Subject: [PATCH 375/422] FEATURE: remove Short --- pkg/strategy/dca2/open_position.go | 27 +++++++------------------ pkg/strategy/dca2/open_position_test.go | 3 +-- pkg/strategy/dca2/recover.go | 13 ++++-------- pkg/strategy/dca2/recover_test.go | 18 ++++++++--------- pkg/strategy/dca2/state.go | 9 ++++----- pkg/strategy/dca2/strategy.go | 10 +-------- pkg/strategy/dca2/take_profit.go | 8 ++------ pkg/strategy/dca2/take_profit_test.go | 4 ++-- 8 files changed, 30 insertions(+), 62 deletions(-) diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index 6a25c5c649..04e886e3d9 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -15,12 +15,12 @@ type cancelOrdersByGroupIDApi interface { func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { s.logger.Infof("[DCA] start placing open position orders") - price, err := getBestPriceUntilSuccess(ctx, s.Session.Exchange, s.Symbol, s.Short) + price, err := getBestPriceUntilSuccess(ctx, s.Session.Exchange, s.Symbol) if err != nil { return err } - orders, err := generateOpenPositionOrders(s.Market, s.Short, s.Budget, price, s.PriceDeviation, s.MaxOrderNum, s.OrderGroupID) + orders, err := generateOpenPositionOrders(s.Market, s.Budget, price, s.PriceDeviation, s.MaxOrderNum, s.OrderGroupID) if err != nil { return err } @@ -35,24 +35,17 @@ func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { return nil } -func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol string, short bool) (fixedpoint.Value, error) { +func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol string) (fixedpoint.Value, error) { ticker, err := retry.QueryTickerUntilSuccessful(ctx, ex, symbol) if err != nil { return fixedpoint.Zero, err } - if short { - return ticker.Buy, nil - } else { - return ticker.Sell, nil - } + return ticker.Sell, nil } -func generateOpenPositionOrders(market types.Market, short bool, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64, orderGroupID uint32) ([]types.SubmitOrder, error) { +func generateOpenPositionOrders(market types.Market, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64, orderGroupID uint32) ([]types.SubmitOrder, error) { factor := fixedpoint.One.Sub(priceDeviation) - if short { - factor = fixedpoint.One.Add(priceDeviation) - } // calculate all valid prices var prices []fixedpoint.Value @@ -68,15 +61,12 @@ func generateOpenPositionOrders(market types.Market, short bool, budget, price, prices = append(prices, price) } - notional, orderNum := calculateNotionalAndNum(market, short, budget, prices) + notional, orderNum := calculateNotionalAndNum(market, budget, prices) if orderNum == 0 { return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, budget: %s", price, budget) } side := types.SideTypeBuy - if short { - side = types.SideTypeSell - } var submitOrders []types.SubmitOrder for i := 0; i < orderNum; i++ { @@ -99,7 +89,7 @@ func generateOpenPositionOrders(market types.Market, short bool, budget, price, // calculateNotionalAndNum calculates the notional and num of open position orders // DCA2 is notional-based, every order has the same notional -func calculateNotionalAndNum(market types.Market, short bool, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { +func calculateNotionalAndNum(market types.Market, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { for num := len(prices); num > 0; num-- { notional := budget.Div(fixedpoint.NewFromInt(int64(num))) if notional.Compare(market.MinNotional) < 0 { @@ -107,9 +97,6 @@ func calculateNotionalAndNum(market types.Market, short bool, budget fixedpoint. } maxPriceIdx := 0 - if short { - maxPriceIdx = num - 1 - } quantity := market.TruncateQuantity(notional.Div(prices[maxPriceIdx])) if quantity.Compare(market.MinQuantity) < 0 { continue diff --git a/pkg/strategy/dca2/open_position_test.go b/pkg/strategy/dca2/open_position_test.go index a9fd33cf08..4d68df5106 100644 --- a/pkg/strategy/dca2/open_position_test.go +++ b/pkg/strategy/dca2/open_position_test.go @@ -36,7 +36,6 @@ func newTestStrategy(va ...string) *Strategy { logger: logrus.NewEntry(logrus.New()), Symbol: symbol, Market: market, - Short: false, TakeProfitRatio: Number("10%"), } return s @@ -51,7 +50,7 @@ func TestGenerateOpenPositionOrders(t *testing.T) { budget := Number("10500") askPrice := Number("30000") margin := Number("0.05") - submitOrders, err := generateOpenPositionOrders(strategy.Market, false, budget, askPrice, margin, 4, strategy.OrderGroupID) + submitOrders, err := generateOpenPositionOrders(strategy.Market, budget, askPrice, margin, 4, strategy.OrderGroupID) if !assert.NoError(err) { return } diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index 5be876a6df..6a09dd6c9f 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -39,14 +39,14 @@ func (s *Strategy) recover(ctx context.Context) error { return err } - currentRound, err := getCurrentRoundOrders(s.Short, openOrders, closedOrders, s.OrderGroupID) + currentRound, err := getCurrentRoundOrders(openOrders, closedOrders, s.OrderGroupID) if err != nil { return err } debugRoundOrders(s.logger, "current", currentRound) // recover state - state, err := recoverState(ctx, s.Symbol, s.Short, int(s.MaxOrderNum), openOrders, currentRound, s.OrderExecutor.ActiveMakerOrders(), s.OrderExecutor.OrderStore(), s.OrderGroupID) + state, err := recoverState(ctx, s.Symbol, int(s.MaxOrderNum), openOrders, currentRound, s.OrderExecutor.ActiveMakerOrders(), s.OrderExecutor.OrderStore(), s.OrderGroupID) if err != nil { return err } @@ -72,7 +72,7 @@ func (s *Strategy) recover(ctx context.Context) error { } // recover state -func recoverState(ctx context.Context, symbol string, short bool, maxOrderNum int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { +func recoverState(ctx context.Context, symbol string, maxOrderNum int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { if len(currentRound.OpenPositionOrders) == 0 { // new strategy return WaitToOpenPosition, nil @@ -225,15 +225,10 @@ type Round struct { TakeProfitOrder types.Order } -func getCurrentRoundOrders(short bool, openOrders, closedOrders []types.Order, groupID uint32) (Round, error) { +func getCurrentRoundOrders(openOrders, closedOrders []types.Order, groupID uint32) (Round, error) { openPositionSide := types.SideTypeBuy takeProfitSide := types.SideTypeSell - if short { - openPositionSide = types.SideTypeSell - takeProfitSide = types.SideTypeBuy - } - var allOrders []types.Order allOrders = append(allOrders, openOrders...) allOrders = append(allOrders, closedOrders...) diff --git a/pkg/strategy/dca2/recover_test.go b/pkg/strategy/dca2/recover_test.go index 0b7a501d23..55ebda5101 100644 --- a/pkg/strategy/dca2/recover_test.go +++ b/pkg/strategy/dca2/recover_test.go @@ -39,7 +39,7 @@ func Test_GetCurrenctAndLastRoundOrders(t *testing.T) { generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-5*time.Second)), } - currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) + currentRound, err := getCurrentRoundOrders(openOrders, closedOrders, 0) assert.NoError(t, err) assert.NotEqual(t, 0, currentRound.TakeProfitOrder.OrderID) @@ -72,7 +72,7 @@ func Test_GetCurrenctAndLastRoundOrders(t *testing.T) { generateTestOrder(types.SideTypeBuy, types.OrderStatusFilled, now.Add(-17*time.Second)), } - currentRound, err := getCurrentRoundOrders(false, openOrders, closedOrders, 0) + currentRound, err := getCurrentRoundOrders(openOrders, closedOrders, 0) assert.NoError(t, err) assert.NotEqual(t, 0, currentRound.TakeProfitOrder.OrderID) @@ -101,7 +101,7 @@ func Test_RecoverState(t *testing.T) { currentRound := Round{} activeOrderBook := bbgo.NewActiveOrderBook(symbol) orderStore := core.NewOrderStore(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, WaitToOpenPosition, state) }) @@ -120,7 +120,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, OpenPositionReady, state) }) @@ -144,7 +144,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, OpenPositionOrderFilled, state) }) @@ -165,7 +165,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, OpenPositionOrdersCancelling, state) }) @@ -184,7 +184,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, OpenPositionOrdersCancelled, state) }) @@ -206,7 +206,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, TakeProfitReady, state) }) @@ -226,7 +226,7 @@ func Test_RecoverState(t *testing.T) { } orderStore := core.NewOrderStore(symbol) activeOrderBook := bbgo.NewActiveOrderBook(symbol) - state, err := recoverState(context.Background(), symbol, false, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) + state, err := recoverState(context.Background(), symbol, 5, openOrders, currentRound, activeOrderBook, orderStore, 0) assert.NoError(t, err) assert.Equal(t, WaitToOpenPosition, state) }) diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index f852aa100f..6936079950 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -178,12 +178,11 @@ func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next Stat } func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { + // wait 3 seconds to avoid position not update + time.Sleep(3 * time.Second) + s.logger.Info("[State] TakeProfitReady - start reseting position and calculate budget for next round") - if s.Short { - s.Budget = s.Budget.Add(s.Position.Base) - } else { - s.Budget = s.Budget.Add(s.Position.Quote) - } + s.Budget = s.Budget.Add(s.Position.Quote) // reset position s.Position.Reset() diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 92dfa8db73..d1eab7ffd0 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -34,7 +34,6 @@ type Strategy struct { Symbol string `json:"symbol"` // setting - Short bool `json:"short"` Budget fixedpoint.Value `json:"budget"` MaxOrderNum int64 `json:"maxOrderNum"` PriceDeviation fixedpoint.Value `json:"priceDeviation"` @@ -122,10 +121,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Infof("[DCA] FILLED ORDER: %s", o.String()) openPositionSide := types.SideTypeBuy takeProfitSide := types.SideTypeSell - if s.Short { - openPositionSide = types.SideTypeSell - takeProfitSide = types.SideTypeBuy - } switch o.Side { case openPositionSide: @@ -145,7 +140,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. compRes := kline.Close.Compare(s.takeProfitPrice) // price doesn't hit the take profit price - if (s.Short && compRes > 0) || (!s.Short && compRes < 0) { + if compRes < 0 { return } @@ -193,9 +188,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. func (s *Strategy) updateTakeProfitPrice() { takeProfitRatio := s.TakeProfitRatio - if s.Short { - takeProfitRatio = takeProfitRatio.Neg() - } s.takeProfitPrice = s.Market.TruncatePrice(s.Position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) s.logger.Infof("[DCA] cost: %s, ratio: %s, price: %s", s.Position.AverageCost, takeProfitRatio, s.takeProfitPrice) } diff --git a/pkg/strategy/dca2/take_profit.go b/pkg/strategy/dca2/take_profit.go index 148312abbf..adfa0bb579 100644 --- a/pkg/strategy/dca2/take_profit.go +++ b/pkg/strategy/dca2/take_profit.go @@ -9,7 +9,7 @@ import ( func (s *Strategy) placeTakeProfitOrders(ctx context.Context) error { s.logger.Info("[DCA] start placing take profit orders") - order := generateTakeProfitOrder(s.Market, s.Short, s.TakeProfitRatio, s.Position, s.OrderGroupID) + order := generateTakeProfitOrder(s.Market, s.TakeProfitRatio, s.Position, s.OrderGroupID) createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, order) if err != nil { return err @@ -22,12 +22,8 @@ func (s *Strategy) placeTakeProfitOrders(ctx context.Context) error { return nil } -func generateTakeProfitOrder(market types.Market, short bool, takeProfitRatio fixedpoint.Value, position *types.Position, orderGroupID uint32) types.SubmitOrder { +func generateTakeProfitOrder(market types.Market, takeProfitRatio fixedpoint.Value, position *types.Position, orderGroupID uint32) types.SubmitOrder { side := types.SideTypeSell - if short { - takeProfitRatio = takeProfitRatio.Neg() - side = types.SideTypeBuy - } takeProfitPrice := market.TruncatePrice(position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) return types.SubmitOrder{ Symbol: market.Symbol, diff --git a/pkg/strategy/dca2/take_profit_test.go b/pkg/strategy/dca2/take_profit_test.go index 1080e3ba85..70df92868d 100644 --- a/pkg/strategy/dca2/take_profit_test.go +++ b/pkg/strategy/dca2/take_profit_test.go @@ -24,7 +24,7 @@ func TestGenerateTakeProfitOrder(t *testing.T) { FeeCurrency: strategy.Market.BaseCurrency, }) - o := generateTakeProfitOrder(strategy.Market, false, strategy.TakeProfitRatio, position, strategy.OrderGroupID) + o := generateTakeProfitOrder(strategy.Market, strategy.TakeProfitRatio, position, strategy.OrderGroupID) assert.Equal(Number("31397.09"), o.Price) assert.Equal(Number("0.9985"), o.Quantity) assert.Equal(types.SideTypeSell, o.Side) @@ -38,7 +38,7 @@ func TestGenerateTakeProfitOrder(t *testing.T) { Fee: Number("0.00075"), FeeCurrency: strategy.Market.BaseCurrency, }) - o = generateTakeProfitOrder(strategy.Market, false, strategy.TakeProfitRatio, position, strategy.OrderGroupID) + o = generateTakeProfitOrder(strategy.Market, strategy.TakeProfitRatio, position, strategy.OrderGroupID) assert.Equal(Number("30846.26"), o.Price) assert.Equal(Number("1.49775"), o.Quantity) assert.Equal(types.SideTypeSell, o.Side) From 9ad94aa7e0be96f5ec49c91d46a7f8e7eb817bf5 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 2 Jan 2024 12:02:33 +0800 Subject: [PATCH 376/422] pkg/exchange: add stream test for book --- pkg/exchange/okex/stream_test.go | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 pkg/exchange/okex/stream_test.go diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go new file mode 100644 index 0000000000..596b4b9780 --- /dev/null +++ b/pkg/exchange/okex/stream_test.go @@ -0,0 +1,51 @@ +package okex + +import ( + "context" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/testutil" + "github.com/c9s/bbgo/pkg/types" +) + +func getTestClientOrSkip(t *testing.T) *Stream { + if b, _ := strconv.ParseBool(os.Getenv("CI")); b { + t.Skip("skip test for CI") + } + + key, secret, passphrase, ok := testutil.IntegrationTestWithPassphraseConfigured(t, "OKEX") + if !ok { + t.Skip("OKEX_* env vars are not configured") + return nil + } + + exchange := New(key, secret, passphrase) + return NewStream(exchange.client) +} + +func TestStream(t *testing.T) { + t.Skip() + s := getTestClientOrSkip(t) + + t.Run("book test", func(t *testing.T) { + s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ + Depth: types.DepthLevel50, + }) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBookSnapshot(func(book types.SliceOrderBook) { + t.Log("got snapshot", book) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", book) + }) + c := make(chan struct{}) + <-c + }) +} From 90020a65a4c0d707264b4b542c8a272393785e3b Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Tue, 2 Jan 2024 16:56:38 +0800 Subject: [PATCH 377/422] improve/sync-futures: do not use GetSessionAttributes() --- pkg/backtest/exchange.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index 183b41c39a..b0600d54fc 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -30,7 +30,6 @@ package backtest import ( "context" "fmt" - exchange2 "github.com/c9s/bbgo/pkg/exchange" "strconv" "sync" "time" @@ -382,7 +381,14 @@ func (e *Exchange) SubscribeMarketData( intervals = append(intervals, interval) } - _, isFutures, _, _ := exchange2.GetSessionAttributes(e.publicExchange) + //_, isFutures, _, _ := exchange2.GetSessionAttributes(e.publicExchange) + var isFutures bool + if futuresExchange, ok := e.publicExchange.(types.FuturesExchange); ok { + isFutures = futuresExchange.GetFuturesSettings().IsFutures + } else { + isFutures = false + } + if isFutures { log.Infof("querying futures klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) } else { From 05536b6693091faa1ad7a0e78dca5cc79755acc7 Mon Sep 17 00:00:00 2001 From: Andy Cheng Date: Wed, 3 Jan 2024 10:36:01 +0800 Subject: [PATCH 378/422] improve/sync-futures: remove unused code --- pkg/backtest/exchange.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index b0600d54fc..328cd6ffe1 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -381,7 +381,6 @@ func (e *Exchange) SubscribeMarketData( intervals = append(intervals, interval) } - //_, isFutures, _, _ := exchange2.GetSessionAttributes(e.publicExchange) var isFutures bool if futuresExchange, ok := e.publicExchange.(types.FuturesExchange); ok { isFutures = futuresExchange.GetFuturesSettings().IsFutures From 30164acdcff551e74dc9a510a9cac313c8d002cc Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 3 Jan 2024 11:25:46 +0800 Subject: [PATCH 379/422] pkg/exchange: use v2 get account asset api --- .../bitgetapi/v2/get_account_assets_request.go | 2 +- pkg/exchange/bitget/convert.go | 7 +++---- pkg/exchange/bitget/convert_test.go | 15 +++++++-------- pkg/exchange/bitget/exchange.go | 4 ++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go index d87468143e..51ab058066 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_account_assets_request.go @@ -31,7 +31,7 @@ type GetAccountAssetsRequest struct { client requestgen.AuthenticatedAPIClient coin *string `param:"symbol,query"` - assetType AssetType `param:"limit,query"` + assetType AssetType `param:"assetType,query"` } func (c *Client) NewGetAccountAssetsRequest() *GetAccountAssetsRequest { diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index c0167d16fc..9e6a65504b 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -7,17 +7,16 @@ import ( "strconv" "time" - "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) -func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { +func toGlobalBalance(asset v2.AccountAsset) types.Balance { return types.Balance{ - Currency: asset.CoinName, + Currency: asset.Coin, Available: asset.Available, - Locked: asset.Lock.Add(asset.Frozen), + Locked: asset.Locked.Add(asset.Frozen), Borrowed: fixedpoint.Zero, Interest: fixedpoint.Zero, NetAsset: fixedpoint.Zero, diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index a0898e21de..cf3f0ab94a 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" v2 "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi/v2" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -23,13 +22,13 @@ func Test_toGlobalBalance(t *testing.T) { // "lock":"0", // "uTime":"1622697148" // } - asset := bitgetapi.AccountAsset{ - CoinId: 2, - CoinName: "USDT", - Available: fixedpoint.NewFromFloat(1.2), - Frozen: fixedpoint.NewFromFloat(0.5), - Lock: fixedpoint.NewFromFloat(0.5), - UTime: types.NewMillisecondTimestampFromInt(1622697148), + asset := v2.AccountAsset{ + Coin: "USDT", + Available: fixedpoint.NewFromFloat(1.2), + Frozen: fixedpoint.NewFromFloat(0.5), + Locked: fixedpoint.NewFromFloat(0.5), + LimitAvailable: fixedpoint.Zero, + UpdatedTime: types.NewMillisecondTimestampFromInt(1622697148), } assert.Equal(t, types.Balance{ diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index dc0ca8e7ab..4cacd200d4 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -254,7 +254,7 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, return nil, fmt.Errorf("account rate limiter wait error: %w", err) } - req := e.client.NewGetAccountAssetsRequest() + req := e.v2client.NewGetAccountAssetsRequest().AssetType(v2.AssetTypeHoldOnly) resp, err := req.Do(ctx) if err != nil { return nil, fmt.Errorf("failed to query account assets: %w", err) @@ -263,7 +263,7 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, bals := types.BalanceMap{} for _, asset := range resp { b := toGlobalBalance(asset) - bals[asset.CoinName] = b + bals[asset.Coin] = b } return bals, nil From b5ff066aa23697b8b335ff6c00fcf3134db04a97 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 3 Jan 2024 11:30:50 +0800 Subject: [PATCH 380/422] pkg/exchange: print symbol --- pkg/exchange/bitget/exchange.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 4cacd200d4..38b96bf423 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -143,10 +143,10 @@ func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticke req.Symbol(symbol) resp, err := req.Do(ctx) if err != nil { - return nil, fmt.Errorf("failed to query ticker: %w", err) + return nil, fmt.Errorf("failed to query ticker, symbol: %s, err: %w", symbol, err) } if len(resp) != 1 { - return nil, fmt.Errorf("unexpected length of query single symbol: %+v", resp) + return nil, fmt.Errorf("unexpected length of query single symbol: %s, resp: %+v", symbol, resp) } ticker := toGlobalTicker(resp[0]) From 94fb883a0f26aafefe37de488b8b800c2d462f8d Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Thu, 4 Jan 2024 18:21:24 +0800 Subject: [PATCH 381/422] xgap: fix order cancel error --- pkg/strategy/xgap/strategy.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/xgap/strategy.go b/pkg/strategy/xgap/strategy.go index c7d0546fd9..491d9edd20 100644 --- a/pkg/strategy/xgap/strategy.go +++ b/pkg/strategy/xgap/strategy.go @@ -18,8 +18,6 @@ import ( const ID = "xgap" -const stateKey = "state-v1" - var log = logrus.WithField("strategy", ID) var StepPercentageGap = fixedpoint.NewFromFloat(0.05) @@ -77,6 +75,8 @@ type Strategy struct { sourceBook, tradingBook *types.StreamOrderBook groupID uint32 + activeOrderBook *bbgo.ActiveOrderBook + stopC chan struct{} } @@ -213,6 +213,9 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se s.groupID = util.FNV32(instanceID) % math.MaxInt32 log.Infof("using group id %d from fnv32(%s)", s.groupID, instanceID) + s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) + s.activeOrderBook.BindStream(s.tradingSession.UserDataStream) + go func() { ticker := time.NewTicker( util.MillisecondsJitter(s.UpdateInterval.Duration(), 1000), @@ -355,9 +358,11 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se log.WithError(err).Error("order submit error") } + s.activeOrderBook.Add(createdOrders...) + time.Sleep(time.Second) - if err := tradingSession.Exchange.CancelOrders(ctx, createdOrders...); err != nil { + if err := s.activeOrderBook.GracefulCancel(ctx, s.tradingSession.Exchange); err != nil { log.WithError(err).Error("cancel order error") } } From 29f36423951c9995937baf1a2c0c161307c51214 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sat, 6 Jan 2024 14:05:39 +0800 Subject: [PATCH 382/422] specify the version of morphy2k/revive-action to 2.5.4 --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 79ca5258cf..f13aa092a9 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -87,7 +87,7 @@ jobs: sed -i -e '/_requestgen.go/d' coverage_dnum.txt - name: Revive Check - uses: morphy2k/revive-action@v2 + uses: morphy2k/revive-action@v2.5.4 # https://github.com/mgechev/revive/issues/956 with: reporter: github-pr-review fail_on_error: true From 012fc333766a23d447d06dd947a2159665acb957 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Fri, 5 Jan 2024 10:23:12 +0800 Subject: [PATCH 383/422] xgap: refactor with common strategy --- pkg/strategy/xgap/strategy.go | 284 ++++++++++++++++++---------------- 1 file changed, 151 insertions(+), 133 deletions(-) diff --git a/pkg/strategy/xgap/strategy.go b/pkg/strategy/xgap/strategy.go index 491d9edd20..8f48e100b8 100644 --- a/pkg/strategy/xgap/strategy.go +++ b/pkg/strategy/xgap/strategy.go @@ -12,6 +12,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -32,6 +33,10 @@ func (s *Strategy) ID() string { return ID } +func (s *Strategy) InstanceID() string { + return fmt.Sprintf("%s:%s", ID, s.Symbol) +} + type State struct { AccumulatedFeeStartedAt time.Time `json:"accumulatedFeeStartedAt,omitempty"` AccumulatedFees map[string]fixedpoint.Value `json:"accumulatedFees,omitempty"` @@ -54,6 +59,10 @@ func (s *State) Reset() { } type Strategy struct { + *common.Strategy + + Environment *bbgo.Environment + Symbol string `json:"symbol"` SourceExchange string `json:"sourceExchange"` TradingExchange string `json:"tradingExchange"` @@ -73,13 +82,28 @@ type Strategy struct { mu sync.Mutex lastSourceKLine, lastTradingKLine types.KLine sourceBook, tradingBook *types.StreamOrderBook - groupID uint32 - - activeOrderBook *bbgo.ActiveOrderBook stopC chan struct{} } +func (s *Strategy) Initialize() error { + if s.Strategy == nil { + s.Strategy = &common.Strategy{} + } + return nil +} + +func (s *Strategy) Validate() error { + return nil +} + +func (s *Strategy) Defaults() error { + if s.UpdateInterval == 0 { + s.UpdateInterval = types.Duration(time.Second) + } + return nil +} + func (s *Strategy) isBudgetAllowed() bool { if s.DailyFeeBudgets == nil { return true @@ -141,10 +165,6 @@ func (s *Strategy) CrossSubscribe(sessions map[string]*bbgo.ExchangeSession) { } func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, sessions map[string]*bbgo.ExchangeSession) error { - if s.UpdateInterval == 0 { - s.UpdateInterval = types.Duration(time.Second) - } - sourceSession, ok := sessions[s.SourceExchange] if !ok { return fmt.Errorf("source session %s is not defined", s.SourceExchange) @@ -167,6 +187,8 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se return fmt.Errorf("trading session market %s is not defined", s.Symbol) } + s.Strategy.Initialize(ctx, s.Environment, tradingSession, s.tradingMarket, ID, s.InstanceID()) + s.stopC = make(chan struct{}) if s.State == nil { @@ -209,13 +231,6 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se s.tradingSession.UserDataStream.OnTradeUpdate(s.handleTradeUpdate) - instanceID := fmt.Sprintf("%s-%s", ID, s.Symbol) - s.groupID = util.FNV32(instanceID) % math.MaxInt32 - log.Infof("using group id %d from fnv32(%s)", s.groupID, instanceID) - - s.activeOrderBook = bbgo.NewActiveOrderBook(s.Symbol) - s.activeOrderBook.BindStream(s.tradingSession.UserDataStream) - go func() { ticker := time.NewTicker( util.MillisecondsJitter(s.UpdateInterval.Duration(), 1000), @@ -241,133 +256,136 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se time.Sleep(delay) } - bestBid, hasBid := s.tradingBook.BestBid() - bestAsk, hasAsk := s.tradingBook.BestAsk() - - // try to use the bid/ask price from the trading book - if hasBid && hasAsk { - var spread = bestAsk.Price.Sub(bestBid.Price) - var spreadPercentage = spread.Div(bestAsk.Price) - log.Infof("trading book spread=%s %s", - spread.String(), spreadPercentage.Percentage()) - - // use the source book price if the spread percentage greater than 10% - if spreadPercentage.Compare(StepPercentageGap) > 0 { - log.Warnf("spread too large (%s %s), using source book", - spread.String(), spreadPercentage.Percentage()) - bestBid, hasBid = s.sourceBook.BestBid() - bestAsk, hasAsk = s.sourceBook.BestAsk() - } - - if s.MinSpread.Sign() > 0 { - if spread.Compare(s.MinSpread) < 0 { - log.Warnf("spread < min spread, spread=%s minSpread=%s bid=%s ask=%s", - spread.String(), s.MinSpread.String(), - bestBid.Price.String(), bestAsk.Price.String()) - continue - } - } - - // if the spread is less than 100 ticks (100 pips), skip - if spread.Compare(s.tradingMarket.TickSize.MulExp(2)) < 0 { - log.Warnf("spread too small, we can't place orders: spread=%v bid=%v ask=%v", - spread, bestBid.Price, bestAsk.Price) - continue - } - - } else { - bestBid, hasBid = s.sourceBook.BestBid() - bestAsk, hasAsk = s.sourceBook.BestAsk() - } + s.placeOrders(ctx) - if !hasBid || !hasAsk { - log.Warn("no bids or asks on the source book or the trading book") - continue - } + s.cancelOrders(ctx) + } + } + }() - var spread = bestAsk.Price.Sub(bestBid.Price) - var spreadPercentage = spread.Div(bestAsk.Price) - log.Infof("spread=%v %s ask=%v bid=%v", - spread, spreadPercentage.Percentage(), - bestAsk.Price, bestBid.Price) - // var spreadPercentage = spread.Float64() / bestBid.Price.Float64() - - var midPrice = bestAsk.Price.Add(bestBid.Price).Div(Two) - var price = midPrice - - log.Infof("mid price %v", midPrice) - - var balances = s.tradingSession.GetAccount().Balances() - var quantity = s.tradingMarket.MinQuantity - - if s.Quantity.Sign() > 0 { - quantity = fixedpoint.Min(s.Quantity, s.tradingMarket.MinQuantity) - } else if s.SimulateVolume { - s.mu.Lock() - if s.lastTradingKLine.Volume.Sign() > 0 && s.lastSourceKLine.Volume.Sign() > 0 { - volumeDiff := s.lastSourceKLine.Volume.Sub(s.lastTradingKLine.Volume) - // change the current quantity only diff is positive - if volumeDiff.Sign() > 0 { - quantity = volumeDiff - } - - if baseBalance, ok := balances[s.tradingMarket.BaseCurrency]; ok { - quantity = fixedpoint.Min(quantity, baseBalance.Available) - } - - if quoteBalance, ok := balances[s.tradingMarket.QuoteCurrency]; ok { - maxQuantity := quoteBalance.Available.Div(price) - quantity = fixedpoint.Min(quantity, maxQuantity) - } - } - s.mu.Unlock() - } else { - // plus a 2% quantity jitter - jitter := 1.0 + math.Max(0.02, rand.Float64()) - quantity = quantity.Mul(fixedpoint.NewFromFloat(jitter)) - } + return nil +} - var quoteAmount = price.Mul(quantity) - if quoteAmount.Compare(s.tradingMarket.MinNotional) <= 0 { - quantity = fixedpoint.Max( - s.tradingMarket.MinQuantity, - s.tradingMarket.MinNotional.Mul(NotionModifier).Div(price)) - } +func (s *Strategy) placeOrders(ctx context.Context) { + bestBid, hasBid := s.tradingBook.BestBid() + bestAsk, hasAsk := s.tradingBook.BestAsk() + + // try to use the bid/ask price from the trading book + if hasBid && hasAsk { + var spread = bestAsk.Price.Sub(bestBid.Price) + var spreadPercentage = spread.Div(bestAsk.Price) + log.Infof("trading book spread=%s %s", + spread.String(), spreadPercentage.Percentage()) + + // use the source book price if the spread percentage greater than 10% + if spreadPercentage.Compare(StepPercentageGap) > 0 { + log.Warnf("spread too large (%s %s), using source book", + spread.String(), spreadPercentage.Percentage()) + bestBid, hasBid = s.sourceBook.BestBid() + bestAsk, hasAsk = s.sourceBook.BestAsk() + } - createdOrders, _, err := bbgo.BatchPlaceOrder(ctx, tradingSession.Exchange, nil, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeBuy, - Type: types.OrderTypeLimit, - Quantity: quantity, - Price: price, - Market: s.tradingMarket, - // TimeInForce: types.TimeInForceGTC, - GroupID: s.groupID, - }, types.SubmitOrder{ - Symbol: s.Symbol, - Side: types.SideTypeSell, - Type: types.OrderTypeLimit, - Quantity: quantity, - Price: price, - Market: s.tradingMarket, - // TimeInForce: types.TimeInForceGTC, - GroupID: s.groupID, - }) - - if err != nil { - log.WithError(err).Error("order submit error") - } + if s.MinSpread.Sign() > 0 { + if spread.Compare(s.MinSpread) < 0 { + log.Warnf("spread < min spread, spread=%s minSpread=%s bid=%s ask=%s", + spread.String(), s.MinSpread.String(), + bestBid.Price.String(), bestAsk.Price.String()) + return + } + } - s.activeOrderBook.Add(createdOrders...) + // if the spread is less than 100 ticks (100 pips), skip + if spread.Compare(s.tradingMarket.TickSize.MulExp(2)) < 0 { + log.Warnf("spread too small, we can't place orders: spread=%v bid=%v ask=%v", + spread, bestBid.Price, bestAsk.Price) + return + } - time.Sleep(time.Second) + } else { + bestBid, hasBid = s.sourceBook.BestBid() + bestAsk, hasAsk = s.sourceBook.BestAsk() + } - if err := s.activeOrderBook.GracefulCancel(ctx, s.tradingSession.Exchange); err != nil { - log.WithError(err).Error("cancel order error") - } + if !hasBid || !hasAsk { + log.Warn("no bids or asks on the source book or the trading book") + return + } + + var spread = bestAsk.Price.Sub(bestBid.Price) + var spreadPercentage = spread.Div(bestAsk.Price) + log.Infof("spread=%v %s ask=%v bid=%v", + spread, spreadPercentage.Percentage(), + bestAsk.Price, bestBid.Price) + // var spreadPercentage = spread.Float64() / bestBid.Price.Float64() + + var midPrice = bestAsk.Price.Add(bestBid.Price).Div(Two) + var price = midPrice + + log.Infof("mid price %v", midPrice) + + var balances = s.tradingSession.GetAccount().Balances() + var quantity = s.tradingMarket.MinQuantity + + if s.Quantity.Sign() > 0 { + quantity = fixedpoint.Min(s.Quantity, s.tradingMarket.MinQuantity) + } else if s.SimulateVolume { + s.mu.Lock() + if s.lastTradingKLine.Volume.Sign() > 0 && s.lastSourceKLine.Volume.Sign() > 0 { + volumeDiff := s.lastSourceKLine.Volume.Sub(s.lastTradingKLine.Volume) + // change the current quantity only diff is positive + if volumeDiff.Sign() > 0 { + quantity = volumeDiff + } + + if baseBalance, ok := balances[s.tradingMarket.BaseCurrency]; ok { + quantity = fixedpoint.Min(quantity, baseBalance.Available) + } + + if quoteBalance, ok := balances[s.tradingMarket.QuoteCurrency]; ok { + maxQuantity := quoteBalance.Available.Div(price) + quantity = fixedpoint.Min(quantity, maxQuantity) } } - }() + s.mu.Unlock() + } else { + // plus a 2% quantity jitter + jitter := 1.0 + math.Max(0.02, rand.Float64()) + quantity = quantity.Mul(fixedpoint.NewFromFloat(jitter)) + } - return nil + var quoteAmount = price.Mul(quantity) + if quoteAmount.Compare(s.tradingMarket.MinNotional) <= 0 { + quantity = fixedpoint.Max( + s.tradingMarket.MinQuantity, + s.tradingMarket.MinNotional.Mul(NotionModifier).Div(price)) + } + + orderForm := []types.SubmitOrder{{ + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: quantity, + Price: price, + Market: s.tradingMarket, + }, { + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeLimit, + Quantity: quantity, + Price: price, + Market: s.tradingMarket, + }} + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForm...) + if err != nil { + log.WithError(err).Error("order submit error") + } + log.Infof("created orders: %+v", createdOrders) + + time.Sleep(time.Second) +} + +func (s *Strategy) cancelOrders(ctx context.Context) { + if err := s.OrderExecutor.GracefulCancel(ctx); err != nil { + log.WithError(err).Error("cancel order error") + } } From 3ee5bf29ef341e6acb97444aa49620022c59edf5 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sat, 6 Jan 2024 15:24:14 +0800 Subject: [PATCH 384/422] xgap: improve log message --- pkg/strategy/xgap/strategy.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/strategy/xgap/strategy.go b/pkg/strategy/xgap/strategy.go index 8f48e100b8..a72aaf1d3d 100644 --- a/pkg/strategy/xgap/strategy.go +++ b/pkg/strategy/xgap/strategy.go @@ -126,7 +126,7 @@ func (s *Strategy) isBudgetAllowed() bool { } func (s *Strategy) handleTradeUpdate(trade types.Trade) { - log.Infof("received trade %+v", trade) + log.Infof("received trade %s", trade.String()) if trade.Symbol != s.Symbol { return @@ -209,15 +209,11 @@ func (s *Strategy) CrossRun(ctx context.Context, _ bbgo.OrderExecutionRouter, se // from here, set data binding s.sourceSession.MarketDataStream.OnKLine(func(kline types.KLine) { - log.Infof("source exchange %s price: %s volume: %s", - s.Symbol, kline.Close.String(), kline.Volume.String()) s.mu.Lock() s.lastSourceKLine = kline s.mu.Unlock() }) s.tradingSession.MarketDataStream.OnKLine(func(kline types.KLine) { - log.Infof("trading exchange %s price: %s volume: %s", - s.Symbol, kline.Close.String(), kline.Volume.String()) s.mu.Lock() s.lastTradingKLine = kline s.mu.Unlock() @@ -296,8 +292,8 @@ func (s *Strategy) placeOrders(ctx context.Context) { // if the spread is less than 100 ticks (100 pips), skip if spread.Compare(s.tradingMarket.TickSize.MulExp(2)) < 0 { - log.Warnf("spread too small, we can't place orders: spread=%v bid=%v ask=%v", - spread, bestBid.Price, bestAsk.Price) + log.Warnf("spread too small, we can't place orders: spread=%s bid=%s ask=%s", + spread.String(), bestBid.Price.String(), bestAsk.Price.String()) return } @@ -313,15 +309,15 @@ func (s *Strategy) placeOrders(ctx context.Context) { var spread = bestAsk.Price.Sub(bestBid.Price) var spreadPercentage = spread.Div(bestAsk.Price) - log.Infof("spread=%v %s ask=%v bid=%v", - spread, spreadPercentage.Percentage(), - bestAsk.Price, bestBid.Price) + log.Infof("spread=%s %s ask=%s bid=%s", + spread.String(), spreadPercentage.Percentage(), + bestAsk.Price.String(), bestBid.Price.String()) // var spreadPercentage = spread.Float64() / bestBid.Price.Float64() var midPrice = bestAsk.Price.Add(bestBid.Price).Div(Two) var price = midPrice - log.Infof("mid price %v", midPrice) + log.Infof("mid price %s", midPrice.String()) var balances = s.tradingSession.GetAccount().Balances() var quantity = s.tradingMarket.MinQuantity @@ -331,6 +327,11 @@ func (s *Strategy) placeOrders(ctx context.Context) { } else if s.SimulateVolume { s.mu.Lock() if s.lastTradingKLine.Volume.Sign() > 0 && s.lastSourceKLine.Volume.Sign() > 0 { + log.Infof("trading exchange %s price: %s volume: %s", + s.Symbol, s.lastTradingKLine.Close.String(), s.lastTradingKLine.Volume.String()) + log.Infof("source exchange %s price: %s volume: %s", + s.Symbol, s.lastSourceKLine.Close.String(), s.lastSourceKLine.Volume.String()) + volumeDiff := s.lastSourceKLine.Volume.Sub(s.lastTradingKLine.Volume) // change the current quantity only diff is positive if volumeDiff.Sign() > 0 { @@ -360,7 +361,7 @@ func (s *Strategy) placeOrders(ctx context.Context) { s.tradingMarket.MinNotional.Mul(NotionModifier).Div(price)) } - orderForm := []types.SubmitOrder{{ + orderForms := []types.SubmitOrder{{ Symbol: s.Symbol, Side: types.SideTypeBuy, Type: types.OrderTypeLimit, @@ -375,7 +376,7 @@ func (s *Strategy) placeOrders(ctx context.Context) { Price: price, Market: s.tradingMarket, }} - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForm...) + createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) if err != nil { log.WithError(err).Error("order submit error") } From dc2895c4dc8fde1492bceb2b7f370d953b1c2497 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sat, 6 Jan 2024 17:37:13 +0800 Subject: [PATCH 385/422] rename cronExpression to schedule --- config/random.yaml | 2 +- config/rebalance.yaml | 36 +++++------------------------- pkg/strategy/random/strategy.go | 14 ++++++------ pkg/strategy/rebalance/strategy.go | 18 +++++++-------- 4 files changed, 22 insertions(+), 48 deletions(-) diff --git a/config/random.yaml b/config/random.yaml index 269c5d8a8b..290adf12b9 100644 --- a/config/random.yaml +++ b/config/random.yaml @@ -4,7 +4,7 @@ exchangeStrategies: random: symbol: USDCUSDT # https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules - cronExpression: "@every 8h" + schedule: "@every 8h" quantity: 8 onStart: true dryRun: true diff --git a/config/rebalance.yaml b/config/rebalance.yaml index bdcd5f6f56..e98c5a82c5 100644 --- a/config/rebalance.yaml +++ b/config/rebalance.yaml @@ -1,39 +1,13 @@ --- -notifications: - slack: - defaultChannel: "bbgo" - errorChannel: "bbgo-error" - switches: - trade: true - orderUpdate: true - submitOrder: true - -backtest: - startTime: "2022-01-01" - endTime: "2022-10-01" - symbols: - - BTCUSDT - - ETHUSDT - - MAXUSDT - account: - max: - makerFeeRate: 0.075% - takerFeeRate: 0.075% - balances: - BTC: 0.0 - ETH: 0.0 - MAX: 0.0 - USDT: 10000.0 - exchangeStrategies: - on: max rebalance: - cronExpression: "@every 1s" - quoteCurrency: USDT + schedule: "@every 1s" + quoteCurrency: TWD targetWeights: - BTC: 50% - ETH: 25% - USDT: 25% + BTC: 60% + ETH: 30% + TWD: 10% threshold: 1% maxAmount: 1_000 # max amount to buy or sell per order orderType: LIMIT_MAKER # LIMIT, LIMIT_MAKER or MARKET diff --git a/pkg/strategy/random/strategy.go b/pkg/strategy/random/strategy.go index a9ccd98882..686be91126 100644 --- a/pkg/strategy/random/strategy.go +++ b/pkg/strategy/random/strategy.go @@ -28,10 +28,10 @@ type Strategy struct { Environment *bbgo.Environment Market types.Market - Symbol string `json:"symbol"` - CronExpression string `json:"cronExpression"` - OnStart bool `json:"onStart"` - DryRun bool `json:"dryRun"` + Symbol string `json:"symbol"` + Schedule string `json:"schedule"` + OnStart bool `json:"onStart"` + DryRun bool `json:"dryRun"` bbgo.QuantityOrAmount cron *cron.Cron @@ -55,8 +55,8 @@ func (s *Strategy) InstanceID() string { } func (s *Strategy) Validate() error { - if s.CronExpression == "" { - return fmt.Errorf("cronExpression is required") + if s.Schedule == "" { + return fmt.Errorf("schedule is required") } if err := s.QuantityOrAmount.Validate(); err != nil { @@ -83,7 +83,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) s.cron = cron.New() - s.cron.AddFunc(s.CronExpression, s.placeOrder) + s.cron.AddFunc(s.Schedule, s.placeOrder) s.cron.Start() return nil diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index 33ea2ef8be..0b1a0909a2 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -27,14 +27,14 @@ type Strategy struct { Environment *bbgo.Environment - CronExpression string `json:"cronExpression"` - QuoteCurrency string `json:"quoteCurrency"` - TargetWeights types.ValueMap `json:"targetWeights"` - Threshold fixedpoint.Value `json:"threshold"` - MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order - OrderType types.OrderType `json:"orderType"` - DryRun bool `json:"dryRun"` - OnStart bool `json:"onStart"` // rebalance on start + Schedule string `json:"schedule"` + QuoteCurrency string `json:"quoteCurrency"` + TargetWeights types.ValueMap `json:"targetWeights"` + Threshold fixedpoint.Value `json:"threshold"` + MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order + OrderType types.OrderType `json:"orderType"` + DryRun bool `json:"dryRun"` + OnStart bool `json:"onStart"` // rebalance on start symbols []string markets map[string]types.Market @@ -130,7 +130,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. }) s.cron = cron.New() - s.cron.AddFunc(s.CronExpression, func() { + s.cron.AddFunc(s.Schedule, func() { s.rebalance(ctx) }) s.cron.Start() From 36aadf74a11d9056fc4c9b927efe9a4fe2698856 Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sat, 6 Jan 2024 21:50:43 +0800 Subject: [PATCH 386/422] xgap: check balance before placing orders --- pkg/strategy/xgap/strategy.go | 89 ++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/pkg/strategy/xgap/strategy.go b/pkg/strategy/xgap/strategy.go index a72aaf1d3d..7797182726 100644 --- a/pkg/strategy/xgap/strategy.go +++ b/pkg/strategy/xgap/strategy.go @@ -22,7 +22,7 @@ const ID = "xgap" var log = logrus.WithField("strategy", ID) var StepPercentageGap = fixedpoint.NewFromFloat(0.05) -var NotionModifier = fixedpoint.NewFromFloat(1.01) + var Two = fixedpoint.NewFromInt(2) func init() { @@ -68,6 +68,7 @@ type Strategy struct { TradingExchange string `json:"tradingExchange"` MinSpread fixedpoint.Value `json:"minSpread"` Quantity fixedpoint.Value `json:"quantity"` + DryRun bool `json:"dryRun"` DailyFeeBudgets map[string]fixedpoint.Value `json:"dailyFeeBudgets,omitempty"` DailyMaxVolume fixedpoint.Value `json:"dailyMaxVolume,omitempty"` @@ -320,10 +321,35 @@ func (s *Strategy) placeOrders(ctx context.Context) { log.Infof("mid price %s", midPrice.String()) var balances = s.tradingSession.GetAccount().Balances() - var quantity = s.tradingMarket.MinQuantity + + baseBalance, ok := balances[s.tradingMarket.BaseCurrency] + if !ok { + log.Errorf("base balance %s not found", s.tradingMarket.BaseCurrency) + return + } + quoteBalance, ok := balances[s.tradingMarket.QuoteCurrency] + if !ok { + log.Errorf("quote balance %s not found", s.tradingMarket.QuoteCurrency) + return + } + + minQuantity := s.tradingMarket.AdjustQuantityByMinNotional(s.tradingMarket.MinQuantity, price) + + if baseBalance.Available.Compare(minQuantity) < 0 { + log.Infof("base balance: %s is not enough, skip", baseBalance.Available.String()) + return + } + + if quoteBalance.Available.Div(price).Compare(minQuantity) < 0 { + log.Infof("quote balance: %s is not enough, skip", quoteBalance.Available.String()) + return + } + + maxQuantity := fixedpoint.Min(baseBalance.Available, quoteBalance.Available.Div(price)) + quantity := minQuantity if s.Quantity.Sign() > 0 { - quantity = fixedpoint.Min(s.Quantity, s.tradingMarket.MinQuantity) + quantity = fixedpoint.Max(s.Quantity, quantity) } else if s.SimulateVolume { s.mu.Lock() if s.lastTradingKLine.Volume.Sign() > 0 && s.lastSourceKLine.Volume.Sign() > 0 { @@ -337,15 +363,6 @@ func (s *Strategy) placeOrders(ctx context.Context) { if volumeDiff.Sign() > 0 { quantity = volumeDiff } - - if baseBalance, ok := balances[s.tradingMarket.BaseCurrency]; ok { - quantity = fixedpoint.Min(quantity, baseBalance.Available) - } - - if quoteBalance, ok := balances[s.tradingMarket.QuoteCurrency]; ok { - maxQuantity := quoteBalance.Available.Div(price) - quantity = fixedpoint.Min(quantity, maxQuantity) - } } s.mu.Unlock() } else { @@ -354,33 +371,37 @@ func (s *Strategy) placeOrders(ctx context.Context) { quantity = quantity.Mul(fixedpoint.NewFromFloat(jitter)) } - var quoteAmount = price.Mul(quantity) - if quoteAmount.Compare(s.tradingMarket.MinNotional) <= 0 { - quantity = fixedpoint.Max( - s.tradingMarket.MinQuantity, - s.tradingMarket.MinNotional.Mul(NotionModifier).Div(price)) + quantity = fixedpoint.Min(quantity, maxQuantity) + + orderForms := []types.SubmitOrder{ + { + Symbol: s.Symbol, + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: quantity, + Price: price, + Market: s.tradingMarket, + }, + { + Symbol: s.Symbol, + Side: types.SideTypeSell, + Type: types.OrderTypeLimit, + Quantity: quantity, + Price: price, + Market: s.tradingMarket, + }, + } + log.Infof("order forms: %+v", orderForms) + + if s.DryRun { + log.Infof("dry run, skip") + return } - orderForms := []types.SubmitOrder{{ - Symbol: s.Symbol, - Side: types.SideTypeBuy, - Type: types.OrderTypeLimit, - Quantity: quantity, - Price: price, - Market: s.tradingMarket, - }, { - Symbol: s.Symbol, - Side: types.SideTypeSell, - Type: types.OrderTypeLimit, - Quantity: quantity, - Price: price, - Market: s.tradingMarket, - }} - createdOrders, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) + _, err := s.OrderExecutor.SubmitOrders(ctx, orderForms...) if err != nil { log.WithError(err).Error("order submit error") } - log.Infof("created orders: %+v", createdOrders) time.Sleep(time.Second) } From 9c108380e8a7223d1a2e87666859bec84fc4bdfe Mon Sep 17 00:00:00 2001 From: narumi <4680567+narumiruna@users.noreply.github.com> Date: Sun, 7 Jan 2024 18:56:57 +0800 Subject: [PATCH 387/422] xgap: print currency --- pkg/strategy/xgap/strategy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/xgap/strategy.go b/pkg/strategy/xgap/strategy.go index 7797182726..232301bd03 100644 --- a/pkg/strategy/xgap/strategy.go +++ b/pkg/strategy/xgap/strategy.go @@ -336,12 +336,12 @@ func (s *Strategy) placeOrders(ctx context.Context) { minQuantity := s.tradingMarket.AdjustQuantityByMinNotional(s.tradingMarket.MinQuantity, price) if baseBalance.Available.Compare(minQuantity) < 0 { - log.Infof("base balance: %s is not enough, skip", baseBalance.Available.String()) + log.Infof("base balance: %s %s is not enough, skip", baseBalance.Available.String(), s.tradingMarket.BaseCurrency) return } if quoteBalance.Available.Div(price).Compare(minQuantity) < 0 { - log.Infof("quote balance: %s is not enough, skip", quoteBalance.Available.String()) + log.Infof("quote balance: %s %s is not enough, skip", quoteBalance.Available.String(), s.tradingMarket.QuoteCurrency) return } From ad8ea8617386d5062951b4cf6723ba05d54dfef7 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 7 Jan 2024 19:08:54 +0800 Subject: [PATCH 388/422] change max borrowable query from error to warn --- pkg/bbgo/order_executor_general.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bbgo/order_executor_general.go b/pkg/bbgo/order_executor_general.go index 0268294668..652f1bb4da 100644 --- a/pkg/bbgo/order_executor_general.go +++ b/pkg/bbgo/order_executor_general.go @@ -125,7 +125,7 @@ func (e *GeneralOrderExecutor) updateMarginAssetMaxBorrowable( ) { maxBorrowable, err := marginService.QueryMarginAssetMaxBorrowable(ctx, market.BaseCurrency) if err != nil { - log.WithError(err).Errorf("can not query margin base asset %s max borrowable", market.BaseCurrency) + log.WithError(err).Warnf("can not query margin base asset %s max borrowable", market.BaseCurrency) } else { log.Infof("updating margin base asset %s max borrowable amount: %f", market.BaseCurrency, maxBorrowable.Float64()) e.marginBaseMaxBorrowable = maxBorrowable @@ -133,7 +133,7 @@ func (e *GeneralOrderExecutor) updateMarginAssetMaxBorrowable( maxBorrowable, err = marginService.QueryMarginAssetMaxBorrowable(ctx, market.QuoteCurrency) if err != nil { - log.WithError(err).Errorf("can not query margin quote asset %s max borrowable", market.QuoteCurrency) + log.WithError(err).Warnf("can not query margin quote asset %s max borrowable", market.QuoteCurrency) } else { log.Infof("updating margin quote asset %s max borrowable amount: %f", market.QuoteCurrency, maxBorrowable.Float64()) e.marginQuoteMaxBorrowable = maxBorrowable From 0b906606fe15c27a440656cca16cc42600b8cdb3 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 2 Jan 2024 23:18:57 +0800 Subject: [PATCH 389/422] pkg/exchange: refactor book and kline --- pkg/exchange/okex/parse.go | 477 ++++++++++----------- pkg/exchange/okex/parse_test.go | 577 ++++++++++++++++++++++++++ pkg/exchange/okex/stream.go | 43 +- pkg/exchange/okex/stream_callbacks.go | 10 +- pkg/exchange/okex/stream_test.go | 17 + 5 files changed, 857 insertions(+), 267 deletions(-) create mode 100644 pkg/exchange/okex/parse_test.go diff --git a/pkg/exchange/okex/parse.go b/pkg/exchange/okex/parse.go index 93a6b0c938..cd2cbffe18 100644 --- a/pkg/exchange/okex/parse.go +++ b/pkg/exchange/okex/parse.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "time" @@ -15,288 +14,326 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func parseWebSocketEvent(str []byte) (interface{}, error) { - v, err := fastjson.ParseBytes(str) +type Channel string + +const ( + ChannelBooks Channel = "books" + ChannelBook5 Channel = "book5" + ChannelCandlePrefix Channel = "candle" + ChannelAccount Channel = "account" + ChannelOrders Channel = "orders" +) + +type ActionType string + +const ( + ActionTypeSnapshot ActionType = "snapshot" + ActionTypeUpdate ActionType = "update" +) + +func parseWebSocketEvent(in []byte) (interface{}, error) { + v, err := fastjson.ParseBytes(in) if err != nil { return nil, err } - if v.Exists("event") { - return parseEvent(v) + var event WebSocketEvent + err = json.Unmarshal(in, &event) + if err != nil { + return nil, err + } + if event.Event != "" { + // TODO: remove fastjson + return event, nil } - if v.Exists("data") { - return parseData(v) + switch event.Arg.Channel { + case ChannelAccount: + // TODO: remove fastjson + return parseAccount(v) + + case ChannelBooks, ChannelBook5: + var bookEvent BookEvent + err = json.Unmarshal(event.Data, &bookEvent.Data) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into BookEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + instId := event.Arg.InstId + bookEvent.InstrumentID = instId + bookEvent.Symbol = toGlobalSymbol(instId) + bookEvent.channel = event.Arg.Channel + bookEvent.Action = event.ActionType + return &bookEvent, nil + + case ChannelOrders: + // TODO: remove fastjson + return parseOrder(v) + + default: + if strings.HasPrefix(string(event.Arg.Channel), string(ChannelCandlePrefix)) { + // TODO: Support kline subscription. The kline requires another URL to subscribe, which is why we cannot + // support it at this time. + var kLineEvt KLineEvent + err = json.Unmarshal(event.Data, &kLineEvt.Events) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into KLineEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + kLineEvt.Channel = event.Arg.Channel + kLineEvt.InstrumentID = event.Arg.InstId + kLineEvt.Symbol = toGlobalSymbol(event.Arg.InstId) + kLineEvt.Interval = strings.ToLower(strings.TrimPrefix(string(event.Arg.Channel), string(ChannelCandlePrefix))) + return &kLineEvt, nil + } } return nil, nil } type WebSocketEvent struct { - Event string `json:"event"` - Code string `json:"code,omitempty"` - Message string `json:"msg,omitempty"` - Arg interface{} `json:"arg,omitempty"` -} - -func parseEvent(v *fastjson.Value) (*WebSocketEvent, error) { - // event could be "subscribe", "unsubscribe" or "error" - event := string(v.GetStringBytes("event")) - code := string(v.GetStringBytes("code")) - message := string(v.GetStringBytes("msg")) - arg := v.GetObject("arg") - return &WebSocketEvent{ - Event: event, - Code: code, - Message: message, - Arg: arg, - }, nil + Event string `json:"event"` + Code string `json:"code,omitempty"` + Message string `json:"msg,omitempty"` + Arg struct { + Channel Channel `json:"channel"` + InstId string `json:"instId"` + } `json:"arg,omitempty"` + Data json.RawMessage `json:"data"` + ActionType ActionType `json:"action"` } type BookEvent struct { - InstrumentID string - Symbol string - Action string - Bids []BookEntry - Asks []BookEntry - MillisecondTimestamp int64 - Checksum int - channel string + InstrumentID string + Symbol string + Action ActionType + channel Channel + + Data []struct { + Bids PriceVolumeOrderSlice `json:"bids"` + Asks PriceVolumeOrderSlice `json:"asks"` + MillisecondTimestamp types.MillisecondTimestamp `json:"ts"` + Checksum int `json:"checksum"` + } } -func (data *BookEvent) BookTicker() types.BookTicker { +func (event *BookEvent) BookTicker() types.BookTicker { ticker := types.BookTicker{ - Symbol: data.Symbol, + Symbol: event.Symbol, } - if len(data.Bids) > 0 { - ticker.Buy = data.Bids[0].Price - ticker.BuySize = data.Bids[0].Volume - } + if len(event.Data) > 0 { + if len(event.Data[0].Bids) > 0 { + ticker.Buy = event.Data[0].Bids[0].Price + ticker.BuySize = event.Data[0].Bids[0].Volume + } - if len(data.Asks) > 0 { - ticker.Sell = data.Asks[0].Price - ticker.SellSize = data.Asks[0].Volume + if len(event.Data[0].Asks) > 0 { + ticker.Sell = event.Data[0].Asks[0].Price + ticker.SellSize = event.Data[0].Asks[0].Volume + } } return ticker } -func (data *BookEvent) Book() types.SliceOrderBook { +func (event *BookEvent) Book() types.SliceOrderBook { book := types.SliceOrderBook{ - Symbol: data.Symbol, - Time: types.NewMillisecondTimestampFromInt(data.MillisecondTimestamp).Time(), + Symbol: event.Symbol, } - for _, bid := range data.Bids { - book.Bids = append(book.Bids, types.PriceVolume{Price: bid.Price, Volume: bid.Volume}) + if len(event.Data) > 0 { + book.Time = event.Data[0].MillisecondTimestamp.Time() } - for _, ask := range data.Asks { - book.Asks = append(book.Asks, types.PriceVolume{Price: ask.Price, Volume: ask.Volume}) + for _, data := range event.Data { + for _, bid := range data.Bids { + book.Bids = append(book.Bids, types.PriceVolume{Price: bid.Price, Volume: bid.Volume}) + } + + for _, ask := range data.Asks { + book.Asks = append(book.Asks, types.PriceVolume{Price: ask.Price, Volume: ask.Volume}) + } } return book } -type BookEntry struct { - Price fixedpoint.Value - Volume fixedpoint.Value +type PriceVolumeOrder struct { + types.PriceVolume + // NumLiquidated is part of a deprecated feature and it is always "0" NumLiquidated int - NumOrders int + // NumOrders is the number of orders at the price. + NumOrders int } -func parseBookEntry(v *fastjson.Value) (*BookEntry, error) { - arr, err := v.Array() - if err != nil { - return nil, err - } - - if len(arr) < 4 { - return nil, fmt.Errorf("unexpected book entry size: %d", len(arr)) - } +type PriceVolumeOrderSlice []PriceVolumeOrder - price := fixedpoint.Must(fixedpoint.NewFromString(string(arr[0].GetStringBytes()))) - volume := fixedpoint.Must(fixedpoint.NewFromString(string(arr[1].GetStringBytes()))) - numLiquidated, err := strconv.Atoi(string(arr[2].GetStringBytes())) +func (slice *PriceVolumeOrderSlice) UnmarshalJSON(b []byte) error { + s, err := ParsePriceVolumeOrderSliceJSON(b) if err != nil { - return nil, err + return err } - numOrders, err := strconv.Atoi(string(arr[3].GetStringBytes())) - if err != nil { - return nil, err - } - - return &BookEntry{ - Price: price, - Volume: volume, - NumLiquidated: numLiquidated, - NumOrders: numOrders, - }, nil + *slice = s + return nil } -func parseBookData(v *fastjson.Value) (*BookEvent, error) { - instrumentId := string(v.GetStringBytes("arg", "instId")) - data := v.GetArray("data") - if len(data) == 0 { - return nil, errors.New("empty data payload") - } - - // "snapshot" or "update" - action := string(v.GetStringBytes("action")) +// ParsePriceVolumeOrderSliceJSON tries to parse a 2 dimensional string array into a PriceVolumeOrderSlice +// +// [["8476.98", "415", "0", "13"], ["8477", "7", "0", "2"], ... ] +func ParsePriceVolumeOrderSliceJSON(b []byte) (slice PriceVolumeOrderSlice, err error) { + var as [][]fixedpoint.Value - millisecondTimestamp, err := strconv.ParseInt(string(data[0].GetStringBytes("ts")), 10, 64) + err = json.Unmarshal(b, &as) if err != nil { - return nil, err + return slice, fmt.Errorf("failed to unmarshal price volume order slice: %w", err) } - checksum := data[0].GetInt("checksum") - - var asks []BookEntry - var bids []BookEntry - - for _, v := range data[0].GetArray("asks") { - entry, err := parseBookEntry(v) - if err != nil { - return nil, err - } - asks = append(asks, *entry) - } + for _, a := range as { + var pv PriceVolumeOrder + pv.Price = a[0] + pv.Volume = a[1] + pv.NumLiquidated = a[2].Int() + pv.NumOrders = a[3].Int() - for _, v := range data[0].GetArray("bids") { - entry, err := parseBookEntry(v) - if err != nil { - return nil, err - } - bids = append(bids, *entry) + slice = append(slice, pv) } - return &BookEvent{ - InstrumentID: instrumentId, - Symbol: toGlobalSymbol(instrumentId), - Action: action, - Bids: bids, - Asks: asks, - Checksum: checksum, - MillisecondTimestamp: millisecondTimestamp, - }, nil + return slice, nil } -type Candle struct { - Channel string - InstrumentID string - Symbol string - Interval string - Open fixedpoint.Value - High fixedpoint.Value - Low fixedpoint.Value - Close fixedpoint.Value +type KLine struct { + StartTime types.MillisecondTimestamp + OpenPrice fixedpoint.Value + HighestPrice fixedpoint.Value + LowestPrice fixedpoint.Value + ClosePrice fixedpoint.Value + // Volume trading volume, with a unit of contract.cccccbcvefkeibbhtrebbfklrbetukhrgjgkiilufbde - // Trading volume, with a unit of contact. // If it is a derivatives contract, the value is the number of contracts. - // If it is SPOT/MARGIN, the value is the amount of trading currency. + // If it is SPOT/MARGIN, the value is the quantity in base currency. Volume fixedpoint.Value - - // Trading volume, with a unit of currency. - // If it is a derivatives contract, the value is the number of settlement currency. - // If it is SPOT/MARGIN, the value is the number of quote currency. - VolumeInCurrency fixedpoint.Value - - MillisecondTimestamp int64 - - StartTime time.Time + // VolumeCcy trading volume, with a unit of currency. + // If it is a derivatives contract, the value is the number of base currency. + // If it is SPOT/MARGIN, the value is the quantity in quote currency. + VolumeCcy fixedpoint.Value + // VolumeCcyQuote Trading volume, the value is the quantity in quote currency + // e.g. The unit is USDT for BTC-USDT and BTC-USDT-SWAP; + // The unit is USD for BTC-USD-SWAP + VolumeCcyQuote fixedpoint.Value + // The state of candlesticks. + // 0 represents that it is uncompleted, 1 represents that it is completed. + Confirm fixedpoint.Value } -func (c *Candle) KLine() types.KLine { - interval := types.Interval(c.Interval) - endTime := c.StartTime.Add(interval.Duration() - 1*time.Millisecond) +func (k KLine) ToGlobal(interval types.Interval, symbol string) types.KLine { + startTime := k.StartTime.Time() + return types.KLine{ - Exchange: types.ExchangeOKEx, - Interval: interval, - Open: c.Open, - High: c.High, - Low: c.Low, - Close: c.Close, - Volume: c.Volume, - QuoteVolume: c.VolumeInCurrency, - StartTime: types.Time(c.StartTime), - EndTime: types.Time(endTime), + Exchange: types.ExchangeOKEx, + Symbol: symbol, + StartTime: types.Time(startTime), + EndTime: types.Time(startTime.Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: k.OpenPrice, + Close: k.ClosePrice, + High: k.HighestPrice, + Low: k.LowestPrice, + Volume: k.Volume, + QuoteVolume: k.VolumeCcy, // not supported + TakerBuyBaseAssetVolume: fixedpoint.Zero, // not supported + TakerBuyQuoteAssetVolume: fixedpoint.Zero, // not supported + LastTradeID: 0, // not supported + NumberOfTrades: 0, // not supported + Closed: !k.Confirm.IsZero(), } } -func parseCandle(channel string, v *fastjson.Value) (*Candle, error) { - instrumentID := string(v.GetStringBytes("arg", "instId")) - data, err := v.Get("data").Array() - if err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, errors.New("candle data is empty") - } - - arr, err := data[0].Array() - if err != nil { - return nil, err - } +type KLineSlice []KLine - if len(arr) < 7 { - return nil, fmt.Errorf("unexpected candle data length: %d", len(arr)) +func (m *KLineSlice) UnmarshalJSON(b []byte) error { + if m == nil { + return errors.New("nil pointer of kline slice") } - - interval := strings.ToLower(strings.TrimPrefix(channel, "candle")) - - timestamp, err := strconv.ParseInt(string(arr[0].GetStringBytes()), 10, 64) + s, err := parseKLineSliceJSON(b) if err != nil { - return nil, err + return err } - open, err := fixedpoint.NewFromString(string(arr[1].GetStringBytes())) - if err != nil { - return nil, err - } + *m = s + return nil +} - high, err := fixedpoint.NewFromString(string(arr[2].GetStringBytes())) +// parseKLineSliceJSON tries to parse a 2 dimensional string array into a KLineSlice +// +// [ +// [ +// "1597026383085", +// "8533.02", +// "8553.74", +// "8527.17", +// "8548.26", +// "45247", +// "529.5858061", +// "5529.5858061", +// "0" +// ] +// ] +func parseKLineSliceJSON(in []byte) (slice KLineSlice, err error) { + var rawKLines [][]json.RawMessage + + err = json.Unmarshal(in, &rawKLines) if err != nil { - return nil, err + return slice, err } - low, err := fixedpoint.NewFromString(string(arr[3].GetStringBytes())) - if err != nil { - return nil, err - } + for _, raw := range rawKLines { + if len(raw) != 9 { + return nil, fmt.Errorf("unexpected kline length: %d, data: %q", len(raw), raw) + } + var kline KLine + if err = json.Unmarshal(raw[0], &kline.StartTime); err != nil { + return nil, fmt.Errorf("failed to unmarshal into timestamp: %q", raw[0]) + } + if err = json.Unmarshal(raw[1], &kline.OpenPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into open price: %q", raw[1]) + } + if err = json.Unmarshal(raw[2], &kline.HighestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into highest price: %q", raw[2]) + } + if err = json.Unmarshal(raw[3], &kline.LowestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into lowest price: %q", raw[3]) + } + if err = json.Unmarshal(raw[4], &kline.ClosePrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into close price: %q", raw[4]) + } + if err = json.Unmarshal(raw[5], &kline.Volume); err != nil { + return nil, fmt.Errorf("failed to unmarshal into volume: %q", raw[5]) + } + if err = json.Unmarshal(raw[6], &kline.VolumeCcy); err != nil { + return nil, fmt.Errorf("failed to unmarshal into volume currency: %q", raw[6]) + } + if err = json.Unmarshal(raw[7], &kline.VolumeCcyQuote); err != nil { + return nil, fmt.Errorf("failed to unmarshal into trading currency quote: %q", raw[7]) + } + if err = json.Unmarshal(raw[8], &kline.Confirm); err != nil { + return nil, fmt.Errorf("failed to unmarshal into confirm: %q", raw[8]) + } - cls, err := fixedpoint.NewFromString(string(arr[4].GetStringBytes())) - if err != nil { - return nil, err + slice = append(slice, kline) } - vol, err := fixedpoint.NewFromString(string(arr[5].GetStringBytes())) - if err != nil { - return nil, err - } + return slice, nil +} - volCurrency, err := fixedpoint.NewFromString(string(arr[6].GetStringBytes())) - if err != nil { - return nil, err - } +type KLineEvent struct { + Events KLineSlice - candleTime := time.Unix(0, timestamp*int64(time.Millisecond)) - return &Candle{ - Channel: channel, - InstrumentID: instrumentID, - Symbol: toGlobalSymbol(instrumentID), - Interval: interval, - Open: open, - High: high, - Low: low, - Close: cls, - Volume: vol, - VolumeInCurrency: volCurrency, - MillisecondTimestamp: timestamp, - StartTime: candleTime, - }, nil + InstrumentID string + Symbol string + Interval string + Channel Channel } func parseAccount(v *fastjson.Value) (*okexapi.Account, error) { @@ -326,31 +363,3 @@ func parseOrder(v *fastjson.Value) ([]okexapi.OrderDetails, error) { return orderDetails, nil } - -func parseData(v *fastjson.Value) (interface{}, error) { - - channel := string(v.GetStringBytes("arg", "channel")) - - switch channel { - case "books5": - data, err := parseBookData(v) - data.channel = channel - return data, err - case "books": - data, err := parseBookData(v) - data.channel = channel - return data, err - case "account": - return parseAccount(v) - case "orders": - return parseOrder(v) - default: - if strings.HasPrefix(channel, "candle") { - data, err := parseCandle(channel, v) - return data, err - } - - } - - return nil, nil -} diff --git a/pkg/exchange/okex/parse_test.go b/pkg/exchange/okex/parse_test.go new file mode 100644 index 0000000000..42c41cf2d8 --- /dev/null +++ b/pkg/exchange/okex/parse_test.go @@ -0,0 +1,577 @@ +package okex + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestParsePriceVolumeOrderSliceJSON(t *testing.T) { + t.Run("snapshot", func(t *testing.T) { + in := ` +{ + "arg": { + "channel": "books", + "instId": "BTC-USDT" + }, + "action": "snapshot", + "data": [ + { + "asks": [ + ["8476.98", "415", "0", "13"], + ["8477", "7", "0", "2"] + ], + "bids": [ + ["8476", "256", "0", "12"] + ], + "ts": "1597026383085", + "checksum": -855196043, + "prevSeqId": -1, + "seqId": 123456 + } + ] +} +` + + asks := PriceVolumeOrderSlice{ + { + PriceVolume: types.PriceVolume{ + Price: fixedpoint.NewFromFloat(8476.98), + Volume: fixedpoint.NewFromFloat(415), + }, + NumLiquidated: fixedpoint.Zero.Int(), + NumOrders: fixedpoint.NewFromFloat(13).Int(), + }, + { + PriceVolume: types.PriceVolume{ + Price: fixedpoint.NewFromFloat(8477), + Volume: fixedpoint.NewFromFloat(7), + }, + NumLiquidated: fixedpoint.Zero.Int(), + NumOrders: fixedpoint.NewFromFloat(2).Int(), + }, + } + bids := PriceVolumeOrderSlice{ + { + PriceVolume: types.PriceVolume{ + Price: fixedpoint.NewFromFloat(8476), + Volume: fixedpoint.NewFromFloat(256), + }, + NumLiquidated: fixedpoint.Zero.Int(), + NumOrders: fixedpoint.NewFromFloat(12).Int(), + }, + } + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*BookEvent) + assert.True(t, ok) + assert.Equal(t, "BTCUSDT", event.Symbol) + assert.Equal(t, ChannelBooks, event.channel) + assert.Equal(t, ActionTypeSnapshot, event.Action) + assert.Len(t, event.Data, 1) + assert.Len(t, event.Data[0].Asks, 2) + assert.Equal(t, asks, event.Data[0].Asks) + assert.Len(t, event.Data[0].Bids, 1) + assert.Equal(t, bids, event.Data[0].Bids) + }) + + t.Run("unexpected asks", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "books", + "instId": "BTC-USDT" + }, + "action": "snapshot", + "data": [ + { + "asks": [ + ["XYZ", "415", "0", "13"] + ], + "bids": [ + ["8476", "256", "0", "12"] + ], + "ts": "1597026383085", + "checksum": -855196043, + "prevSeqId": -1, + "seqId": 123456 + } + ] +} +` + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "price volume order") + }) +} + +func TestBookEvent_BookTicker(t *testing.T) { + in := ` +{ + "arg": { + "channel": "books", + "instId": "BTC-USDT" + }, + "action": "snapshot", + "data": [ + { + "asks": [ + ["8476.98", "415", "0", "13"], + ["8477", "7", "0", "2"] + ], + "bids": [ + ["8476", "256", "0", "12"] + ], + "ts": "1597026383085", + "checksum": -855196043, + "prevSeqId": -1, + "seqId": 123456 + } + ] +} +` + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*BookEvent) + assert.True(t, ok) + + ticker := event.BookTicker() + assert.Equal(t, types.BookTicker{ + Symbol: "BTCUSDT", + Buy: fixedpoint.NewFromFloat(8476), + BuySize: fixedpoint.NewFromFloat(256), + Sell: fixedpoint.NewFromFloat(8476.98), + SellSize: fixedpoint.NewFromFloat(415), + }, ticker) +} + +func TestBookEvent_Book(t *testing.T) { + in := ` +{ + "arg": { + "channel": "books", + "instId": "BTC-USDT" + }, + "action": "snapshot", + "data": [ + { + "asks": [ + ["8476.98", "415", "0", "13"], + ["8477", "7", "0", "2"] + ], + "bids": [ + ["8476", "256", "0", "12"] + ], + "ts": "1597026383085", + "checksum": -855196043, + "prevSeqId": -1, + "seqId": 123456 + } + ] +} +` + bids := types.PriceVolumeSlice{ + { + Price: fixedpoint.NewFromFloat(8476), + Volume: fixedpoint.NewFromFloat(256), + }, + } + asks := types.PriceVolumeSlice{ + { + Price: fixedpoint.NewFromFloat(8476.98), + Volume: fixedpoint.NewFromFloat(415), + }, + { + Price: fixedpoint.NewFromFloat(8477), + Volume: fixedpoint.NewFromFloat(7), + }, + } + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*BookEvent) + assert.True(t, ok) + + book := event.Book() + assert.Equal(t, types.SliceOrderBook{ + Symbol: "BTCUSDT", + Time: types.NewMillisecondTimestampFromInt(1597026383085).Time(), + Bids: bids, + Asks: asks, + }, book) +} + +func Test_parseKLineSliceJSON(t *testing.T) { + t.Run("snapshot", func(t *testing.T) { + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + exp := &KLineEvent{ + Events: KLineSlice{ + { + StartTime: types.NewMillisecondTimestampFromInt(1597026383085), + OpenPrice: fixedpoint.NewFromFloat(8533), + HighestPrice: fixedpoint.NewFromFloat(8553.74), + LowestPrice: fixedpoint.NewFromFloat(8527.17), + ClosePrice: fixedpoint.NewFromFloat(8548.26), + Volume: fixedpoint.NewFromFloat(45247), + VolumeCcy: fixedpoint.NewFromFloat(529.5858061), + VolumeCcyQuote: fixedpoint.NewFromFloat(529.5858061), + Confirm: fixedpoint.Zero, + }, + }, + InstrumentID: "BTC-USDT", + Symbol: "BTCUSDT", + Interval: "1d", + Channel: "candle1D", + } + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*KLineEvent) + assert.True(t, ok) + assert.Len(t, event.Events, 1) + assert.Equal(t, exp, event) + }) + + t.Run("failed to convert timestamp", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "x", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "timestamp") + }) + + t.Run("failed to convert open price", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "x", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "open price") + }) + + t.Run("failed to convert highest price", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "x", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "highest price") + }) + t.Run("failed to convert lowest price", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "x", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "lowest price") + }) + t.Run("failed to convert close price", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "x", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "close price") + }) + t.Run("failed to convert volume", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "x", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "volume") + }) + t.Run("failed to convert volume currency", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "x", + "529.5858061", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "volume currency") + }) + t.Run("failed to convert trading currency quote ", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "x", + "0" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "trading currency") + }) + t.Run("failed to convert confirm", func(t *testing.T) { + t.Skip("this will cause panic, so i skip it") + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "g" + ] + ] +} +` + + _, err := parseWebSocketEvent([]byte(in)) + assert.ErrorContains(t, err, "confirm") + }) + +} + +func TestKLine_ToGlobal(t *testing.T) { + t.Run("snapshot", func(t *testing.T) { + in := ` +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "529.5858061", + "0" + ] + ] +} +` + exp := &KLineEvent{ + Events: KLineSlice{ + { + StartTime: types.NewMillisecondTimestampFromInt(1597026383085), + OpenPrice: fixedpoint.NewFromFloat(8533), + HighestPrice: fixedpoint.NewFromFloat(8553.74), + LowestPrice: fixedpoint.NewFromFloat(8527.17), + ClosePrice: fixedpoint.NewFromFloat(8548.26), + Volume: fixedpoint.NewFromFloat(45247), + VolumeCcy: fixedpoint.NewFromFloat(529.5858061), + VolumeCcyQuote: fixedpoint.NewFromFloat(529.5858061), + Confirm: fixedpoint.Zero, + }, + }, + InstrumentID: "BTC-USDT", + Symbol: "BTCUSDT", + Interval: "1d", + Channel: "candle1D", + } + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*KLineEvent) + assert.True(t, ok) + + assert.Equal(t, types.KLine{ + Exchange: types.ExchangeOKEx, + Symbol: "BTCUSDT", + StartTime: types.Time(types.NewMillisecondTimestampFromInt(1597026383085)), + EndTime: types.Time(types.NewMillisecondTimestampFromInt(1597026383085).Time().Add(types.Interval(exp.Interval).Duration() - time.Millisecond)), + Interval: types.Interval(exp.Interval), + Open: exp.Events[0].OpenPrice, + Close: exp.Events[0].ClosePrice, + High: exp.Events[0].HighestPrice, + Low: exp.Events[0].LowestPrice, + Volume: exp.Events[0].Volume, + QuoteVolume: exp.Events[0].VolumeCcy, + TakerBuyBaseAssetVolume: fixedpoint.Zero, + TakerBuyQuoteAssetVolume: fixedpoint.Zero, + LastTradeID: 0, + NumberOfTrades: 0, + Closed: false, + }, event.Events[0].ToGlobal(types.Interval(event.Interval), event.Symbol)) + }) + +} diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index 7d7c7a77eb..9d45ae3b27 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -28,32 +28,24 @@ type Stream struct { client *okexapi.RestClient // public callbacks - candleEventCallbacks []func(candle Candle) + kLineEventCallbacks []func(candle KLineEvent) bookEventCallbacks []func(book BookEvent) eventCallbacks []func(event WebSocketEvent) accountEventCallbacks []func(account okexapi.Account) orderDetailsEventCallbacks []func(orderDetails []okexapi.OrderDetails) - - lastCandle map[CandleKey]Candle -} - -type CandleKey struct { - InstrumentID string - Channel string } func NewStream(client *okexapi.RestClient) *Stream { stream := &Stream{ client: client, StandardStream: types.NewStandardStream(), - lastCandle: make(map[CandleKey]Candle), } stream.SetParser(parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) stream.SetEndpointCreator(stream.createEndpoint) - stream.OnCandleEvent(stream.handleCandleEvent) + stream.OnKLineEvent(stream.handleKLineEvent) stream.OnBookEvent(stream.handleBookEvent) stream.OnAccountEvent(stream.handleAccountEvent) stream.OnOrderDetailsEvent(stream.handleOrderDetailsEvent) @@ -167,27 +159,22 @@ func (s *Stream) handleAccountEvent(account okexapi.Account) { func (s *Stream) handleBookEvent(data BookEvent) { book := data.Book() switch data.Action { - case "snapshot": + case ActionTypeSnapshot: s.EmitBookSnapshot(book) - case "update": + case ActionTypeUpdate: s.EmitBookUpdate(book) } } -func (s *Stream) handleCandleEvent(candle Candle) { - key := CandleKey{Channel: candle.Channel, InstrumentID: candle.InstrumentID} - kline := candle.KLine() - - // check if we need to close previous kline - lastCandle, ok := s.lastCandle[key] - if ok && candle.StartTime.After(lastCandle.StartTime) { - lastKline := lastCandle.KLine() - lastKline.Closed = true - s.EmitKLineClosed(lastKline) +func (s *Stream) handleKLineEvent(k KLineEvent) { + for _, event := range k.Events { + kline := event.ToGlobal(types.Interval(k.Interval), k.Symbol) + if kline.Closed { + s.EmitKLineClosed(kline) + } else { + s.EmitKLine(kline) + } } - - s.EmitKLine(kline) - s.lastCandle[key] = candle } func (s *Stream) createEndpoint(ctx context.Context) (string, error) { @@ -207,12 +194,12 @@ func (s *Stream) dispatchEvent(e interface{}) { case *BookEvent: // there's "books" for 400 depth and books5 for 5 depth - if et.channel != "books5" { + if et.channel != ChannelBook5 { s.EmitBookEvent(*et) } s.EmitBookTickerUpdate(et.BookTicker()) - case *Candle: - s.EmitCandleEvent(*et) + case *KLineEvent: + s.EmitKLineEvent(*et) case *okexapi.Account: s.EmitAccountEvent(*et) diff --git a/pkg/exchange/okex/stream_callbacks.go b/pkg/exchange/okex/stream_callbacks.go index 6fb6e72313..750614b7c6 100644 --- a/pkg/exchange/okex/stream_callbacks.go +++ b/pkg/exchange/okex/stream_callbacks.go @@ -6,12 +6,12 @@ import ( "github.com/c9s/bbgo/pkg/exchange/okex/okexapi" ) -func (s *Stream) OnCandleEvent(cb func(candle Candle)) { - s.candleEventCallbacks = append(s.candleEventCallbacks, cb) +func (s *Stream) OnKLineEvent(cb func(candle KLineEvent)) { + s.kLineEventCallbacks = append(s.kLineEventCallbacks, cb) } -func (s *Stream) EmitCandleEvent(candle Candle) { - for _, cb := range s.candleEventCallbacks { +func (s *Stream) EmitKLineEvent(candle KLineEvent) { + for _, cb := range s.kLineEventCallbacks { cb(candle) } } @@ -57,7 +57,7 @@ func (s *Stream) EmitOrderDetailsEvent(orderDetails []okexapi.OrderDetails) { } type StreamEventHub interface { - OnCandleEvent(cb func(candle Candle)) + OnKLineEvent(cb func(candle KLineEvent)) OnBookEvent(cb func(book BookEvent)) diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go index 596b4b9780..1cc4e5e5da 100644 --- a/pkg/exchange/okex/stream_test.go +++ b/pkg/exchange/okex/stream_test.go @@ -48,4 +48,21 @@ func TestStream(t *testing.T) { c := make(chan struct{}) <-c }) + t.Run("kline test", func(t *testing.T) { + s.Subscribe(types.KLineChannel, "LTC-USD-200327", types.SubscribeOptions{ + Interval: types.Interval1m, + }) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnKLine(func(kline types.KLine) { + t.Log("got update", kline) + }) + s.OnKLineClosed(func(kline types.KLine) { + t.Log("got closed", kline) + }) + c := make(chan struct{}) + <-c + }) } From 33deaea6e5d9a94462027027332244c9d494ccd2 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 8 Jan 2024 17:46:09 +0800 Subject: [PATCH 390/422] bitget: bitget ignore offline symbols --- .../bitget/bitgetapi/v2/get_symbols_request.go | 14 ++++++++------ pkg/exchange/bitget/convert.go | 3 ++- pkg/exchange/bitget/convert_test.go | 12 ++++++------ pkg/exchange/bitget/exchange.go | 5 +++++ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go index cab2ed5541..180c6d9d8e 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request.go @@ -12,12 +12,14 @@ import ( type SymbolStatus string const ( - // SymbolOffline represent market is suspended, users cannot trade. - SymbolOffline SymbolStatus = "offline" - // SymbolGray represents market is online, but user trading is not available. - SymbolGray SymbolStatus = "gray" - // SymbolOnline trading begins, users can trade. - SymbolOnline SymbolStatus = "online" + // SymbolStatusOffline represent market is suspended, users cannot trade. + SymbolStatusOffline SymbolStatus = "offline" + + // SymbolStatusGray represents market is online, but user trading is not available. + SymbolStatusGray SymbolStatus = "gray" + + // SymbolStatusOnline trading begins, users can trade. + SymbolStatusOnline SymbolStatus = "online" ) type Symbol struct { diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 9e6a65504b..ae9ae4acc6 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -25,9 +25,10 @@ func toGlobalBalance(asset v2.AccountAsset) types.Balance { } func toGlobalMarket(s v2.Symbol) types.Market { - if s.Status != v2.SymbolOnline { + if s.Status != v2.SymbolStatusOnline { log.Warnf("The symbol %s is not online", s.Symbol) } + return types.Market{ Symbol: s.Symbol, LocalSymbol: s.Symbol, diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go index cf3f0ab94a..6f34b50214 100644 --- a/pkg/exchange/bitget/convert_test.go +++ b/pkg/exchange/bitget/convert_test.go @@ -44,7 +44,7 @@ func Test_toGlobalBalance(t *testing.T) { func Test_toGlobalMarket(t *testing.T) { // sample: - //{ + // { // "symbol":"BTCUSDT", // "baseCoin":"BTC", // "quoteCoin":"USDT", @@ -59,7 +59,7 @@ func Test_toGlobalMarket(t *testing.T) { // "minTradeUSDT":"5", // "buyLimitPriceRatio":"0.05", // "sellLimitPriceRatio":"0.05" - //} + // } inst := v2.Symbol{ Symbol: "BTCUSDT", BaseCoin: "BTC", @@ -72,7 +72,7 @@ func Test_toGlobalMarket(t *testing.T) { QuantityPrecision: fixedpoint.NewFromFloat(4), QuotePrecision: fixedpoint.NewFromFloat(6), MinTradeUSDT: fixedpoint.NewFromFloat(5), - Status: v2.SymbolOnline, + Status: v2.SymbolStatusOnline, BuyLimitPriceRatio: fixedpoint.NewFromFloat(0.05), SellLimitPriceRatio: fixedpoint.NewFromFloat(0.05), } @@ -99,7 +99,7 @@ func Test_toGlobalMarket(t *testing.T) { func Test_toGlobalTicker(t *testing.T) { // sample: - //{ + // { // "open":"36465.96", // "symbol":"BTCUSDT", // "high24h":"37040.25", @@ -116,7 +116,7 @@ func Test_toGlobalTicker(t *testing.T) { // "openUtc":"36465.96", // "changeUtc24h":"0.00599", // "change24h":"-0.00426" - //} + // } ticker := v2.Ticker{ Symbol: "BTCUSDT", High24H: fixedpoint.NewFromFloat(24175.65), @@ -540,7 +540,7 @@ func Test_toGlobalTrade(t *testing.T) { // "tradeScope":"taker", // "cTime":"1699020564676", // "uTime":"1699020564687" - //} + // } trade := v2.Trade{ UserId: types.StrInt64(8672173294), Symbol: "APEUSDT", diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 894c5fc4e9..7d4fc67c9f 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -111,6 +111,11 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { markets := types.MarketMap{} for _, s := range symbols { + if s.Status == v2.SymbolStatusOffline { + // ignore offline symbols + continue + } + markets[s.Symbol] = toGlobalMarket(s) } From cfe3b6466c396bc47e6f5c276dc1aecab5f1b1a4 Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 8 Jan 2024 17:47:52 +0800 Subject: [PATCH 391/422] update bitget v2 get_symbols_request_requestgen --- .../v2/get_symbols_request_requestgen.go | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go index 7005830947..b91ad44c98 100644 --- a/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go +++ b/pkg/exchange/bitget/bitgetapi/v2/get_symbols_request_requestgen.go @@ -137,15 +137,29 @@ func (g *GetSymbolsRequest) Do(ctx context.Context) ([]Symbol, error) { } var apiResponse bitgetapi.APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err + + type responseUnmarshaler interface { + Unmarshal(data []byte) error + } + + if unmarshaler, ok := interface{}(&apiResponse).(responseUnmarshaler); ok { + if err := unmarshaler.Unmarshal(response.Body); err != nil { + return nil, err + } + } else { + // The line below checks the content type, however, some API server might not send the correct content type header, + // Hence, this is commented for backward compatibility + // response.IsJSON() + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } } type responseValidator interface { Validate() error } - validator, ok := interface{}(apiResponse).(responseValidator) - if ok { + + if validator, ok := interface{}(&apiResponse).(responseValidator); ok { if err := validator.Validate(); err != nil { return nil, err } From e358da10ddd229abe55b3b7970b18c56062d254d Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 8 Jan 2024 18:13:26 +0800 Subject: [PATCH 392/422] bitget: log symbol status --- pkg/exchange/bitget/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index ae9ae4acc6..54ca48498c 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -26,7 +26,7 @@ func toGlobalBalance(asset v2.AccountAsset) types.Balance { func toGlobalMarket(s v2.Symbol) types.Market { if s.Status != v2.SymbolStatusOnline { - log.Warnf("The symbol %s is not online", s.Symbol) + log.Warnf("The market symbol status %s is not online: %s", s.Symbol, s.Status) } return types.Market{ From 006256a9df5445f49e4f9edd2ed0c4464bccd829 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 27 Dec 2023 16:13:34 +0800 Subject: [PATCH 393/422] FEATURE: add callbacks and shutdown function --- pkg/strategy/dca2/open_position.go | 4 +- pkg/strategy/dca2/state.go | 2 +- pkg/strategy/dca2/strategy.go | 70 +++++++++++++++++++++---- pkg/strategy/dca2/strategy_callbacks.go | 57 ++++++++++++++++++++ 4 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 pkg/strategy/dca2/strategy_callbacks.go diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index 04e886e3d9..66af3477b2 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -108,8 +108,8 @@ func calculateNotionalAndNum(market types.Market, budget fixedpoint.Value, price return fixedpoint.Zero, 0 } -func (s *Strategy) cancelOpenPositionOrders(ctx context.Context) error { - s.logger.Info("[DCA] cancel open position orders") +func (s *Strategy) cancelAllOrders(ctx context.Context) error { + s.logger.Info("[DCA] cancel all orders") e, ok := s.Session.Exchange.(cancelOrdersByGroupIDApi) if ok { cancelledOrders, err := e.CancelOrdersByGroupID(ctx, int64(s.OrderGroupID)) diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 6936079950..3fc626b38a 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -156,7 +156,7 @@ func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { s.logger.Info("[State] OpenPositionOrdersCancelling - start cancelling open-position orders") - if err := s.cancelOpenPositionOrders(ctx); err != nil { + if err := s.cancelAllOrders(ctx); err != nil { s.logger.WithError(err).Error("failed to cancel maker orders") return } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index d1eab7ffd0..1dcffcaa49 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -12,6 +12,7 @@ import ( "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -43,16 +44,32 @@ type Strategy struct { // OrderGroupID is the group ID used for the strategy instance for canceling orders OrderGroupID uint32 `json:"orderGroupID"` + // RecoverWhenStart option is used for recovering dca states + RecoverWhenStart bool `json:"recoverWhenStart"` + + // KeepOrdersWhenShutdown option is used for keeping the grid orders when shutting down bbgo + KeepOrdersWhenShutdown bool `json:"keepOrdersWhenShutdown"` + // log logger *logrus.Entry LogFields logrus.Fields `json:"logFields"` + // PrometheusLabels will be used as the base prometheus labels + PrometheusLabels prometheus.Labels `json:"prometheusLabels"` + // private field mu sync.Mutex takeProfitPrice fixedpoint.Value startTimeOfNextRound time.Time nextStateC chan State state State + + // callbacks + readyCallbacks []func() + positionCallbacks []func(*types.Position) + profitCallbacks []func(*types.ProfitStats) + closedCallbacks []func() + errorCallbacks []func(error) } func (s *Strategy) ID() string { @@ -151,17 +168,21 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Info("[DCA] user data stream authenticated") time.AfterFunc(3*time.Second, func() { if isInitialize := s.initializeNextStateC(); !isInitialize { - // recover - if err := s.recover(ctx); err != nil { - s.logger.WithError(err).Error("[DCA] something wrong when state recovering") - return + if s.RecoverWhenStart { + // recover + if err := s.recover(ctx); err != nil { + s.logger.WithError(err).Error("[DCA] something wrong when state recovering") + return + } + + s.logger.Infof("[DCA] recovered state: %d", s.state) + s.logger.Infof("[DCA] recovered position %s", s.Position.String()) + s.logger.Infof("[DCA] recovered budget %s", s.Budget) + s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) + } else { + s.state = WaitToOpenPosition } - s.logger.Infof("[DCA] recovered state: %d", s.state) - s.logger.Infof("[DCA] recovered position %s", s.Position.String()) - s.logger.Infof("[DCA] recovered budget %s", s.Budget) - s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) - s.updateTakeProfitPrice() // store persistence @@ -169,6 +190,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. // start running state machine s.runState(ctx) + + s.EmitReady() } }) }) @@ -183,6 +206,19 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. return fmt.Errorf("the available balance of %s is %s which is less than budget setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.Budget) } + bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + + if s.KeepOrdersWhenShutdown { + s.logger.Infof("keepOrdersWhenShutdown is set, will keep the orders on the exchange") + return + } + + if err := s.Close(ctx); err != nil { + s.logger.WithError(err).Errorf("dca2 graceful order cancel error") + } + }) + return nil } @@ -191,3 +227,19 @@ func (s *Strategy) updateTakeProfitPrice() { s.takeProfitPrice = s.Market.TruncatePrice(s.Position.AverageCost.Mul(fixedpoint.One.Add(takeProfitRatio))) s.logger.Infof("[DCA] cost: %s, ratio: %s, price: %s", s.Position.AverageCost, takeProfitRatio, s.takeProfitPrice) } + +func (s *Strategy) Close(ctx context.Context) error { + s.logger.Infof("[DCA] closing %s dca2", s.Symbol) + + defer s.EmitClosed() + + bbgo.Sync(ctx, s) + + return s.cancelAllOrders(ctx) +} + +func (s *Strategy) CleanUp(ctx context.Context) error { + _ = s.Initialize() + defer s.EmitClosed() + return s.cancelAllOrders(ctx) +} diff --git a/pkg/strategy/dca2/strategy_callbacks.go b/pkg/strategy/dca2/strategy_callbacks.go new file mode 100644 index 0000000000..695e223d5e --- /dev/null +++ b/pkg/strategy/dca2/strategy_callbacks.go @@ -0,0 +1,57 @@ +// Code generated by "callbackgen -type Strategy"; DO NOT EDIT. + +package dca2 + +import ( + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) OnReady(cb func()) { + s.readyCallbacks = append(s.readyCallbacks, cb) +} + +func (s *Strategy) EmitReady() { + for _, cb := range s.readyCallbacks { + cb() + } +} + +func (s *Strategy) OnPosition(cb func(*types.Position)) { + s.positionCallbacks = append(s.positionCallbacks, cb) +} + +func (s *Strategy) EmitPosition(position *types.Position) { + for _, cb := range s.positionCallbacks { + cb(position) + } +} + +func (s *Strategy) OnProfit(cb func(*types.ProfitStats)) { + s.profitCallbacks = append(s.profitCallbacks, cb) +} + +func (s *Strategy) EmitProfit(profitStats *types.ProfitStats) { + for _, cb := range s.profitCallbacks { + cb(profitStats) + } +} + +func (s *Strategy) OnClosed(cb func()) { + s.closedCallbacks = append(s.closedCallbacks, cb) +} + +func (s *Strategy) EmitClosed() { + for _, cb := range s.closedCallbacks { + cb() + } +} + +func (s *Strategy) OnError(cb func(err error)) { + s.errorCallbacks = append(s.errorCallbacks, cb) +} + +func (s *Strategy) EmitError(err error) { + for _, cb := range s.errorCallbacks { + cb(err) + } +} From 05870c5d6061f0ae5390d14b11cb4dc5c7211f28 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 2 Jan 2024 14:09:38 +0800 Subject: [PATCH 394/422] move EmitReady and add go:generate --- pkg/strategy/dca2/strategy.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 1dcffcaa49..b29686fe63 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -26,6 +26,7 @@ func init() { bbgo.RegisterStrategy(ID, &Strategy{}) } +//go:generate callbackgen -type Strateg type Strategy struct { *common.Strategy @@ -188,10 +189,11 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. // store persistence bbgo.Sync(ctx, s) + // ready + s.EmitReady() + // start running state machine s.runState(ctx) - - s.EmitReady() } }) }) From b965dbe757f6691fe62f0c4a20474d8a3f58589b Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 3 Jan 2024 16:41:59 +0800 Subject: [PATCH 395/422] use OrderExecutor.GracefulCancel to replace cancelAllOrders --- pkg/strategy/dca2/open_position.go | 21 --------------------- pkg/strategy/dca2/state.go | 2 +- pkg/strategy/dca2/strategy.go | 17 ++++++++++++++--- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index 66af3477b2..ea4cae9bfa 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -107,24 +107,3 @@ func calculateNotionalAndNum(market types.Market, budget fixedpoint.Value, price return fixedpoint.Zero, 0 } - -func (s *Strategy) cancelAllOrders(ctx context.Context) error { - s.logger.Info("[DCA] cancel all orders") - e, ok := s.Session.Exchange.(cancelOrdersByGroupIDApi) - if ok { - cancelledOrders, err := e.CancelOrdersByGroupID(ctx, int64(s.OrderGroupID)) - if err != nil { - return err - } - - for _, cancelledOrder := range cancelledOrders { - s.logger.Info("CANCEL ", cancelledOrder.String()) - } - } else { - if err := s.OrderExecutor.ActiveMakerOrders().GracefulCancel(ctx, s.Session.Exchange); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 3fc626b38a..aa8ad2c2d4 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -156,7 +156,7 @@ func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { s.logger.Info("[State] OpenPositionOrdersCancelling - start cancelling open-position orders") - if err := s.cancelAllOrders(ctx); err != nil { + if err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...); err != nil { s.logger.WithError(err).Error("failed to cancel maker orders") return } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index b29686fe63..f23be66daf 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -235,13 +235,24 @@ func (s *Strategy) Close(ctx context.Context) error { defer s.EmitClosed() - bbgo.Sync(ctx, s) + err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...) + if err != nil { + s.logger.WithError(err).Errorf("[DCA] there are errors when cancelling orders at close") + } - return s.cancelAllOrders(ctx) + bbgo.Sync(ctx, s) + return err } func (s *Strategy) CleanUp(ctx context.Context) error { _ = s.Initialize() defer s.EmitClosed() - return s.cancelAllOrders(ctx) + + err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...) + if err != nil { + s.logger.WithError(err).Errorf("[DCA] there are errors when cancelling orders at clean up") + } + + bbgo.Sync(ctx, s) + return err } From 0d6c6666a181375374235a08476aa19c0efc2bc5 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 3 Jan 2024 17:02:03 +0800 Subject: [PATCH 396/422] fix --- pkg/strategy/dca2/state.go | 2 +- pkg/strategy/dca2/strategy.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index aa8ad2c2d4..771934dc9e 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -156,7 +156,7 @@ func (s *Strategy) runOpenPositionOrderFilled(_ context.Context, next State) { func (s *Strategy) runOpenPositionOrdersCancelling(ctx context.Context, next State) { s.logger.Info("[State] OpenPositionOrdersCancelling - start cancelling open-position orders") - if err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...); err != nil { + if err := s.OrderExecutor.GracefulCancel(ctx); err != nil { s.logger.WithError(err).Error("failed to cancel maker orders") return } diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index f23be66daf..5c92e625e4 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -235,7 +235,7 @@ func (s *Strategy) Close(ctx context.Context) error { defer s.EmitClosed() - err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...) + err := s.OrderExecutor.GracefulCancel(ctx) if err != nil { s.logger.WithError(err).Errorf("[DCA] there are errors when cancelling orders at close") } @@ -248,7 +248,7 @@ func (s *Strategy) CleanUp(ctx context.Context) error { _ = s.Initialize() defer s.EmitClosed() - err := s.OrderExecutor.GracefulCancel(ctx, s.OrderExecutor.ActiveMakerOrders().Orders()...) + err := s.OrderExecutor.GracefulCancel(ctx) if err != nil { s.logger.WithError(err).Errorf("[DCA] there are errors when cancelling orders at clean up") } From faaaaabce3417668baf218aaffffe3965242790e Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 2 Jan 2024 17:16:47 +0800 Subject: [PATCH 397/422] FEATURE: rename and use specific profit stats --- pkg/strategy/dca2/open_position.go | 14 ++-- pkg/strategy/dca2/open_position_test.go | 4 +- pkg/strategy/dca2/profit_stats.go | 90 +++++++++++++++++++++++++ pkg/strategy/dca2/recover.go | 23 ++++--- pkg/strategy/dca2/state.go | 8 ++- pkg/strategy/dca2/strategy.go | 54 +++++++++++---- pkg/strategy/dca2/strategy_callbacks.go | 4 +- 7 files changed, 159 insertions(+), 38 deletions(-) create mode 100644 pkg/strategy/dca2/profit_stats.go diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index ea4cae9bfa..b2a56f1339 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -20,7 +20,7 @@ func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { return err } - orders, err := generateOpenPositionOrders(s.Market, s.Budget, price, s.PriceDeviation, s.MaxOrderNum, s.OrderGroupID) + orders, err := generateOpenPositionOrders(s.Market, s.QuoteInvestment, price, s.PriceDeviation, s.MaxOrderCount, s.OrderGroupID) if err != nil { return err } @@ -44,12 +44,12 @@ func getBestPriceUntilSuccess(ctx context.Context, ex types.Exchange, symbol str return ticker.Sell, nil } -func generateOpenPositionOrders(market types.Market, budget, price, priceDeviation fixedpoint.Value, maxOrderNum int64, orderGroupID uint32) ([]types.SubmitOrder, error) { +func generateOpenPositionOrders(market types.Market, quoteInvestment, price, priceDeviation fixedpoint.Value, maxOrderCount int64, orderGroupID uint32) ([]types.SubmitOrder, error) { factor := fixedpoint.One.Sub(priceDeviation) // calculate all valid prices var prices []fixedpoint.Value - for i := 0; i < int(maxOrderNum); i++ { + for i := 0; i < int(maxOrderCount); i++ { if i > 0 { price = price.Mul(factor) } @@ -61,9 +61,9 @@ func generateOpenPositionOrders(market types.Market, budget, price, priceDeviati prices = append(prices, price) } - notional, orderNum := calculateNotionalAndNum(market, budget, prices) + notional, orderNum := calculateNotionalAndNum(market, quoteInvestment, prices) if orderNum == 0 { - return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, budget: %s", price, budget) + return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, quote investment: %s", price, quoteInvestment) } side := types.SideTypeBuy @@ -89,9 +89,9 @@ func generateOpenPositionOrders(market types.Market, budget, price, priceDeviati // calculateNotionalAndNum calculates the notional and num of open position orders // DCA2 is notional-based, every order has the same notional -func calculateNotionalAndNum(market types.Market, budget fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { +func calculateNotionalAndNum(market types.Market, quoteInvestment fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { for num := len(prices); num > 0; num-- { - notional := budget.Div(fixedpoint.NewFromInt(int64(num))) + notional := quoteInvestment.Div(fixedpoint.NewFromInt(int64(num))) if notional.Compare(market.MinNotional) < 0 { continue } diff --git a/pkg/strategy/dca2/open_position_test.go b/pkg/strategy/dca2/open_position_test.go index 4d68df5106..28b00c0ead 100644 --- a/pkg/strategy/dca2/open_position_test.go +++ b/pkg/strategy/dca2/open_position_test.go @@ -47,10 +47,10 @@ func TestGenerateOpenPositionOrders(t *testing.T) { strategy := newTestStrategy() t.Run("case 1: all config is valid and we can place enough orders", func(t *testing.T) { - budget := Number("10500") + quoteInvestment := Number("10500") askPrice := Number("30000") margin := Number("0.05") - submitOrders, err := generateOpenPositionOrders(strategy.Market, budget, askPrice, margin, 4, strategy.OrderGroupID) + submitOrders, err := generateOpenPositionOrders(strategy.Market, quoteInvestment, askPrice, margin, 4, strategy.OrderGroupID) if !assert.NoError(err) { return } diff --git a/pkg/strategy/dca2/profit_stats.go b/pkg/strategy/dca2/profit_stats.go new file mode 100644 index 0000000000..b87ceaef02 --- /dev/null +++ b/pkg/strategy/dca2/profit_stats.go @@ -0,0 +1,90 @@ +package dca2 + +import ( + "time" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +type ProfitStats struct { + Symbol string `json:"symbol"` + Market types.Market `json:"market,omitempty"` + + CreatedAt time.Time `json:"since,omitempty"` + UpdatedAt time.Time `json:"updatedAt,omitempty"` + Round int64 `json:"round,omitempty"` + QuoteInvestment fixedpoint.Value `json:"quoteInvestment,omitempty"` + + RoundProfit fixedpoint.Value `json:"roundProfit,omitempty"` + RoundFee map[string]fixedpoint.Value `json:"roundFee,omitempty"` + TotalProfit fixedpoint.Value `json:"totalProfit,omitempty"` + TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` + + // ttl is the ttl to keep in persistence + ttl time.Duration +} + +func newProfitStats(market types.Market, quoteInvestment fixedpoint.Value) *ProfitStats { + return &ProfitStats{ + Symbol: market.Symbol, + Market: market, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Round: 0, + QuoteInvestment: quoteInvestment, + RoundFee: make(map[string]fixedpoint.Value), + TotalFee: make(map[string]fixedpoint.Value), + } +} + +func (s *ProfitStats) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } + s.ttl = ttl +} + +func (s *ProfitStats) Expiration() time.Duration { + return s.ttl +} + +func (s *ProfitStats) AddTrade(trade types.Trade) { + if s.RoundFee == nil { + s.RoundFee = make(map[string]fixedpoint.Value) + } + + if fee, ok := s.RoundFee[trade.FeeCurrency]; ok { + s.RoundFee[trade.FeeCurrency] = fee.Add(trade.Fee) + } else { + s.RoundFee[trade.FeeCurrency] = trade.Fee + } + + if s.TotalFee == nil { + s.TotalFee = make(map[string]fixedpoint.Value) + } + + if fee, ok := s.TotalFee[trade.FeeCurrency]; ok { + s.TotalFee[trade.FeeCurrency] = fee.Add(trade.Fee) + } else { + s.TotalFee[trade.FeeCurrency] = trade.Fee + } + + switch trade.Side { + case types.SideTypeSell: + s.RoundProfit = s.RoundProfit.Add(trade.QuoteQuantity) + s.TotalProfit = s.TotalProfit.Add(trade.QuoteQuantity) + case types.SideTypeBuy: + s.RoundProfit = s.RoundProfit.Sub(trade.QuoteQuantity) + s.TotalProfit = s.TotalProfit.Sub(trade.QuoteQuantity) + default: + } + + s.UpdatedAt = trade.Time.Time() +} + +func (s *ProfitStats) FinishRound() { + s.Round++ + s.RoundProfit = fixedpoint.Zero + s.RoundFee = make(map[string]fixedpoint.Value) +} diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index 6a09dd6c9f..2d20078753 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -34,7 +34,8 @@ func (s *Strategy) recover(ctx context.Context) error { return err } - closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Time{}, time.Now(), 0) + closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Date(2024, time.January, 1, 0, 0, 0, 0, time.Local), time.Now(), 0) + // closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Time{}, time.Now(), 0) if err != nil { return err } @@ -46,7 +47,7 @@ func (s *Strategy) recover(ctx context.Context) error { debugRoundOrders(s.logger, "current", currentRound) // recover state - state, err := recoverState(ctx, s.Symbol, int(s.MaxOrderNum), openOrders, currentRound, s.OrderExecutor.ActiveMakerOrders(), s.OrderExecutor.OrderStore(), s.OrderGroupID) + state, err := recoverState(ctx, s.Symbol, int(s.MaxOrderCount), openOrders, currentRound, s.OrderExecutor.ActiveMakerOrders(), s.OrderExecutor.OrderStore(), s.OrderGroupID) if err != nil { return err } @@ -56,15 +57,15 @@ func (s *Strategy) recover(ctx context.Context) error { return err } - // recover budget - budget := recoverBudget(currentRound) + // recover quote investment + quoteInvestment := recoverQuoteInvestment(currentRound) // recover startTimeOfNextRound startTimeOfNextRound := recoverStartTimeOfNextRound(ctx, currentRound, s.CoolDownInterval) s.state = state - if !budget.IsZero() { - s.Budget = budget + if !quoteInvestment.IsZero() { + s.QuoteInvestment = quoteInvestment } s.startTimeOfNextRound = startTimeOfNextRound @@ -72,7 +73,7 @@ func (s *Strategy) recover(ctx context.Context) error { } // recover state -func recoverState(ctx context.Context, symbol string, maxOrderNum int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { +func recoverState(ctx context.Context, symbol string, maxOrderCount int, openOrders []types.Order, currentRound Round, activeOrderBook *bbgo.ActiveOrderBook, orderStore *core.OrderStore, groupID uint32) (State, error) { if len(currentRound.OpenPositionOrders) == 0 { // new strategy return WaitToOpenPosition, nil @@ -101,10 +102,10 @@ func recoverState(ctx context.Context, symbol string, maxOrderNum int, openOrder } numOpenPositionOrders := len(currentRound.OpenPositionOrders) - if numOpenPositionOrders > maxOrderNum { + if numOpenPositionOrders > maxOrderCount { return None, fmt.Errorf("the number of open-position orders is > max order number") - } else if numOpenPositionOrders < maxOrderNum { - // The number of open-position orders should be the same as maxOrderNum + } else if numOpenPositionOrders < maxOrderCount { + // The number of open-position orders should be the same as maxOrderCount // If not, it may be the following possible cause // 1. This strategy at position opening, so it may not place all orders we want successfully // 2. There are some errors when placing open-position orders. e.g. cannot lock fund..... @@ -192,7 +193,7 @@ func recoverPosition(ctx context.Context, position *types.Position, queryService return nil } -func recoverBudget(currentRound Round) fixedpoint.Value { +func recoverQuoteInvestment(currentRound Round) fixedpoint.Value { if len(currentRound.OpenPositionOrders) == 0 { return fixedpoint.Zero } diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 771934dc9e..bd8756976e 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -181,12 +181,16 @@ func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { // wait 3 seconds to avoid position not update time.Sleep(3 * time.Second) - s.logger.Info("[State] TakeProfitReady - start reseting position and calculate budget for next round") - s.Budget = s.Budget.Add(s.Position.Quote) + s.logger.Info("[State] TakeProfitReady - start reseting position and calculate quote investment for next round") + s.QuoteInvestment = s.QuoteInvestment.Add(s.Position.Quote) // reset position s.Position.Reset() + // reset + s.EmitProfit(s.ProfitStats) + s.ProfitStats.FinishRound() + // set the start time of the next round s.startTimeOfNextRound = time.Now().Add(s.CoolDownInterval.Duration()) s.state = WaitToOpenPosition diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 5c92e625e4..5128a52351 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -9,7 +9,6 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" "github.com/prometheus/client_golang/prometheus" @@ -28,16 +27,19 @@ func init() { //go:generate callbackgen -type Strateg type Strategy struct { - *common.Strategy + Position *types.Position `json:"position,omitempty" persistence:"position"` + ProfitStats *ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` - Environment *bbgo.Environment - Market types.Market + Environment *bbgo.Environment + Session *bbgo.ExchangeSession + OrderExecutor *bbgo.GeneralOrderExecutor + Market types.Market Symbol string `json:"symbol"` // setting - Budget fixedpoint.Value `json:"budget"` - MaxOrderNum int64 `json:"maxOrderNum"` + QuoteInvestment fixedpoint.Value `json:"quoteInvestment"` + MaxOrderCount int64 `json:"maxOrderCount"` PriceDeviation fixedpoint.Value `json:"priceDeviation"` TakeProfitRatio fixedpoint.Value `json:"takeProfitRatio"` CoolDownInterval types.Duration `json:"coolDownInterval"` @@ -68,7 +70,7 @@ type Strategy struct { // callbacks readyCallbacks []func() positionCallbacks []func(*types.Position) - profitCallbacks []func(*types.ProfitStats) + profitCallbacks []func(*ProfitStats) closedCallbacks []func() errorCallbacks []func(error) } @@ -78,8 +80,8 @@ func (s *Strategy) ID() string { } func (s *Strategy) Validate() error { - if s.MaxOrderNum < 1 { - return fmt.Errorf("maxOrderNum can not be < 1") + if s.MaxOrderCount < 1 { + return fmt.Errorf("maxOrderCount can not be < 1") } if s.TakeProfitRatio.Sign() <= 0 { @@ -106,7 +108,6 @@ func (s *Strategy) Defaults() error { func (s *Strategy) Initialize() error { s.logger = log.WithFields(s.LogFields) - s.Strategy = &common.Strategy{} return nil } @@ -119,8 +120,29 @@ func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { } func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { - s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) instanceID := s.InstanceID() + s.Session = session + if s.ProfitStats == nil { + s.ProfitStats = newProfitStats(s.Market, s.QuoteInvestment) + } + + if s.Position == nil { + s.Position = types.NewPositionFromMarket(s.Market) + } + + s.Position.Strategy = ID + s.Position.StrategyInstanceID = instanceID + + if session.MakerFeeRate.Sign() > 0 || session.TakerFeeRate.Sign() > 0 { + s.Position.SetExchangeFeeRate(session.ExchangeName, types.ExchangeFee{ + MakerFeeRate: session.MakerFeeRate, + TakerFeeRate: session.TakerFeeRate, + }) + } + + s.OrderExecutor = bbgo.NewGeneralOrderExecutor(session, s.Symbol, ID, instanceID, s.Position) + s.OrderExecutor.BindEnvironment(s.Environment) + s.OrderExecutor.Bind() if s.OrderGroupID == 0 { s.OrderGroupID = util.FNV32(instanceID) % math.MaxInt32 @@ -135,6 +157,10 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.updateTakeProfitPrice() }) + s.OrderExecutor.TradeCollector().OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { + s.ProfitStats.AddTrade(trade) + }) + s.OrderExecutor.ActiveMakerOrders().OnFilled(func(o types.Order) { s.logger.Infof("[DCA] FILLED ORDER: %s", o.String()) openPositionSide := types.SideTypeBuy @@ -178,7 +204,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Infof("[DCA] recovered state: %d", s.state) s.logger.Infof("[DCA] recovered position %s", s.Position.String()) - s.logger.Infof("[DCA] recovered budget %s", s.Budget) + s.logger.Infof("[DCA] recovered quote investment %s", s.QuoteInvestment) s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) } else { s.state = WaitToOpenPosition @@ -204,8 +230,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } balance := balances[s.Market.QuoteCurrency] - if balance.Available.Compare(s.Budget) < 0 { - return fmt.Errorf("the available balance of %s is %s which is less than budget setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.Budget) + if balance.Available.Compare(s.QuoteInvestment) < 0 { + return fmt.Errorf("the available balance of %s is %s which is less than quote investment setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.QuoteInvestment) } bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { diff --git a/pkg/strategy/dca2/strategy_callbacks.go b/pkg/strategy/dca2/strategy_callbacks.go index 695e223d5e..64781b4b2c 100644 --- a/pkg/strategy/dca2/strategy_callbacks.go +++ b/pkg/strategy/dca2/strategy_callbacks.go @@ -26,11 +26,11 @@ func (s *Strategy) EmitPosition(position *types.Position) { } } -func (s *Strategy) OnProfit(cb func(*types.ProfitStats)) { +func (s *Strategy) OnProfit(cb func(*ProfitStats)) { s.profitCallbacks = append(s.profitCallbacks, cb) } -func (s *Strategy) EmitProfit(profitStats *types.ProfitStats) { +func (s *Strategy) EmitProfit(profitStats *ProfitStats) { for _, cb := range s.profitCallbacks { cb(profitStats) } From 468b73abb67d59ebd5ac5bc3fe50d8cb8ada084b Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 3 Jan 2024 13:54:38 +0800 Subject: [PATCH 398/422] bbgo.Sync profit stats --- config/dca2.yaml | 11 +++++------ pkg/strategy/dca2/state.go | 7 ++++++- pkg/strategy/dca2/strategy.go | 3 +++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/config/dca2.yaml b/config/dca2.yaml index 6cb8b6ca9a..894afc6342 100644 --- a/config/dca2.yaml +++ b/config/dca2.yaml @@ -23,9 +23,8 @@ exchangeStrategies: dca2: symbol: ETHUSDT short: false - budget: 200 - maxOrderNum: 5 - priceDeviation: 1% - takeProfitRatio: 0.2% - coolDownInterval: 3m - circuitBreakLossThreshold: -0.9 + quoteInvestment: "200" + maxOrderCount: 5 + priceDeviation: "0.01" + takeProfitRatio: "0.002" + coolDownInterval: 180 diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index bd8756976e..24aa15cef6 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -3,6 +3,8 @@ package dca2 import ( "context" "time" + + "github.com/c9s/bbgo/pkg/bbgo" ) type State int64 @@ -177,12 +179,13 @@ func (s *Strategy) runOpenPositionOrdersCancelled(ctx context.Context, next Stat s.logger.Info("[State] OpenPositionOrdersCancelled -> TakeProfitReady") } -func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { +func (s *Strategy) runTakeProfitReady(ctx context.Context, next State) { // wait 3 seconds to avoid position not update time.Sleep(3 * time.Second) s.logger.Info("[State] TakeProfitReady - start reseting position and calculate quote investment for next round") s.QuoteInvestment = s.QuoteInvestment.Add(s.Position.Quote) + s.ProfitStats.QuoteInvestment = s.QuoteInvestment // reset position s.Position.Reset() @@ -191,6 +194,8 @@ func (s *Strategy) runTakeProfitReady(_ context.Context, next State) { s.EmitProfit(s.ProfitStats) s.ProfitStats.FinishRound() + bbgo.Sync(ctx, s) + // set the start time of the next round s.startTimeOfNextRound = time.Now().Add(s.CoolDownInterval.Duration()) s.state = WaitToOpenPosition diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 5128a52351..f68077ec2b 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -159,6 +159,7 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.OrderExecutor.TradeCollector().OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { s.ProfitStats.AddTrade(trade) + bbgo.Sync(ctx, s) }) s.OrderExecutor.ActiveMakerOrders().OnFilled(func(o types.Order) { @@ -206,6 +207,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.logger.Infof("[DCA] recovered position %s", s.Position.String()) s.logger.Infof("[DCA] recovered quote investment %s", s.QuoteInvestment) s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) + + bbgo.Sync(ctx, s) } else { s.state = WaitToOpenPosition } From 21e87079b5b1a8f02834886981a3f12c09c5bc77 Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Mon, 8 Jan 2024 18:24:11 +0800 Subject: [PATCH 399/422] FEATURE: ProfitStats for dca2 --- pkg/strategy/dca2/open_position.go | 2 +- pkg/strategy/dca2/profit_stats.go | 90 +++++++++++++++++++++++++----- pkg/strategy/dca2/recover.go | 20 ++++--- pkg/strategy/dca2/state.go | 20 ++++--- pkg/strategy/dca2/strategy.go | 19 ++----- 5 files changed, 107 insertions(+), 44 deletions(-) diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index b2a56f1339..f40d17b94e 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -20,7 +20,7 @@ func (s *Strategy) placeOpenPositionOrders(ctx context.Context) error { return err } - orders, err := generateOpenPositionOrders(s.Market, s.QuoteInvestment, price, s.PriceDeviation, s.MaxOrderCount, s.OrderGroupID) + orders, err := generateOpenPositionOrders(s.Market, s.ProfitStats.QuoteInvestment, price, s.PriceDeviation, s.MaxOrderCount, s.OrderGroupID) if err != nil { return err } diff --git a/pkg/strategy/dca2/profit_stats.go b/pkg/strategy/dca2/profit_stats.go index b87ceaef02..5ceb21b7a2 100644 --- a/pkg/strategy/dca2/profit_stats.go +++ b/pkg/strategy/dca2/profit_stats.go @@ -1,6 +1,10 @@ package dca2 import ( + "context" + "fmt" + "strconv" + "strings" "time" "github.com/c9s/bbgo/pkg/fixedpoint" @@ -11,8 +15,7 @@ type ProfitStats struct { Symbol string `json:"symbol"` Market types.Market `json:"market,omitempty"` - CreatedAt time.Time `json:"since,omitempty"` - UpdatedAt time.Time `json:"updatedAt,omitempty"` + FromOrderID uint64 `json:"fromOrderID,omitempty"` Round int64 `json:"round,omitempty"` QuoteInvestment fixedpoint.Value `json:"quoteInvestment,omitempty"` @@ -29,8 +32,6 @@ func newProfitStats(market types.Market, quoteInvestment fixedpoint.Value) *Prof return &ProfitStats{ Symbol: market.Symbol, Market: market, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), Round: 0, QuoteInvestment: quoteInvestment, RoundFee: make(map[string]fixedpoint.Value), @@ -70,21 +71,82 @@ func (s *ProfitStats) AddTrade(trade types.Trade) { s.TotalFee[trade.FeeCurrency] = trade.Fee } - switch trade.Side { - case types.SideTypeSell: - s.RoundProfit = s.RoundProfit.Add(trade.QuoteQuantity) - s.TotalProfit = s.TotalProfit.Add(trade.QuoteQuantity) - case types.SideTypeBuy: - s.RoundProfit = s.RoundProfit.Sub(trade.QuoteQuantity) - s.TotalProfit = s.TotalProfit.Sub(trade.QuoteQuantity) - default: + quoteQuantity := trade.QuoteQuantity + if trade.Side == types.SideTypeBuy { + quoteQuantity = quoteQuantity.Neg() } - s.UpdatedAt = trade.Time.Time() + s.RoundProfit = s.RoundProfit.Add(quoteQuantity) + s.TotalProfit = s.TotalProfit.Add(quoteQuantity) + + if s.Market.QuoteCurrency == trade.FeeCurrency { + s.RoundProfit.Sub(trade.Fee) + s.TotalProfit.Sub(trade.Fee) + } } -func (s *ProfitStats) FinishRound() { +func (s *ProfitStats) NewRound() { s.Round++ s.RoundProfit = fixedpoint.Zero s.RoundFee = make(map[string]fixedpoint.Value) } + +func (s *ProfitStats) CalculateProfitOfRound(ctx context.Context, exchange types.Exchange) error { + historyService, ok := exchange.(types.ExchangeTradeHistoryService) + if !ok { + return fmt.Errorf("exchange %s doesn't support ExchangeTradeHistoryService", exchange.Name()) + } + + queryService, ok := exchange.(types.ExchangeOrderQueryService) + if !ok { + return fmt.Errorf("exchange %s doesn't support ExchangeOrderQueryService", exchange.Name()) + } + + // query the orders of this round + orders, err := historyService.QueryClosedOrders(ctx, s.Symbol, time.Time{}, time.Time{}, s.FromOrderID) + if err != nil { + return err + } + + // query the trades of this round + for _, order := range orders { + if order.ExecutedQuantity.Sign() == 0 { + // skip no trade orders + continue + } + + trades, err := queryService.QueryOrderTrades(ctx, types.OrderQuery{ + Symbol: order.Symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }) + + if err != nil { + return err + } + + for _, trade := range trades { + s.AddTrade(trade) + } + } + + s.FromOrderID = s.FromOrderID + 1 + s.QuoteInvestment = s.QuoteInvestment.Add(s.RoundProfit) + + return nil +} + +func (s *ProfitStats) String() string { + var sb strings.Builder + sb.WriteString("[------------------ Profit Stats ------------------]\n") + sb.WriteString(fmt.Sprintf("Round: %d\n", s.Round)) + sb.WriteString(fmt.Sprintf("From Order ID: %d\n", s.FromOrderID)) + sb.WriteString(fmt.Sprintf("Quote Investment: %s\n", s.QuoteInvestment)) + sb.WriteString(fmt.Sprintf("Round Profit: %s\n", s.RoundProfit)) + sb.WriteString(fmt.Sprintf("Total Profit: %s\n", s.TotalProfit)) + for currency, fee := range s.RoundFee { + sb.WriteString(fmt.Sprintf("FEE (%s): %s\n", currency, fee)) + } + sb.WriteString("[------------------ Profit Stats ------------------]\n") + + return sb.String() +} diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index 2d20078753..fd5de6fbf0 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -35,7 +35,6 @@ func (s *Strategy) recover(ctx context.Context) error { } closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Date(2024, time.January, 1, 0, 0, 0, 0, time.Local), time.Now(), 0) - // closedOrders, err := queryService.QueryClosedOrdersDesc(ctx, s.Symbol, time.Time{}, time.Now(), 0) if err != nil { return err } @@ -57,16 +56,13 @@ func (s *Strategy) recover(ctx context.Context) error { return err } - // recover quote investment - quoteInvestment := recoverQuoteInvestment(currentRound) + // recover profit stats + recoverProfitStats(ctx, s.ProfitStats, s.Session.Exchange) // recover startTimeOfNextRound startTimeOfNextRound := recoverStartTimeOfNextRound(ctx, currentRound, s.CoolDownInterval) s.state = state - if !quoteInvestment.IsZero() { - s.QuoteInvestment = quoteInvestment - } s.startTimeOfNextRound = startTimeOfNextRound return nil @@ -155,7 +151,7 @@ func recoverState(ctx context.Context, symbol string, maxOrderCount int, openOrd func recoverPosition(ctx context.Context, position *types.Position, queryService RecoverApiQueryService, currentRound Round) error { if position == nil { - return nil + return fmt.Errorf("position is nil, please check it") } var positionOrders []types.Order @@ -193,6 +189,16 @@ func recoverPosition(ctx context.Context, position *types.Position, queryService return nil } +func recoverProfitStats(ctx context.Context, profitStats *ProfitStats, exchange types.Exchange) error { + if profitStats == nil { + return fmt.Errorf("profit stats is nil, please check it") + } + + profitStats.CalculateProfitOfRound(ctx, exchange) + + return nil +} + func recoverQuoteInvestment(currentRound Round) fixedpoint.Value { if len(currentRound.OpenPositionOrders) == 0 { return fixedpoint.Zero diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 24aa15cef6..53ee3386ce 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -123,12 +123,19 @@ func (s *Strategy) triggerNextState() { } } -func (s *Strategy) runWaitToOpenPositionState(_ context.Context, next State) { +func (s *Strategy) runWaitToOpenPositionState(ctx context.Context, next State) { s.logger.Info("[State] WaitToOpenPosition - check startTimeOfNextRound") if time.Now().Before(s.startTimeOfNextRound) { return } + // reset position and open new round for profit stats before position opening + s.Position.Reset() + s.ProfitStats.NewRound() + + // store into redis + bbgo.Sync(ctx, s) + s.state = PositionOpening s.logger.Info("[State] WaitToOpenPosition -> PositionOpening") } @@ -184,17 +191,12 @@ func (s *Strategy) runTakeProfitReady(ctx context.Context, next State) { time.Sleep(3 * time.Second) s.logger.Info("[State] TakeProfitReady - start reseting position and calculate quote investment for next round") - s.QuoteInvestment = s.QuoteInvestment.Add(s.Position.Quote) - s.ProfitStats.QuoteInvestment = s.QuoteInvestment - // reset position - s.Position.Reset() + // calculate profit stats + s.ProfitStats.CalculateProfitOfRound(ctx, s.Session.Exchange) + bbgo.Sync(ctx, s) - // reset s.EmitProfit(s.ProfitStats) - s.ProfitStats.FinishRound() - - bbgo.Sync(ctx, s) // set the start time of the next round s.startTimeOfNextRound = time.Now().Add(s.CoolDownInterval.Duration()) diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index f68077ec2b..3fec4a6c96 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -157,11 +157,6 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. s.updateTakeProfitPrice() }) - s.OrderExecutor.TradeCollector().OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { - s.ProfitStats.AddTrade(trade) - bbgo.Sync(ctx, s) - }) - s.OrderExecutor.ActiveMakerOrders().OnFilled(func(o types.Order) { s.logger.Infof("[DCA] FILLED ORDER: %s", o.String()) openPositionSide := types.SideTypeBuy @@ -203,12 +198,10 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. return } - s.logger.Infof("[DCA] recovered state: %d", s.state) - s.logger.Infof("[DCA] recovered position %s", s.Position.String()) - s.logger.Infof("[DCA] recovered quote investment %s", s.QuoteInvestment) - s.logger.Infof("[DCA] recovered startTimeOfNextRound %s", s.startTimeOfNextRound) - - bbgo.Sync(ctx, s) + s.logger.Infof("[DCA] state: %d", s.state) + s.logger.Infof("[DCA] position %s", s.Position.String()) + s.logger.Infof("[DCA] profit stats %s", s.ProfitStats.String()) + s.logger.Infof("[DCA] startTimeOfNextRound %s", s.startTimeOfNextRound) } else { s.state = WaitToOpenPosition } @@ -233,8 +226,8 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } balance := balances[s.Market.QuoteCurrency] - if balance.Available.Compare(s.QuoteInvestment) < 0 { - return fmt.Errorf("the available balance of %s is %s which is less than quote investment setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.QuoteInvestment) + if balance.Available.Compare(s.ProfitStats.QuoteInvestment) < 0 { + return fmt.Errorf("the available balance of %s is %s which is less than quote investment setting %s, please check it", s.Market.QuoteCurrency, balance.Available, s.ProfitStats.QuoteInvestment) } bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { From 2e34f7840a0f27777131bf51dfa5651bfe939e02 Mon Sep 17 00:00:00 2001 From: Edwin Date: Mon, 8 Jan 2024 15:05:32 +0800 Subject: [PATCH 400/422] pkg/exchange: support market trade streaming --- pkg/exchange/okex/convert.go | 17 +++-- pkg/exchange/okex/parse.go | 60 +++++++++++++++++ pkg/exchange/okex/parse_test.go | 93 +++++++++++++++++++++++++++ pkg/exchange/okex/stream.go | 24 +++++++ pkg/exchange/okex/stream_callbacks.go | 12 ++++ pkg/exchange/okex/stream_test.go | 14 ++++ 6 files changed, 214 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 742ad56f3a..f1889bb0af 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -55,9 +55,9 @@ func toGlobalBalance(account *okexapi.Account) types.BalanceMap { } type WebsocketSubscription struct { - Channel string `json:"channel"` - InstrumentID string `json:"instId,omitempty"` - InstrumentType string `json:"instType,omitempty"` + Channel Channel `json:"channel"` + InstrumentID string `json:"instId,omitempty"` + InstrumentType string `json:"instType,omitempty"` } var CandleChannels = []string{ @@ -92,18 +92,23 @@ func convertSubscription(s types.Subscription) (WebsocketSubscription, error) { case types.KLineChannel: // Channel names are: return WebsocketSubscription{ - Channel: convertIntervalToCandle(s.Options.Interval), + Channel: Channel(convertIntervalToCandle(s.Options.Interval)), InstrumentID: toLocalSymbol(s.Symbol), }, nil case types.BookChannel: return WebsocketSubscription{ - Channel: "books", + Channel: ChannelBooks, InstrumentID: toLocalSymbol(s.Symbol), }, nil case types.BookTickerChannel: return WebsocketSubscription{ - Channel: "books5", + Channel: ChannelBook5, + InstrumentID: toLocalSymbol(s.Symbol), + }, nil + case types.MarketTradeChannel: + return WebsocketSubscription{ + Channel: ChannelMarketTrades, InstrumentID: toLocalSymbol(s.Symbol), }, nil } diff --git a/pkg/exchange/okex/parse.go b/pkg/exchange/okex/parse.go index cd2cbffe18..df538b373e 100644 --- a/pkg/exchange/okex/parse.go +++ b/pkg/exchange/okex/parse.go @@ -21,6 +21,7 @@ const ( ChannelBook5 Channel = "book5" ChannelCandlePrefix Channel = "candle" ChannelAccount Channel = "account" + ChannelMarketTrades Channel = "trades" ChannelOrders Channel = "orders" ) @@ -66,6 +67,14 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { bookEvent.Action = event.ActionType return &bookEvent, nil + case ChannelMarketTrades: + var trade []MarketTradeEvent + err = json.Unmarshal(event.Data, &trade) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into MarketTradeEvent: %+v, err: %w", string(event.Data), err) + } + return trade, nil + case ChannelOrders: // TODO: remove fastjson return parseOrder(v) @@ -363,3 +372,54 @@ func parseOrder(v *fastjson.Value) ([]okexapi.OrderDetails, error) { return orderDetails, nil } + +func toGlobalSideType(side okexapi.SideType) (types.SideType, error) { + switch side { + case okexapi.SideTypeBuy: + return types.SideTypeBuy, nil + + case okexapi.SideTypeSell: + return types.SideTypeSell, nil + + default: + return types.SideType(side), fmt.Errorf("unexpected side: %s", side) + } +} + +type MarketTradeEvent struct { + InstId string `json:"instId"` + TradeId types.StrInt64 `json:"tradeId"` + Px fixedpoint.Value `json:"px"` + Sz fixedpoint.Value `json:"sz"` + Side okexapi.SideType `json:"side"` + Timestamp types.MillisecondTimestamp `json:"ts"` + Count types.StrInt64 `json:"count"` +} + +func (m *MarketTradeEvent) toGlobalTrade() (types.Trade, error) { + symbol := toGlobalSymbol(m.InstId) + if symbol == "" { + return types.Trade{}, fmt.Errorf("unexpected inst id: %s", m.InstId) + } + + side, err := toGlobalSideType(m.Side) + if err != nil { + return types.Trade{}, err + } + + return types.Trade{ + ID: uint64(m.TradeId), + OrderID: 0, // not supported + Exchange: types.ExchangeOKEx, + Price: m.Px, + Quantity: m.Sz, + QuoteQuantity: m.Px.Mul(m.Sz), + Symbol: symbol, + Side: side, + IsBuyer: side == types.SideTypeBuy, + IsMaker: false, // not supported + Time: types.Time(m.Timestamp.Time()), + Fee: fixedpoint.Zero, // not supported + FeeCurrency: "", // not supported + }, nil +} diff --git a/pkg/exchange/okex/parse_test.go b/pkg/exchange/okex/parse_test.go index 42c41cf2d8..5ebcf19893 100644 --- a/pkg/exchange/okex/parse_test.go +++ b/pkg/exchange/okex/parse_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/c9s/bbgo/pkg/exchange/okex/okexapi" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -575,3 +576,95 @@ func TestKLine_ToGlobal(t *testing.T) { }) } + +func Test_parseWebSocketEvent(t *testing.T) { + in := ` +{ + "arg": { + "channel": "trades", + "instId": "BTC-USDT" + }, + "data": [ + { + "instId": "BTC-USDT", + "tradeId": "130639474", + "px": "42219.9", + "sz": "0.12060306", + "side": "buy", + "ts": "1630048897897", + "count": "3" + } + ] +} +` + exp := []MarketTradeEvent{{ + InstId: "BTC-USDT", + TradeId: 130639474, + Px: fixedpoint.NewFromFloat(42219.9), + Sz: fixedpoint.NewFromFloat(0.12060306), + Side: okexapi.SideTypeBuy, + Timestamp: types.NewMillisecondTimestampFromInt(1630048897897), + Count: 3, + }} + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.([]MarketTradeEvent) + assert.True(t, ok) + assert.Len(t, event, 1) + assert.Equal(t, exp, event) + +} + +func Test_toGlobalTrade(t *testing.T) { + // { + // "instId": "BTC-USDT", + // "tradeId": "130639474", + // "px": "42219.9", + // "sz": "0.12060306", + // "side": "buy", + // "ts": "1630048897897", + // "count": "3" + // } + marketTrade := MarketTradeEvent{ + InstId: "BTC-USDT", + TradeId: 130639474, + Px: fixedpoint.NewFromFloat(42219.9), + Sz: fixedpoint.NewFromFloat(0.12060306), + Side: okexapi.SideTypeBuy, + Timestamp: types.NewMillisecondTimestampFromInt(1630048897897), + Count: 3, + } + t.Run("succeeds", func(t *testing.T) { + trade, err := marketTrade.toGlobalTrade() + assert.NoError(t, err) + assert.Equal(t, types.Trade{ + ID: uint64(130639474), + OrderID: uint64(0), + Exchange: types.ExchangeOKEx, + Price: fixedpoint.NewFromFloat(42219.9), + Quantity: fixedpoint.NewFromFloat(0.12060306), + QuoteQuantity: marketTrade.Px.Mul(marketTrade.Sz), + Symbol: "BTCUSDT", + Side: types.SideTypeBuy, + IsBuyer: true, + IsMaker: false, + Time: types.Time(types.NewMillisecondTimestampFromInt(1630048897897)), + Fee: fixedpoint.Zero, + FeeCurrency: "", + FeeDiscounted: false, + }, trade) + }) + t.Run("unexpected side", func(t *testing.T) { + newTrade := marketTrade + newTrade.Side = "both" + _, err := newTrade.toGlobalTrade() + assert.ErrorContains(t, err, "both") + }) + t.Run("unexpected symbol", func(t *testing.T) { + newTrade := marketTrade + newTrade.InstId = "" + _, err := newTrade.toGlobalTrade() + assert.ErrorContains(t, err, "unexpected inst id") + }) +} diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index 9d45ae3b27..34d6c853ce 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -2,6 +2,7 @@ package okex import ( "context" + "golang.org/x/time/rate" "strconv" "time" @@ -9,6 +10,10 @@ import ( "github.com/c9s/bbgo/pkg/types" ) +var ( + tradeLogLimiter = rate.NewLimiter(rate.Every(time.Minute), 1) +) + type WebsocketOp struct { Op string `json:"op"` Args interface{} `json:"args"` @@ -33,6 +38,7 @@ type Stream struct { eventCallbacks []func(event WebSocketEvent) accountEventCallbacks []func(account okexapi.Account) orderDetailsEventCallbacks []func(orderDetails []okexapi.OrderDetails) + marketTradeEventCallbacks []func(tradeDetail []MarketTradeEvent) } func NewStream(client *okexapi.RestClient) *Stream { @@ -48,6 +54,7 @@ func NewStream(client *okexapi.RestClient) *Stream { stream.OnKLineEvent(stream.handleKLineEvent) stream.OnBookEvent(stream.handleBookEvent) stream.OnAccountEvent(stream.handleAccountEvent) + stream.OnMarketTradeEvent(stream.handleMarketTradeEvent) stream.OnOrderDetailsEvent(stream.handleOrderDetailsEvent) stream.OnEvent(stream.handleEvent) stream.OnConnect(stream.handleConnect) @@ -166,6 +173,20 @@ func (s *Stream) handleBookEvent(data BookEvent) { } } +func (s *Stream) handleMarketTradeEvent(data []MarketTradeEvent) { + for _, event := range data { + trade, err := event.toGlobalTrade() + if err != nil { + if tradeLogLimiter.Allow() { + log.WithError(err).Error("failed to convert to market trade") + } + continue + } + + s.EmitMarketTrade(trade) + } +} + func (s *Stream) handleKLineEvent(k KLineEvent) { for _, event := range k.Events { kline := event.ToGlobal(types.Interval(k.Interval), k.Symbol) @@ -207,5 +228,8 @@ func (s *Stream) dispatchEvent(e interface{}) { case []okexapi.OrderDetails: s.EmitOrderDetailsEvent(et) + case []MarketTradeEvent: + s.EmitMarketTradeEvent(et) + } } diff --git a/pkg/exchange/okex/stream_callbacks.go b/pkg/exchange/okex/stream_callbacks.go index 750614b7c6..b735d09850 100644 --- a/pkg/exchange/okex/stream_callbacks.go +++ b/pkg/exchange/okex/stream_callbacks.go @@ -56,6 +56,16 @@ func (s *Stream) EmitOrderDetailsEvent(orderDetails []okexapi.OrderDetails) { } } +func (s *Stream) OnMarketTradeEvent(cb func(tradeDetail []MarketTradeEvent)) { + s.marketTradeEventCallbacks = append(s.marketTradeEventCallbacks, cb) +} + +func (s *Stream) EmitMarketTradeEvent(tradeDetail []MarketTradeEvent) { + for _, cb := range s.marketTradeEventCallbacks { + cb(tradeDetail) + } +} + type StreamEventHub interface { OnKLineEvent(cb func(candle KLineEvent)) @@ -66,4 +76,6 @@ type StreamEventHub interface { OnAccountEvent(cb func(account okexapi.Account)) OnOrderDetailsEvent(cb func(orderDetails []okexapi.OrderDetails)) + + OnMarketTradeEvent(cb func(tradeDetail []MarketTradeEvent)) } diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go index 1cc4e5e5da..b9b758a6aa 100644 --- a/pkg/exchange/okex/stream_test.go +++ b/pkg/exchange/okex/stream_test.go @@ -48,6 +48,20 @@ func TestStream(t *testing.T) { c := make(chan struct{}) <-c }) + + t.Run("market trade test", func(t *testing.T) { + s.Subscribe(types.MarketTradeChannel, "BTCUSDT", types.SubscribeOptions{}) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnMarketTrade(func(trade types.Trade) { + t.Log("got trade upgrade", trade) + }) + c := make(chan struct{}) + <-c + }) + t.Run("kline test", func(t *testing.T) { s.Subscribe(types.KLineChannel, "LTC-USD-200327", types.SubscribeOptions{ Interval: types.Interval1m, From 2ff74a5f86dafedb31815e7af958fcb8222b54bd Mon Sep 17 00:00:00 2001 From: c9s Date: Tue, 9 Jan 2024 09:58:20 +0800 Subject: [PATCH 401/422] autoborrow: add repaid alert --- config/autoborrow.yaml | 16 ++++-- pkg/strategy/autoborrow/strategy.go | 77 +++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/config/autoborrow.yaml b/config/autoborrow.yaml index b651ae8b05..87bb105f7b 100644 --- a/config/autoborrow.yaml +++ b/config/autoborrow.yaml @@ -13,11 +13,17 @@ exchangeStrategies: # if the margin ratio is high enough, we don't have the urge to repay maxMarginLevel: 20.0 - marginLevelAlertInterval: 5m - marginLevelAlertMinMargin: 2.0 - marginLevelAlertSlackMentions: - - '<@USER_ID>' - - '' + marginRepayAlert: + slackMentions: + - '<@USER_ID>' + - '' + + marginLevelAlert: + interval: 5m + minMargin: 2.0 + slackMentions: + - '<@USER_ID>' + - '' assets: - asset: ETH diff --git a/pkg/strategy/autoborrow/strategy.go b/pkg/strategy/autoborrow/strategy.go index eed96abbed..a263737a94 100644 --- a/pkg/strategy/autoborrow/strategy.go +++ b/pkg/strategy/autoborrow/strategy.go @@ -43,6 +43,8 @@ func init() { maxQuantityPerBorrow: 100.0 maxTotalBorrow: 10.0 */ + +// MarginAlert is used to send the slack mention alerts when the current margin is less than the required margin level type MarginAlert struct { CurrentMarginLevel fixedpoint.Value MinimalMarginLevel fixedpoint.Value @@ -78,6 +80,36 @@ func (m *MarginAlert) SlackAttachment() slack.Attachment { } } +// RepaidAlert +type RepaidAlert struct { + SessionName string + Asset string + Amount fixedpoint.Value + SlackMentions []string +} + +func (m *RepaidAlert) SlackAttachment() slack.Attachment { + return slack.Attachment{ + Color: "red", + Title: fmt.Sprintf("Margin Repaid on %s session", m.SessionName), + Text: strings.Join(m.SlackMentions, " "), + Fields: []slack.AttachmentField{ + { + Title: "Session", + Value: m.SessionName, + Short: true, + }, + { + Title: "Asset", + Value: m.Amount.String() + " " + m.Asset, + Short: true, + }, + }, + // Footer: "", + // FooterIcon: "", + } +} + type MarginAsset struct { Asset string `json:"asset"` Low fixedpoint.Value `json:"low"` @@ -87,15 +119,24 @@ type MarginAsset struct { DebtRatio fixedpoint.Value `json:"debtRatio"` } +type MarginLevelAlert struct { + Interval types.Duration `json:"interval"` + MinMargin fixedpoint.Value `json:"minMargin"` + SlackMentions []string `json:"slackMentions"` +} + +type MarginRepayAlert struct { + SlackMentions []string `json:"slackMentions"` +} + type Strategy struct { Interval types.Interval `json:"interval"` MinMarginLevel fixedpoint.Value `json:"minMarginLevel"` MaxMarginLevel fixedpoint.Value `json:"maxMarginLevel"` AutoRepayWhenDeposit bool `json:"autoRepayWhenDeposit"` - MarginLevelAlertInterval types.Duration `json:"marginLevelAlertInterval"` - MarginLevelAlertMinMargin fixedpoint.Value `json:"marginLevelAlertMinMargin"` - MarginLevelAlertSlackMentions []string `json:"marginLevelAlertSlackMentions"` + MarginLevelAlert *MarginLevelAlert `json:"marginLevelAlert"` + MarginRepayAlert *MarginRepayAlert `json:"marginRepayAlert"` Assets []MarginAsset `json:"assets"` @@ -155,6 +196,15 @@ func (s *Strategy) tryToRepayAnyDebt(ctx context.Context) { log.WithError(err).Errorf("margin repay error") } + if s.MarginRepayAlert != nil { + bbgo.Notify(&RepaidAlert{ + SessionName: s.ExchangeSession.Name, + Asset: b.Currency, + Amount: toRepay, + SlackMentions: s.MarginRepayAlert.SlackMentions, + }) + } + return } } @@ -255,6 +305,15 @@ func (s *Strategy) reBalanceDebt(ctx context.Context) { log.WithError(err).Errorf("margin repay error") } + if s.MarginRepayAlert != nil { + bbgo.Notify(&RepaidAlert{ + SessionName: s.ExchangeSession.Name, + Asset: b.Currency, + Amount: toRepay, + SlackMentions: s.MarginRepayAlert.SlackMentions, + }) + } + if accountUpdate, err2 := s.ExchangeSession.UpdateAccount(ctx); err2 != nil { log.WithError(err).Errorf("unable to update account") } else { @@ -584,10 +643,10 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } } - if !s.MarginLevelAlertMinMargin.IsZero() { + if s.MarginLevelAlert != nil && !s.MarginLevelAlert.MinMargin.IsZero() { alertInterval := time.Minute * 5 - if s.MarginLevelAlertInterval > 0 { - alertInterval = s.MarginLevelAlertInterval.Duration() + if s.MarginLevelAlert.Interval > 0 { + alertInterval = s.MarginLevelAlert.Interval.Duration() } go s.marginAlertWorker(ctx, alertInterval) @@ -607,11 +666,11 @@ func (s *Strategy) marginAlertWorker(ctx context.Context, alertInterval time.Dur return case <-ticker.C: account := s.ExchangeSession.GetAccount() - if account.MarginLevel.Compare(s.MarginLevelAlertMinMargin) <= 0 { + if s.MarginLevelAlert != nil && account.MarginLevel.Compare(s.MarginLevelAlert.MinMargin) <= 0 { bbgo.Notify(&MarginAlert{ CurrentMarginLevel: account.MarginLevel, - MinimalMarginLevel: s.MarginLevelAlertMinMargin, - SlackMentions: s.MarginLevelAlertSlackMentions, + MinimalMarginLevel: s.MarginLevelAlert.MinMargin, + SlackMentions: s.MarginLevelAlert.SlackMentions, SessionName: s.ExchangeSession.Name, }) bbgo.Notify(account.Balances().Debts()) From 147b31d81d8e084c74eb19ccb1b6da49c6ef4dd9 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 10:57:33 +0800 Subject: [PATCH 402/422] pkg/exchange: refactor account stream --- pkg/exchange/okex/parse.go | 49 +++++-- pkg/exchange/okex/parse_test.go | 182 ++++++++++++++++++++++++++ pkg/exchange/okex/stream.go | 43 +++--- pkg/exchange/okex/stream_callbacks.go | 12 -- pkg/exchange/okex/stream_test.go | 14 ++ 5 files changed, 253 insertions(+), 47 deletions(-) diff --git a/pkg/exchange/okex/parse.go b/pkg/exchange/okex/parse.go index df538b373e..193518474c 100644 --- a/pkg/exchange/okex/parse.go +++ b/pkg/exchange/okex/parse.go @@ -44,14 +44,12 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { return nil, err } if event.Event != "" { - // TODO: remove fastjson - return event, nil + return &event, nil } switch event.Arg.Channel { case ChannelAccount: - // TODO: remove fastjson - return parseAccount(v) + return parseAccount(event.Data) case ChannelBooks, ChannelBook5: var bookEvent BookEvent @@ -100,10 +98,17 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { return nil, nil } +type WsEventType string + +const ( + WsEventTypeLogin = "login" + WsEventTypeError = "error" +) + type WebSocketEvent struct { - Event string `json:"event"` - Code string `json:"code,omitempty"` - Message string `json:"msg,omitempty"` + Event WsEventType `json:"event"` + Code string `json:"code,omitempty"` + Message string `json:"msg,omitempty"` Arg struct { Channel Channel `json:"channel"` InstId string `json:"instId"` @@ -112,6 +117,28 @@ type WebSocketEvent struct { ActionType ActionType `json:"action"` } +func (w *WebSocketEvent) IsValid() error { + switch w.Event { + case WsEventTypeError: + return fmt.Errorf("websocket request error, code: %s, msg: %s", w.Code, w.Message) + + case WsEventTypeLogin: + // Actually, this code is unnecessary because the events are either `Subscribe` or `Unsubscribe`, But to avoid bugs + // in the exchange, we still check. + if w.Code != "0" || len(w.Message) != 0 { + return fmt.Errorf("websocket request error, code: %s, msg: %s", w.Code, w.Message) + } + return nil + + default: + return fmt.Errorf("unexpected event type: %+v", w) + } +} + +func (w *WebSocketEvent) IsAuthenticated() bool { + return w.Event == WsEventTypeLogin && w.Code == "0" +} + type BookEvent struct { InstrumentID string Symbol string @@ -345,17 +372,15 @@ type KLineEvent struct { Channel Channel } -func parseAccount(v *fastjson.Value) (*okexapi.Account, error) { - data := v.Get("data").MarshalTo(nil) - +func parseAccount(v []byte) (*okexapi.Account, error) { var accounts []okexapi.Account - err := json.Unmarshal(data, &accounts) + err := json.Unmarshal(v, &accounts) if err != nil { return nil, err } if len(accounts) == 0 { - return nil, errors.New("empty account data") + return &okexapi.Account{}, nil } return &accounts[0], nil diff --git a/pkg/exchange/okex/parse_test.go b/pkg/exchange/okex/parse_test.go index 5ebcf19893..55c1d0e89e 100644 --- a/pkg/exchange/okex/parse_test.go +++ b/pkg/exchange/okex/parse_test.go @@ -11,6 +11,126 @@ import ( "github.com/c9s/bbgo/pkg/types" ) +func Test_parseWebSocketEvent_accountEvent(t *testing.T) { + t.Run("succeeds", func(t *testing.T) { + in := ` +{ + "arg": { + "channel": "account", + "uid": "77982378738415879" + }, + "data": [ + { + "uTime": "1614846244194", + "totalEq": "91884", + "adjEq": "91884.8502560037982063", + "isoEq": "0", + "ordFroz": "0", + "imr": "0", + "mmr": "0", + "borrowFroz": "", + "notionalUsd": "", + "mgnRatio": "100000", + "details": [{ + "availBal": "", + "availEq": "1", + "ccy": "BTC", + "cashBal": "1", + "uTime": "1617279471503", + "disEq": "50559.01", + "eq": "1", + "eqUsd": "45078", + "fixedBal": "0", + "frozenBal": "0", + "interest": "0", + "isoEq": "0", + "liab": "0", + "maxLoan": "", + "mgnRatio": "", + "notionalLever": "0", + "ordFrozen": "0", + "upl": "0", + "uplLiab": "0", + "crossLiab": "0", + "isoLiab": "0", + "coinUsdPrice": "60000", + "stgyEq":"0", + "spotInUseAmt":"", + "isoUpl":"", + "borrowFroz": "" + }, + { + "availBal": "", + "availEq": "41307", + "ccy": "USDT", + "cashBal": "41307", + "uTime": "1617279471503", + "disEq": "41325", + "eq": "41307", + "eqUsd": "45078", + "fixedBal": "0", + "frozenBal": "0", + "interest": "0", + "isoEq": "0", + "liab": "0", + "maxLoan": "", + "mgnRatio": "", + "notionalLever": "0", + "ordFrozen": "0", + "upl": "0", + "uplLiab": "0", + "crossLiab": "0", + "isoLiab": "0", + "coinUsdPrice": "1.00007", + "stgyEq":"0", + "spotInUseAmt":"", + "isoUpl":"", + "borrowFroz": "" + } + ] + } + ] +} +` + + exp := &okexapi.Account{ + TotalEquityInUSD: fixedpoint.NewFromFloat(91884), + UpdateTime: "1614846244194", + Details: []okexapi.BalanceDetail{ + { + Currency: "BTC", + Available: fixedpoint.NewFromFloat(1), + CashBalance: fixedpoint.NewFromFloat(1), + OrderFrozen: fixedpoint.Zero, + Frozen: fixedpoint.Zero, + Equity: fixedpoint.One, + EquityInUSD: fixedpoint.NewFromFloat(45078), + UpdateTime: types.NewMillisecondTimestampFromInt(1617279471503), + UnrealizedProfitAndLoss: fixedpoint.Zero, + }, + { + Currency: "USDT", + Available: fixedpoint.NewFromFloat(41307), + CashBalance: fixedpoint.NewFromFloat(41307), + OrderFrozen: fixedpoint.Zero, + Frozen: fixedpoint.Zero, + Equity: fixedpoint.NewFromFloat(41307), + EquityInUSD: fixedpoint.NewFromFloat(45078), + UpdateTime: types.NewMillisecondTimestampFromInt(1617279471503), + UnrealizedProfitAndLoss: fixedpoint.Zero, + }, + }, + } + + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + event, ok := res.(*okexapi.Account) + assert.True(t, ok) + assert.Equal(t, exp, event) + }) + +} + func TestParsePriceVolumeOrderSliceJSON(t *testing.T) { t.Run("snapshot", func(t *testing.T) { in := ` @@ -668,3 +788,65 @@ func Test_toGlobalTrade(t *testing.T) { assert.ErrorContains(t, err, "unexpected inst id") }) } + +func TestWebSocketEvent_IsValid(t *testing.T) { + t.Run("op login event", func(t *testing.T) { + input := `{ + "event": "login", + "code": "0", + "msg": "", + "connId": "a4d3ae55" +}` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WebSocketEvent) + assert.True(t, ok) + assert.Equal(t, WebSocketEvent{ + Event: WsEventTypeLogin, + Code: "0", + Message: "", + }, *opEvent) + + assert.NoError(t, opEvent.IsValid()) + }) + + t.Run("op error event", func(t *testing.T) { + input := `{ + "event": "error", + "code": "60009", + "msg": "Login failed.", + "connId": "a4d3ae55" +}` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WebSocketEvent) + assert.True(t, ok) + assert.Equal(t, WebSocketEvent{ + Event: WsEventTypeError, + Code: "60009", + Message: "Login failed.", + }, *opEvent) + + assert.ErrorContains(t, opEvent.IsValid(), "request error") + }) + + t.Run("unexpected event", func(t *testing.T) { + input := `{ + "event": "test gg", + "code": "60009", + "msg": "unexpected", + "connId": "a4d3ae55" +}` + res, err := parseWebSocketEvent([]byte(input)) + assert.NoError(t, err) + opEvent, ok := res.(*WebSocketEvent) + assert.True(t, ok) + assert.Equal(t, WebSocketEvent{ + Event: "test gg", + Code: "60009", + Message: "unexpected", + }, *opEvent) + + assert.ErrorContains(t, opEvent.IsValid(), "unexpected event type") + }) +} diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index 34d6c853ce..1d98e739be 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -35,7 +35,6 @@ type Stream struct { // public callbacks kLineEventCallbacks []func(candle KLineEvent) bookEventCallbacks []func(book BookEvent) - eventCallbacks []func(event WebSocketEvent) accountEventCallbacks []func(account okexapi.Account) orderDetailsEventCallbacks []func(orderDetails []okexapi.OrderDetails) marketTradeEventCallbacks []func(tradeDetail []MarketTradeEvent) @@ -56,8 +55,8 @@ func NewStream(client *okexapi.RestClient) *Stream { stream.OnAccountEvent(stream.handleAccountEvent) stream.OnMarketTradeEvent(stream.handleMarketTradeEvent) stream.OnOrderDetailsEvent(stream.handleOrderDetailsEvent) - stream.OnEvent(stream.handleEvent) stream.OnConnect(stream.handleConnect) + stream.OnAuth(stream.handleAuth) return stream } @@ -113,26 +112,19 @@ func (s *Stream) handleConnect() { } } -func (s *Stream) handleEvent(event WebSocketEvent) { - switch event.Event { - case "login": - if event.Code == "0" { - s.EmitAuth() - var subs = []WebsocketSubscription{ - {Channel: "account"}, - {Channel: "orders", InstrumentType: string(okexapi.InstrumentTypeSpot)}, - } - - log.Infof("subscribing private channels: %+v", subs) - err := s.Conn.WriteJSON(WebsocketOp{ - Op: "subscribe", - Args: subs, - }) +func (s *Stream) handleAuth() { + var subs = []WebsocketSubscription{ + {Channel: ChannelAccount}, + {Channel: "orders", InstrumentType: string(okexapi.InstrumentTypeSpot)}, + } - if err != nil { - log.WithError(err).Error("private channel subscribe error") - } - } + log.Infof("subscribing private channels: %+v", subs) + err := s.Conn.WriteJSON(WebsocketOp{ + Op: "subscribe", + Args: subs, + }) + if err != nil { + log.WithError(err).Error("private channel subscribe error") } } @@ -160,7 +152,7 @@ func (s *Stream) handleOrderDetailsEvent(orderDetails []okexapi.OrderDetails) { func (s *Stream) handleAccountEvent(account okexapi.Account) { balances := toGlobalBalance(&account) - s.EmitBalanceSnapshot(balances) + s.EmitBalanceUpdate(balances) } func (s *Stream) handleBookEvent(data BookEvent) { @@ -211,7 +203,12 @@ func (s *Stream) createEndpoint(ctx context.Context) (string, error) { func (s *Stream) dispatchEvent(e interface{}) { switch et := e.(type) { case *WebSocketEvent: - s.EmitEvent(*et) + if err := et.IsValid(); err != nil { + log.Errorf("invalid event: %v", err) + } + if et.IsAuthenticated() { + s.EmitAuth() + } case *BookEvent: // there's "books" for 400 depth and books5 for 5 depth diff --git a/pkg/exchange/okex/stream_callbacks.go b/pkg/exchange/okex/stream_callbacks.go index b735d09850..089da09aa9 100644 --- a/pkg/exchange/okex/stream_callbacks.go +++ b/pkg/exchange/okex/stream_callbacks.go @@ -26,16 +26,6 @@ func (s *Stream) EmitBookEvent(book BookEvent) { } } -func (s *Stream) OnEvent(cb func(event WebSocketEvent)) { - s.eventCallbacks = append(s.eventCallbacks, cb) -} - -func (s *Stream) EmitEvent(event WebSocketEvent) { - for _, cb := range s.eventCallbacks { - cb(event) - } -} - func (s *Stream) OnAccountEvent(cb func(account okexapi.Account)) { s.accountEventCallbacks = append(s.accountEventCallbacks, cb) } @@ -71,8 +61,6 @@ type StreamEventHub interface { OnBookEvent(cb func(book BookEvent)) - OnEvent(cb func(event WebSocketEvent)) - OnAccountEvent(cb func(account okexapi.Account)) OnOrderDetailsEvent(cb func(orderDetails []okexapi.OrderDetails)) diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go index b9b758a6aa..cf0125dedb 100644 --- a/pkg/exchange/okex/stream_test.go +++ b/pkg/exchange/okex/stream_test.go @@ -31,6 +31,20 @@ func TestStream(t *testing.T) { t.Skip() s := getTestClientOrSkip(t) + t.Run("account test", func(t *testing.T) { + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBalanceUpdate(func(balances types.BalanceMap) { + t.Log("got snapshot", balances) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", book) + }) + c := make(chan struct{}) + <-c + }) + t.Run("book test", func(t *testing.T) { s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ Depth: types.DepthLevel50, From ba5882f7b66d65aeb9e44a2900b03821402c6248 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 11:55:49 +0800 Subject: [PATCH 403/422] pkg/exchange: generate instrument request by requestgen --- pkg/exchange/okex/okexapi/client_test.go | 6 +- .../okexapi/get_instruments_info_request.go | 47 +++++ ...get_instruments_info_request_requestgen.go | 194 ++++++++++++++++++ pkg/exchange/okex/okexapi/public.go | 76 ------- 4 files changed, 243 insertions(+), 80 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_instruments_info_request.go create mode 100644 pkg/exchange/okex/okexapi/get_instruments_info_request_requestgen.go diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 1c78783304..51ea9e74c2 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -30,11 +30,9 @@ func getTestClientOrSkip(t *testing.T) *RestClient { func TestClient_GetInstrumentsRequest(t *testing.T) { client := NewClient() ctx := context.Background() - req := client.NewGetInstrumentsRequest() + req := client.NewGetInstrumentsInfoRequest() - instruments, err := req. - InstrumentType(InstrumentTypeSpot). - Do(ctx) + instruments, err := req.Do(ctx) assert.NoError(t, err) assert.NotEmpty(t, instruments) t.Logf("instruments: %+v", instruments) diff --git a/pkg/exchange/okex/okexapi/get_instruments_info_request.go b/pkg/exchange/okex/okexapi/get_instruments_info_request.go new file mode 100644 index 0000000000..25e8c8b301 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_instruments_info_request.go @@ -0,0 +1,47 @@ +package okexapi + +import ( + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type InstrumentInfo struct { + InstrumentType string `json:"instType"` + InstrumentID string `json:"instId"` + BaseCurrency string `json:"baseCcy"` + QuoteCurrency string `json:"quoteCcy"` + SettleCurrency string `json:"settleCcy"` + ContractValue string `json:"ctVal"` + ContractMultiplier string `json:"ctMult"` + ContractValueCurrency string `json:"ctValCcy"` + ListTime types.MillisecondTimestamp `json:"listTime"` + ExpiryTime types.MillisecondTimestamp `json:"expTime"` + TickSize fixedpoint.Value `json:"tickSz"` + LotSize fixedpoint.Value `json:"lotSz"` + + // MinSize = min order size + MinSize fixedpoint.Value `json:"minSz"` + + // instrument status + State string `json:"state"` +} + +//go:generate GetRequest -url "/api/v5/public/instruments" -type GetInstrumentsInfoRequest -responseDataType []InstrumentInfo +type GetInstrumentsInfoRequest struct { + client requestgen.APIClient + + instType InstrumentType `param:"instType,query" validValues:"SPOT"` + + instId *string `param:"instId,query"` +} + +func (c *RestClient) NewGetInstrumentsInfoRequest() *GetInstrumentsInfoRequest { + return &GetInstrumentsInfoRequest{ + client: c, + instType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_instruments_info_request_requestgen.go b/pkg/exchange/okex/okexapi/get_instruments_info_request_requestgen.go new file mode 100644 index 0000000000..7302d935f3 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_instruments_info_request_requestgen.go @@ -0,0 +1,194 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/public/instruments -type GetInstrumentsInfoRequest -responseDataType []InstrumentInfo"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (g *GetInstrumentsInfoRequest) InstType(instType InstrumentType) *GetInstrumentsInfoRequest { + g.instType = instType + return g +} + +func (g *GetInstrumentsInfoRequest) InstId(instId string) *GetInstrumentsInfoRequest { + g.instId = &instId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetInstrumentsInfoRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instType field -> json key instType + instType := g.instType + + // TEMPLATE check-valid-values + switch instType { + case "SPOT": + params["instType"] = instType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instType + params["instType"] = instType + // check instId field -> json key instId + if g.instId != nil { + instId := *g.instId + + // assign parameter of instId + params["instId"] = instId + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetInstrumentsInfoRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetInstrumentsInfoRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetInstrumentsInfoRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetInstrumentsInfoRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetInstrumentsInfoRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetInstrumentsInfoRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetInstrumentsInfoRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetInstrumentsInfoRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetInstrumentsInfoRequest) GetPath() string { + return "/api/v5/public/instruments" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetInstrumentsInfoRequest) Do(ctx context.Context) ([]InstrumentInfo, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []InstrumentInfo + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/public.go b/pkg/exchange/okex/okexapi/public.go index af4b45496c..131d013432 100644 --- a/pkg/exchange/okex/okexapi/public.go +++ b/pkg/exchange/okex/okexapi/public.go @@ -10,12 +10,6 @@ import ( "github.com/pkg/errors" ) -func (s *RestClient) NewGetInstrumentsRequest() *GetInstrumentsRequest { - return &GetInstrumentsRequest{ - client: s, - } -} - func (s *RestClient) NewGetFundingRate() *GetFundingRateRequest { return &GetFundingRateRequest{ client: s, @@ -71,73 +65,3 @@ func (r *GetFundingRateRequest) Do(ctx context.Context) (*FundingRate, error) { return &data[0], nil } - -type Instrument struct { - InstrumentType string `json:"instType"` - InstrumentID string `json:"instId"` - BaseCurrency string `json:"baseCcy"` - QuoteCurrency string `json:"quoteCcy"` - SettleCurrency string `json:"settleCcy"` - ContractValue string `json:"ctVal"` - ContractMultiplier string `json:"ctMult"` - ContractValueCurrency string `json:"ctValCcy"` - ListTime types.MillisecondTimestamp `json:"listTime"` - ExpiryTime types.MillisecondTimestamp `json:"expTime"` - TickSize fixedpoint.Value `json:"tickSz"` - LotSize fixedpoint.Value `json:"lotSz"` - - // MinSize = min order size - MinSize fixedpoint.Value `json:"minSz"` - - // instrument status - State string `json:"state"` -} - -type GetInstrumentsRequest struct { - client *RestClient - - instType InstrumentType - - instId *string -} - -func (r *GetInstrumentsRequest) InstrumentType(instType InstrumentType) *GetInstrumentsRequest { - r.instType = instType - return r -} - -func (r *GetInstrumentsRequest) InstrumentID(instId string) *GetInstrumentsRequest { - r.instId = &instId - return r -} - -func (r *GetInstrumentsRequest) Do(ctx context.Context) ([]Instrument, error) { - // SPOT, SWAP, FUTURES, OPTION - var params = url.Values{} - params.Add("instType", string(r.instType)) - - if r.instId != nil { - params.Add("instId", *r.instId) - } - - req, err := r.client.NewRequest(ctx, "GET", "/api/v5/public/instruments", params, nil) - if err != nil { - return nil, err - } - - response, err := r.client.SendRequest(req) - if err != nil { - return nil, err - } - - var apiResponse APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err - } - var data []Instrument - if err := json.Unmarshal(apiResponse.Data, &data); err != nil { - return nil, err - } - - return data, nil -} From 6e160e7a3653aa0cd0fc1328379924a3743bb833 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 11:56:10 +0800 Subject: [PATCH 404/422] pkg/exchange: add rate limiter to QueryMarkets --- pkg/exchange/okex/exchange.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index a631629080..ba18389b30 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -3,7 +3,6 @@ package okex import ( "context" "fmt" - "math" "strconv" "time" @@ -24,6 +23,8 @@ import ( var ( marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) + + queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) ) const ID = "okex" @@ -68,10 +69,11 @@ func (e *Exchange) Name() types.ExchangeName { } func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { - instruments, err := e.client.NewGetInstrumentsRequest(). - InstrumentType(okexapi.InstrumentTypeSpot). - Do(ctx) + if err := queryMarketLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("markets rate limiter wait error: %w", err) + } + instruments, err := e.client.NewGetInstrumentsInfoRequest().Do(ctx) if err != nil { return nil, err } @@ -87,8 +89,8 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { BaseCurrency: instrument.BaseCurrency, // convert tick size OKEx to precision - PricePrecision: int(-math.Log10(instrument.TickSize.Float64())), - VolumePrecision: int(-math.Log10(instrument.LotSize.Float64())), + PricePrecision: instrument.TickSize.NumFractionalDigits(), + VolumePrecision: instrument.LotSize.NumFractionalDigits(), // TickSize: OKEx's price tick, for BTC-USDT it's "0.1" TickSize: instrument.TickSize, From caef31d760592093b96f353c4bce4ee43b917d52 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 11:58:43 +0800 Subject: [PATCH 405/422] pkg/exchange: early return if error --- pkg/exchange/bitget/stream.go | 1 + pkg/exchange/okex/stream.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index 2c99d265da..23195919ee 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -122,6 +122,7 @@ func (s *Stream) dispatchEvent(event interface{}) { case *WsEvent: if err := e.IsValid(); err != nil { log.Errorf("invalid event: %v", err) + return } if e.IsAuthenticated() { s.EmitAuth() diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index 1d98e739be..c27fedbfa4 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -205,6 +205,7 @@ func (s *Stream) dispatchEvent(e interface{}) { case *WebSocketEvent: if err := et.IsValid(); err != nil { log.Errorf("invalid event: %v", err) + return } if et.IsAuthenticated() { s.EmitAuth() From 6d7a01ffaea45b5aa952fe69f0121056df8d6325 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 13:57:03 +0800 Subject: [PATCH 406/422] pkg/exchange: generate ticker request by requestgen --- pkg/exchange/okex/okexapi/client.go | 85 -------- pkg/exchange/okex/okexapi/client_test.go | 22 +++ .../okex/okexapi/get_ticker_request.go | 21 ++ .../okexapi/get_ticker_request_requestgen.go | 170 ++++++++++++++++ .../okex/okexapi/get_tickers_request.go | 51 +++++ .../okexapi/get_tickers_request_requestgen.go | 181 ++++++++++++++++++ 6 files changed, 445 insertions(+), 85 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_ticker_request.go create mode 100644 pkg/exchange/okex/okexapi/get_ticker_request_requestgen.go create mode 100644 pkg/exchange/okex/okexapi/get_tickers_request.go create mode 100644 pkg/exchange/okex/okexapi/get_tickers_request_requestgen.go diff --git a/pkg/exchange/okex/okexapi/client.go b/pkg/exchange/okex/okexapi/client.go index 5b0cb7c91b..c7dd5021b9 100644 --- a/pkg/exchange/okex/okexapi/client.go +++ b/pkg/exchange/okex/okexapi/client.go @@ -7,7 +7,6 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" - "fmt" "net/http" "net/url" "strings" @@ -273,90 +272,6 @@ func (c *RestClient) AssetCurrencies(ctx context.Context) ([]AssetCurrency, erro return currencyResponse.Data, nil } -type MarketTicker struct { - InstrumentType string `json:"instType"` - InstrumentID string `json:"instId"` - - // last traded price - Last fixedpoint.Value `json:"last"` - - // last traded size - LastSize fixedpoint.Value `json:"lastSz"` - - AskPrice fixedpoint.Value `json:"askPx"` - AskSize fixedpoint.Value `json:"askSz"` - - BidPrice fixedpoint.Value `json:"bidPx"` - BidSize fixedpoint.Value `json:"bidSz"` - - Open24H fixedpoint.Value `json:"open24h"` - High24H fixedpoint.Value `json:"high24H"` - Low24H fixedpoint.Value `json:"low24H"` - Volume24H fixedpoint.Value `json:"vol24h"` - VolumeCurrency24H fixedpoint.Value `json:"volCcy24h"` - - // Millisecond timestamp - Timestamp types.MillisecondTimestamp `json:"ts"` -} - -func (c *RestClient) MarketTicker(ctx context.Context, instId string) (*MarketTicker, error) { - // SPOT, SWAP, FUTURES, OPTION - var params = url.Values{} - params.Add("instId", instId) - - req, err := c.NewRequest(ctx, "GET", "/api/v5/market/ticker", params, nil) - if err != nil { - return nil, err - } - - response, err := c.SendRequest(req) - if err != nil { - return nil, err - } - - var tickerResponse struct { - Code string `json:"code"` - Message string `json:"msg"` - Data []MarketTicker `json:"data"` - } - if err := response.DecodeJSON(&tickerResponse); err != nil { - return nil, err - } - - if len(tickerResponse.Data) == 0 { - return nil, fmt.Errorf("ticker of %s not found", instId) - } - - return &tickerResponse.Data[0], nil -} - -func (c *RestClient) MarketTickers(ctx context.Context, instType InstrumentType) ([]MarketTicker, error) { - // SPOT, SWAP, FUTURES, OPTION - var params = url.Values{} - params.Add("instType", string(instType)) - - req, err := c.NewRequest(ctx, "GET", "/api/v5/market/tickers", params, nil) - if err != nil { - return nil, err - } - - response, err := c.SendRequest(req) - if err != nil { - return nil, err - } - - var tickerResponse struct { - Code string `json:"code"` - Message string `json:"msg"` - Data []MarketTicker `json:"data"` - } - if err := response.DecodeJSON(&tickerResponse); err != nil { - return nil, err - } - - return tickerResponse.Data, nil -} - func Sign(payload string, secret string) string { var sig = hmac.New(sha256.New, []byte(secret)) _, err := sig.Write([]byte(payload)) diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 51ea9e74c2..1d8dc9c67c 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -38,6 +38,28 @@ func TestClient_GetInstrumentsRequest(t *testing.T) { t.Logf("instruments: %+v", instruments) } +func TestClient_GetMarketTickers(t *testing.T) { + client := NewClient() + ctx := context.Background() + req := client.NewGetTickersRequest() + + tickers, err := req.Do(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, tickers) + t.Logf("tickers: %+v", tickers) +} + +func TestClient_GetMarketTicker(t *testing.T) { + client := NewClient() + ctx := context.Background() + req := client.NewGetTickerRequest().InstId("BTC-USDT") + + tickers, err := req.Do(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, tickers) + t.Logf("tickers: %+v", tickers) +} + func TestClient_GetFundingRateRequest(t *testing.T) { client := NewClient() ctx := context.Background() diff --git a/pkg/exchange/okex/okexapi/get_ticker_request.go b/pkg/exchange/okex/okexapi/get_ticker_request.go new file mode 100644 index 0000000000..a1c8c61a05 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_ticker_request.go @@ -0,0 +1,21 @@ +package okexapi + +import ( + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +//go:generate GetRequest -url "/api/v5/market/ticker" -type GetTickerRequest -responseDataType []MarketTicker +type GetTickerRequest struct { + client requestgen.APIClient + + instId string `param:"instId,query"` +} + +func (c *RestClient) NewGetTickerRequest() *GetTickerRequest { + return &GetTickerRequest{ + client: c, + } +} diff --git a/pkg/exchange/okex/okexapi/get_ticker_request_requestgen.go b/pkg/exchange/okex/okexapi/get_ticker_request_requestgen.go new file mode 100644 index 0000000000..e5ab52129c --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_ticker_request_requestgen.go @@ -0,0 +1,170 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/market/ticker -type GetTickerRequest -responseDataType []MarketTicker"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (g *GetTickerRequest) InstId(instId string) *GetTickerRequest { + g.instId = instId + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTickerRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instId field -> json key instId + instId := g.instId + + // assign parameter of instId + params["instId"] = instId + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTickerRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTickerRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTickerRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTickerRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTickerRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTickerRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTickerRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTickerRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetTickerRequest) GetPath() string { + return "/api/v5/market/ticker" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetTickerRequest) Do(ctx context.Context) ([]MarketTicker, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []MarketTicker + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/get_tickers_request.go b/pkg/exchange/okex/okexapi/get_tickers_request.go new file mode 100644 index 0000000000..fbd1107291 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_tickers_request.go @@ -0,0 +1,51 @@ +package okexapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type MarketTicker struct { + InstrumentType string `json:"instType"` + InstrumentID string `json:"instId"` + + // last traded price + Last fixedpoint.Value `json:"last"` + + // last traded size + LastSize fixedpoint.Value `json:"lastSz"` + + AskPrice fixedpoint.Value `json:"askPx"` + AskSize fixedpoint.Value `json:"askSz"` + + BidPrice fixedpoint.Value `json:"bidPx"` + BidSize fixedpoint.Value `json:"bidSz"` + + Open24H fixedpoint.Value `json:"open24h"` + High24H fixedpoint.Value `json:"high24H"` + Low24H fixedpoint.Value `json:"low24H"` + Volume24H fixedpoint.Value `json:"vol24h"` + VolumeCurrency24H fixedpoint.Value `json:"volCcy24h"` + + // Millisecond timestamp + Timestamp types.MillisecondTimestamp `json:"ts"` +} + +//go:generate GetRequest -url "/api/v5/market/tickers" -type GetTickersRequest -responseDataType []MarketTicker +type GetTickersRequest struct { + client requestgen.APIClient + + instType InstrumentType `param:"instType,query" validValues:"SPOT"` +} + +func (c *RestClient) NewGetTickersRequest() *GetTickersRequest { + return &GetTickersRequest{ + client: c, + instType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_tickers_request_requestgen.go b/pkg/exchange/okex/okexapi/get_tickers_request_requestgen.go new file mode 100644 index 0000000000..a847106f56 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_tickers_request_requestgen.go @@ -0,0 +1,181 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/market/tickers -type GetTickersRequest -responseDataType []MarketTicker"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (g *GetTickersRequest) InstType(instType InstrumentType) *GetTickersRequest { + g.instType = instType + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetTickersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instType field -> json key instType + instType := g.instType + + // TEMPLATE check-valid-values + switch instType { + case "SPOT": + params["instType"] = instType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instType + params["instType"] = instType + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetTickersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetTickersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetTickersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetTickersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetTickersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetTickersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetTickersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetTickersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetTickersRequest) GetPath() string { + return "/api/v5/market/tickers" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetTickersRequest) Do(ctx context.Context) ([]MarketTicker, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []MarketTicker + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} From 188b781116db0fa5d195e9c4658793a0b4b93115 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 13:57:19 +0800 Subject: [PATCH 407/422] pkg/exchange: add rate limiter to ticker/tickers --- pkg/exchange/okex/exchange.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index ba18389b30..fe3c5840f3 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -24,7 +24,9 @@ var ( marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) - queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryTickerLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryTickersLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) ) const ID = "okex" @@ -112,18 +114,29 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { } func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { - symbol = toLocalSymbol(symbol) + if err := queryTickerLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("ticker rate limiter wait error: %w", err) + } - marketTicker, err := e.client.MarketTicker(ctx, symbol) + symbol = toLocalSymbol(symbol) + marketTicker, err := e.client.NewGetTickerRequest().InstId(symbol).Do(ctx) if err != nil { return nil, err } - return toGlobalTicker(*marketTicker), nil + if len(marketTicker) != 1 { + return nil, fmt.Errorf("unexpected length of %s market ticker, got: %v", symbol, marketTicker) + } + + return toGlobalTicker(marketTicker[0]), nil } func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[string]types.Ticker, error) { - marketTickers, err := e.client.MarketTickers(ctx, okexapi.InstrumentTypeSpot) + if err := queryTickersLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("tickers rate limiter wait error: %w", err) + } + + marketTickers, err := e.client.NewGetTickersRequest().Do(ctx) if err != nil { return nil, err } From a463c02183c0ebfef8f10de0fa827e8e9c2b8ba4 Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 14:20:56 +0800 Subject: [PATCH 408/422] pkg/exchange: generate account by requestgen --- pkg/exchange/okex/okexapi/client.go | 47 ------ pkg/exchange/okex/okexapi/client_test.go | 11 ++ .../okex/okexapi/get_account_info_request.go | 40 +++++ .../get_account_info_request_requestgen.go | 157 ++++++++++++++++++ 4 files changed, 208 insertions(+), 47 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_account_info_request.go create mode 100644 pkg/exchange/okex/okexapi/get_account_info_request_requestgen.go diff --git a/pkg/exchange/okex/okexapi/client.go b/pkg/exchange/okex/okexapi/client.go index c7dd5021b9..85b00b8072 100644 --- a/pkg/exchange/okex/okexapi/client.go +++ b/pkg/exchange/okex/okexapi/client.go @@ -13,7 +13,6 @@ import ( "time" "github.com/c9s/bbgo/pkg/fixedpoint" - "github.com/c9s/bbgo/pkg/types" "github.com/c9s/requestgen" "github.com/pkg/errors" ) @@ -158,52 +157,6 @@ func (c *RestClient) NewAuthenticatedRequest(ctx context.Context, method, refURL return req, nil } -type BalanceDetail struct { - Currency string `json:"ccy"` - Available fixedpoint.Value `json:"availEq"` - CashBalance fixedpoint.Value `json:"cashBal"` - OrderFrozen fixedpoint.Value `json:"ordFrozen"` - Frozen fixedpoint.Value `json:"frozenBal"` - Equity fixedpoint.Value `json:"eq"` - EquityInUSD fixedpoint.Value `json:"eqUsd"` - UpdateTime types.MillisecondTimestamp `json:"uTime"` - UnrealizedProfitAndLoss fixedpoint.Value `json:"upl"` -} - -type Account struct { - TotalEquityInUSD fixedpoint.Value `json:"totalEq"` - UpdateTime string `json:"uTime"` - Details []BalanceDetail `json:"details"` -} - -func (c *RestClient) AccountBalances(ctx context.Context) (*Account, error) { - req, err := c.NewAuthenticatedRequest(ctx, "GET", "/api/v5/account/balance", nil, nil) - if err != nil { - return nil, err - } - - response, err := c.SendRequest(req) - if err != nil { - return nil, err - } - - var balanceResponse struct { - Code string `json:"code"` - Message string `json:"msg"` - Data []Account `json:"data"` - } - - if err := response.DecodeJSON(&balanceResponse); err != nil { - return nil, err - } - - if len(balanceResponse.Data) == 0 { - return nil, errors.New("empty account data") - } - - return &balanceResponse.Data[0], nil -} - type AssetBalance struct { Currency string `json:"ccy"` Balance fixedpoint.Value `json:"bal"` diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 1d8dc9c67c..218dcfd674 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -60,6 +60,17 @@ func TestClient_GetMarketTicker(t *testing.T) { t.Logf("tickers: %+v", tickers) } +func TestClient_GetAcountInfo(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + req := client.NewGetAccountInfoRequest() + + acct, err := req.Do(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, acct) + t.Logf("acct: %+v", acct) +} + func TestClient_GetFundingRateRequest(t *testing.T) { client := NewClient() ctx := context.Background() diff --git a/pkg/exchange/okex/okexapi/get_account_info_request.go b/pkg/exchange/okex/okexapi/get_account_info_request.go new file mode 100644 index 0000000000..ae1d936a74 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_account_info_request.go @@ -0,0 +1,40 @@ +package okexapi + +import ( + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type BalanceDetail struct { + Currency string `json:"ccy"` + Available fixedpoint.Value `json:"availEq"` + CashBalance fixedpoint.Value `json:"cashBal"` + OrderFrozen fixedpoint.Value `json:"ordFrozen"` + Frozen fixedpoint.Value `json:"frozenBal"` + Equity fixedpoint.Value `json:"eq"` + EquityInUSD fixedpoint.Value `json:"eqUsd"` + UpdateTime types.MillisecondTimestamp `json:"uTime"` + UnrealizedProfitAndLoss fixedpoint.Value `json:"upl"` +} + +type Account struct { + TotalEquityInUSD fixedpoint.Value `json:"totalEq"` + UpdateTime types.MillisecondTimestamp `json:"uTime"` + Details []BalanceDetail `json:"details"` +} + +//go:generate GetRequest -url "/api/v5/account/balance" -type GetAccountInfoRequest -responseDataType []Account +type GetAccountInfoRequest struct { + client requestgen.AuthenticatedAPIClient +} + +func (c *RestClient) NewGetAccountInfoRequest() *GetAccountInfoRequest { + return &GetAccountInfoRequest{ + client: c, + } +} diff --git a/pkg/exchange/okex/okexapi/get_account_info_request_requestgen.go b/pkg/exchange/okex/okexapi/get_account_info_request_requestgen.go new file mode 100644 index 0000000000..b90836d196 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_account_info_request_requestgen.go @@ -0,0 +1,157 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/account/balance -type GetAccountInfoRequest -responseDataType []Account"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetAccountInfoRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetAccountInfoRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetAccountInfoRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetAccountInfoRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetAccountInfoRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetAccountInfoRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetAccountInfoRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetAccountInfoRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetAccountInfoRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetAccountInfoRequest) GetPath() string { + return "/api/v5/account/balance" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetAccountInfoRequest) Do(ctx context.Context) ([]Account, error) { + + // no body params + var params interface{} + query := url.Values{} + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []Account + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} From 9297293a4635d06c34bfa622be9393b64b35ee3a Mon Sep 17 00:00:00 2001 From: Edwin Date: Tue, 9 Jan 2024 14:23:39 +0800 Subject: [PATCH 409/422] pkg/exchange: refactor query account balance --- pkg/exchange/okex/exchange.go | 26 +++++++++++++++----------- pkg/exchange/okex/parse_test.go | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index fe3c5840f3..9d3eac7fbc 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -27,6 +27,7 @@ var ( queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) queryTickerLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) queryTickersLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryAccountLimiter = rate.NewLimiter(rate.Every(200*time.Millisecond), 5) ) const ID = "okex" @@ -167,28 +168,31 @@ func (e *Exchange) PlatformFeeCurrency() string { } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { - accountBalance, err := e.client.AccountBalances(ctx) + bals, err := e.QueryAccountBalances(ctx) if err != nil { return nil, err } - var account = types.Account{ - AccountType: types.AccountTypeSpot, - } - - var balanceMap = toGlobalBalance(accountBalance) - account.UpdateBalances(balanceMap) - return &account, nil + account := types.NewAccount() + account.UpdateBalances(bals) + return account, nil } func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { - accountBalances, err := e.client.AccountBalances(ctx) + if err := queryAccountLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("account rate limiter wait error: %w", err) + } + + accountBalances, err := e.client.NewGetAccountInfoRequest().Do(ctx) if err != nil { return nil, err } - var balanceMap = toGlobalBalance(accountBalances) - return balanceMap, nil + if len(accountBalances) != 1 { + return nil, fmt.Errorf("unexpected length of balances: %v", accountBalances) + } + + return toGlobalBalance(&accountBalances[0]), nil } func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (*types.Order, error) { diff --git a/pkg/exchange/okex/parse_test.go b/pkg/exchange/okex/parse_test.go index 55c1d0e89e..84395b58a5 100644 --- a/pkg/exchange/okex/parse_test.go +++ b/pkg/exchange/okex/parse_test.go @@ -95,7 +95,7 @@ func Test_parseWebSocketEvent_accountEvent(t *testing.T) { exp := &okexapi.Account{ TotalEquityInUSD: fixedpoint.NewFromFloat(91884), - UpdateTime: "1614846244194", + UpdateTime: types.NewMillisecondTimestampFromInt(1614846244194), Details: []okexapi.BalanceDetail{ { Currency: "BTC", From d3bc37f45e8d0387c769e976b643f8da29c32cee Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Tue, 9 Jan 2024 16:01:10 +0800 Subject: [PATCH 410/422] use CommonCallback and pull PersistenceTTL out --- pkg/strategy/dca2/open_position.go | 6 +- pkg/strategy/dca2/profit_stats.go | 107 +++++++----------------- pkg/strategy/dca2/recover.go | 8 +- pkg/strategy/dca2/state.go | 2 +- pkg/strategy/dca2/strategy.go | 49 ++++++++++- pkg/strategy/dca2/strategy_callbacks.go | 30 ------- pkg/types/callbacks.go | 37 ++++++++ 7 files changed, 123 insertions(+), 116 deletions(-) create mode 100644 pkg/types/callbacks.go diff --git a/pkg/strategy/dca2/open_position.go b/pkg/strategy/dca2/open_position.go index f40d17b94e..617a06c44b 100644 --- a/pkg/strategy/dca2/open_position.go +++ b/pkg/strategy/dca2/open_position.go @@ -61,7 +61,7 @@ func generateOpenPositionOrders(market types.Market, quoteInvestment, price, pri prices = append(prices, price) } - notional, orderNum := calculateNotionalAndNum(market, quoteInvestment, prices) + notional, orderNum := calculateNotionalAndNumOrders(market, quoteInvestment, prices) if orderNum == 0 { return nil, fmt.Errorf("failed to calculate notional and num of open position orders, price: %s, quote investment: %s", price, quoteInvestment) } @@ -87,9 +87,9 @@ func generateOpenPositionOrders(market types.Market, quoteInvestment, price, pri return submitOrders, nil } -// calculateNotionalAndNum calculates the notional and num of open position orders +// calculateNotionalAndNumOrders calculates the notional and num of open position orders // DCA2 is notional-based, every order has the same notional -func calculateNotionalAndNum(market types.Market, quoteInvestment fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { +func calculateNotionalAndNumOrders(market types.Market, quoteInvestment fixedpoint.Value, prices []fixedpoint.Value) (fixedpoint.Value, int) { for num := len(prices); num > 0; num-- { notional := quoteInvestment.Div(fixedpoint.NewFromInt(int64(num))) if notional.Compare(market.MinNotional) < 0 { diff --git a/pkg/strategy/dca2/profit_stats.go b/pkg/strategy/dca2/profit_stats.go index 5ceb21b7a2..a468e65cd2 100644 --- a/pkg/strategy/dca2/profit_stats.go +++ b/pkg/strategy/dca2/profit_stats.go @@ -1,9 +1,7 @@ package dca2 import ( - "context" "fmt" - "strconv" "strings" "time" @@ -11,6 +9,21 @@ import ( "github.com/c9s/bbgo/pkg/types" ) +type PersistenceTTL struct { + ttl time.Duration +} + +func (p *PersistenceTTL) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } + p.ttl = ttl +} + +func (p *PersistenceTTL) Expiration() time.Duration { + return p.ttl +} + type ProfitStats struct { Symbol string `json:"symbol"` Market types.Market `json:"market,omitempty"` @@ -19,13 +32,12 @@ type ProfitStats struct { Round int64 `json:"round,omitempty"` QuoteInvestment fixedpoint.Value `json:"quoteInvestment,omitempty"` - RoundProfit fixedpoint.Value `json:"roundProfit,omitempty"` - RoundFee map[string]fixedpoint.Value `json:"roundFee,omitempty"` - TotalProfit fixedpoint.Value `json:"totalProfit,omitempty"` - TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` + CurrentRoundProfit fixedpoint.Value `json:"currentRoundProfit,omitempty"` + CurrentRoundFee map[string]fixedpoint.Value `json:"currentRoundFee,omitempty"` + TotalProfit fixedpoint.Value `json:"totalProfit,omitempty"` + TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` - // ttl is the ttl to keep in persistence - ttl time.Duration + PersistenceTTL } func newProfitStats(market types.Market, quoteInvestment fixedpoint.Value) *ProfitStats { @@ -34,31 +46,20 @@ func newProfitStats(market types.Market, quoteInvestment fixedpoint.Value) *Prof Market: market, Round: 0, QuoteInvestment: quoteInvestment, - RoundFee: make(map[string]fixedpoint.Value), + CurrentRoundFee: make(map[string]fixedpoint.Value), TotalFee: make(map[string]fixedpoint.Value), } } -func (s *ProfitStats) SetTTL(ttl time.Duration) { - if ttl.Nanoseconds() <= 0 { - return - } - s.ttl = ttl -} - -func (s *ProfitStats) Expiration() time.Duration { - return s.ttl -} - func (s *ProfitStats) AddTrade(trade types.Trade) { - if s.RoundFee == nil { - s.RoundFee = make(map[string]fixedpoint.Value) + if s.CurrentRoundFee == nil { + s.CurrentRoundFee = make(map[string]fixedpoint.Value) } - if fee, ok := s.RoundFee[trade.FeeCurrency]; ok { - s.RoundFee[trade.FeeCurrency] = fee.Add(trade.Fee) + if fee, ok := s.CurrentRoundFee[trade.FeeCurrency]; ok { + s.CurrentRoundFee[trade.FeeCurrency] = fee.Add(trade.Fee) } else { - s.RoundFee[trade.FeeCurrency] = trade.Fee + s.CurrentRoundFee[trade.FeeCurrency] = trade.Fee } if s.TotalFee == nil { @@ -76,63 +77,19 @@ func (s *ProfitStats) AddTrade(trade types.Trade) { quoteQuantity = quoteQuantity.Neg() } - s.RoundProfit = s.RoundProfit.Add(quoteQuantity) + s.CurrentRoundProfit = s.CurrentRoundProfit.Add(quoteQuantity) s.TotalProfit = s.TotalProfit.Add(quoteQuantity) if s.Market.QuoteCurrency == trade.FeeCurrency { - s.RoundProfit.Sub(trade.Fee) + s.CurrentRoundProfit.Sub(trade.Fee) s.TotalProfit.Sub(trade.Fee) } } func (s *ProfitStats) NewRound() { s.Round++ - s.RoundProfit = fixedpoint.Zero - s.RoundFee = make(map[string]fixedpoint.Value) -} - -func (s *ProfitStats) CalculateProfitOfRound(ctx context.Context, exchange types.Exchange) error { - historyService, ok := exchange.(types.ExchangeTradeHistoryService) - if !ok { - return fmt.Errorf("exchange %s doesn't support ExchangeTradeHistoryService", exchange.Name()) - } - - queryService, ok := exchange.(types.ExchangeOrderQueryService) - if !ok { - return fmt.Errorf("exchange %s doesn't support ExchangeOrderQueryService", exchange.Name()) - } - - // query the orders of this round - orders, err := historyService.QueryClosedOrders(ctx, s.Symbol, time.Time{}, time.Time{}, s.FromOrderID) - if err != nil { - return err - } - - // query the trades of this round - for _, order := range orders { - if order.ExecutedQuantity.Sign() == 0 { - // skip no trade orders - continue - } - - trades, err := queryService.QueryOrderTrades(ctx, types.OrderQuery{ - Symbol: order.Symbol, - OrderID: strconv.FormatUint(order.OrderID, 10), - }) - - if err != nil { - return err - } - - for _, trade := range trades { - s.AddTrade(trade) - } - } - - s.FromOrderID = s.FromOrderID + 1 - s.QuoteInvestment = s.QuoteInvestment.Add(s.RoundProfit) - - return nil + s.CurrentRoundProfit = fixedpoint.Zero + s.CurrentRoundFee = make(map[string]fixedpoint.Value) } func (s *ProfitStats) String() string { @@ -141,9 +98,9 @@ func (s *ProfitStats) String() string { sb.WriteString(fmt.Sprintf("Round: %d\n", s.Round)) sb.WriteString(fmt.Sprintf("From Order ID: %d\n", s.FromOrderID)) sb.WriteString(fmt.Sprintf("Quote Investment: %s\n", s.QuoteInvestment)) - sb.WriteString(fmt.Sprintf("Round Profit: %s\n", s.RoundProfit)) + sb.WriteString(fmt.Sprintf("Current Round Profit: %s\n", s.CurrentRoundProfit)) sb.WriteString(fmt.Sprintf("Total Profit: %s\n", s.TotalProfit)) - for currency, fee := range s.RoundFee { + for currency, fee := range s.CurrentRoundFee { sb.WriteString(fmt.Sprintf("FEE (%s): %s\n", currency, fee)) } sb.WriteString("[------------------ Profit Stats ------------------]\n") diff --git a/pkg/strategy/dca2/recover.go b/pkg/strategy/dca2/recover.go index fd5de6fbf0..b96474b078 100644 --- a/pkg/strategy/dca2/recover.go +++ b/pkg/strategy/dca2/recover.go @@ -57,7 +57,7 @@ func (s *Strategy) recover(ctx context.Context) error { } // recover profit stats - recoverProfitStats(ctx, s.ProfitStats, s.Session.Exchange) + recoverProfitStats(ctx, s) // recover startTimeOfNextRound startTimeOfNextRound := recoverStartTimeOfNextRound(ctx, currentRound, s.CoolDownInterval) @@ -189,12 +189,12 @@ func recoverPosition(ctx context.Context, position *types.Position, queryService return nil } -func recoverProfitStats(ctx context.Context, profitStats *ProfitStats, exchange types.Exchange) error { - if profitStats == nil { +func recoverProfitStats(ctx context.Context, strategy *Strategy) error { + if strategy.ProfitStats == nil { return fmt.Errorf("profit stats is nil, please check it") } - profitStats.CalculateProfitOfRound(ctx, exchange) + strategy.CalculateProfitOfCurrentRound(ctx) return nil } diff --git a/pkg/strategy/dca2/state.go b/pkg/strategy/dca2/state.go index 53ee3386ce..38190d2d2e 100644 --- a/pkg/strategy/dca2/state.go +++ b/pkg/strategy/dca2/state.go @@ -193,7 +193,7 @@ func (s *Strategy) runTakeProfitReady(ctx context.Context, next State) { s.logger.Info("[State] TakeProfitReady - start reseting position and calculate quote investment for next round") // calculate profit stats - s.ProfitStats.CalculateProfitOfRound(ctx, s.Session.Exchange) + s.CalculateProfitOfCurrentRound(ctx) bbgo.Sync(ctx, s) s.EmitProfit(s.ProfitStats) diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index 3fec4a6c96..d355b5f1fc 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "strconv" "sync" "time" @@ -68,11 +69,9 @@ type Strategy struct { state State // callbacks - readyCallbacks []func() + types.CommonCallback positionCallbacks []func(*types.Position) profitCallbacks []func(*ProfitStats) - closedCallbacks []func() - errorCallbacks []func(error) } func (s *Strategy) ID() string { @@ -278,3 +277,47 @@ func (s *Strategy) CleanUp(ctx context.Context) error { bbgo.Sync(ctx, s) return err } + +func (s *Strategy) CalculateProfitOfCurrentRound(ctx context.Context) error { + historyService, ok := s.Session.Exchange.(types.ExchangeTradeHistoryService) + if !ok { + return fmt.Errorf("exchange %s doesn't support ExchangeTradeHistoryService", s.Session.Exchange.Name()) + } + + queryService, ok := s.Session.Exchange.(types.ExchangeOrderQueryService) + if !ok { + return fmt.Errorf("exchange %s doesn't support ExchangeOrderQueryService", s.Session.Exchange.Name()) + } + + // query the orders of this round + orders, err := historyService.QueryClosedOrders(ctx, s.Symbol, time.Time{}, time.Time{}, s.ProfitStats.FromOrderID) + if err != nil { + return err + } + + // query the trades of this round + for _, order := range orders { + if order.ExecutedQuantity.Sign() == 0 { + // skip no trade orders + continue + } + + trades, err := queryService.QueryOrderTrades(ctx, types.OrderQuery{ + Symbol: order.Symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }) + + if err != nil { + return err + } + + for _, trade := range trades { + s.ProfitStats.AddTrade(trade) + } + } + + s.ProfitStats.FromOrderID = s.ProfitStats.FromOrderID + 1 + s.ProfitStats.QuoteInvestment = s.ProfitStats.QuoteInvestment.Add(s.ProfitStats.CurrentRoundProfit) + + return nil +} diff --git a/pkg/strategy/dca2/strategy_callbacks.go b/pkg/strategy/dca2/strategy_callbacks.go index 64781b4b2c..febebd52e2 100644 --- a/pkg/strategy/dca2/strategy_callbacks.go +++ b/pkg/strategy/dca2/strategy_callbacks.go @@ -6,16 +6,6 @@ import ( "github.com/c9s/bbgo/pkg/types" ) -func (s *Strategy) OnReady(cb func()) { - s.readyCallbacks = append(s.readyCallbacks, cb) -} - -func (s *Strategy) EmitReady() { - for _, cb := range s.readyCallbacks { - cb() - } -} - func (s *Strategy) OnPosition(cb func(*types.Position)) { s.positionCallbacks = append(s.positionCallbacks, cb) } @@ -35,23 +25,3 @@ func (s *Strategy) EmitProfit(profitStats *ProfitStats) { cb(profitStats) } } - -func (s *Strategy) OnClosed(cb func()) { - s.closedCallbacks = append(s.closedCallbacks, cb) -} - -func (s *Strategy) EmitClosed() { - for _, cb := range s.closedCallbacks { - cb() - } -} - -func (s *Strategy) OnError(cb func(err error)) { - s.errorCallbacks = append(s.errorCallbacks, cb) -} - -func (s *Strategy) EmitError(err error) { - for _, cb := range s.errorCallbacks { - cb(err) - } -} diff --git a/pkg/types/callbacks.go b/pkg/types/callbacks.go new file mode 100644 index 0000000000..01a4af82bf --- /dev/null +++ b/pkg/types/callbacks.go @@ -0,0 +1,37 @@ +package types + +type CommonCallback struct { + readyCallbacks []func() + closedCallbacks []func() + errorCallbacks []func(error) +} + +func (c *CommonCallback) OnReady(cb func()) { + c.readyCallbacks = append(c.readyCallbacks, cb) +} + +func (c *CommonCallback) EmitReady() { + for _, cb := range c.readyCallbacks { + cb() + } +} + +func (c *CommonCallback) OnClosed(cb func()) { + c.closedCallbacks = append(c.closedCallbacks, cb) +} + +func (c *CommonCallback) EmitClosed() { + for _, cb := range c.closedCallbacks { + cb() + } +} + +func (c *CommonCallback) OnError(cb func(err error)) { + c.errorCallbacks = append(c.errorCallbacks, cb) +} + +func (c *CommonCallback) EmitError(err error) { + for _, cb := range c.errorCallbacks { + cb(err) + } +} From 1786e6f33c8f09428d3b70eaf45ab9fe2414bff5 Mon Sep 17 00:00:00 2001 From: ShihChi Huang Date: Sat, 6 Jan 2024 23:53:42 -0800 Subject: [PATCH 411/422] DOCS: Translate the README into zh_TW --- README.md | 7 +- README.zh_TW.md | 624 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 630 insertions(+), 1 deletion(-) create mode 100644 README.zh_TW.md diff --git a/README.md b/README.md index f217d8153f..458c1ea1d2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +* [English👈](./README.md) +* [中文](./README.zh_TW.md) + # BBGO A modern crypto trading bot framework written in Go. @@ -645,7 +648,9 @@ See [Contributing](./CONTRIBUTING.md) ### Financial Contributors - +[[Become a backer](https://opencollective.com/bbgo#backer)] + + ## BBGO Tokenomics diff --git a/README.zh_TW.md b/README.zh_TW.md new file mode 100644 index 0000000000..c0ae1396ef --- /dev/null +++ b/README.zh_TW.md @@ -0,0 +1,624 @@ +* [English](./README.md) +* [中文👈](./README.zh_TW.md) + +# BBGO + +一個用Go編寫的現代加密貨幣交易機器人框架。 +A modern crypto trading bot framework written in Go. + + +## 目前狀態 + +[![Go](https://github.com/c9s/bbgo/actions/workflows/go.yml/badge.svg?branch=main)](https://github.com/c9s/bbgo/actions/workflows/go.yml) +[![GoDoc](https://godoc.org/github.com/c9s/bbgo?status.svg)](https://pkg.go.dev/github.com/c9s/bbgo) +[![Go Report Card](https://goreportcard.com/badge/github.com/c9s/bbgo)](https://goreportcard.com/report/github.com/c9s/bbgo) +[![DockerHub](https://img.shields.io/docker/pulls/yoanlin/bbgo.svg)](https://hub.docker.com/r/yoanlin/bbgo) +[![Coverage Status](http://codecov.io/github/c9s/bbgo/coverage.svg?branch=main)](http://codecov.io/github/c9s/bbgo?branch=main) +open collective badge +open collective badge + +## 社群 + +[![Telegram Global](https://img.shields.io/badge/telegram-global-blue.svg)](https://t.me/bbgo_intl) +[![Telegram Taiwan](https://img.shields.io/badge/telegram-tw-blue.svg)](https://t.me/bbgocrypto) +[![Twitter](https://img.shields.io/twitter/follow/bbgotrading?label=Follow&style=social)](https://twitter.com/bbgotrading) + +## 你可以用 BBGO 做什麼 +### 交易機器人用戶 💁‍♀️ 💁‍♂️ +您可以使用 BBGO 運行內置策略。 + +### 策略開發者 🥷 +您可以使用 BBGO 的交易單元和回測單元來實現您自己的策略。 + +### 交易單元開發者 🧑‍💻 +您可以使用 BBGO 的底層共用交易所 API；目前它支持 4+ 個主要交易所，因此您不必重複實現。 + +## 特色 +* 交易所抽象介面。 +* 整合串流（用戶資料 websocket，市場資料 websocket）。 +* 通過 websocket 實時訂單簿整合。 +* TWAP 訂單執行支持。參見 [TWAP 訂單執行](./doc/topics/twap.md) +* 盈虧計算。 +* Slack / Telegram 通知。 +* 回測：基於K線的回測引擎。參見[回測](./doc/topics/back-testing.md) +* 內置參數優化工具。 +* 內置網格策略和許多其他內置策略。 +* 多交易所 session 支持：您可以連接到2個以上不同帳戶或子帳戶的交易所。 +* 類似於 `pandas.Series` 的指標介面 ([series](https://github.com/c9s/bbgo/blob/main/doc/development/series.md))([usage](https://github.com/c9s/bbgo/blob/main/doc/development/indicator.md)) + - [Accumulation/Distribution Indicator](./pkg/indicator/ad.go) + - [Arnaud Legoux Moving Average](./pkg/indicator/alma.go) + - [Average True Range](./pkg/indicator/atr.go) + - [Bollinger Bands](./pkg/indicator/boll.go) + - [Commodity Channel Index](./pkg/indicator/cci.go) + - [Cumulative Moving Average](./pkg/indicator/cma.go) + - [Double Exponential Moving Average](./pkg/indicator/dema.go) + - [Directional Movement Index](./pkg/indicator/dmi.go) + - [Brownian Motion's Drift Factor](./pkg/indicator/drift.go) + - [Ease of Movement](./pkg/indicator/emv.go) + - [Exponentially Weighted Moving Average](./pkg/indicator/ewma.go) + - [Hull Moving Average](./pkg/indicator/hull.go) + - [Trend Line (Tool)](./pkg/indicator/line.go) + - [Moving Average Convergence Divergence Indicator](./pkg/indicator/macd.go) + - [On-Balance Volume](./pkg/indicator/obv.go) + - [Pivot](./pkg/indicator/pivot.go) + - [Running Moving Average](./pkg/indicator/rma.go) + - [Relative Strength Index](./pkg/indicator/rsi.go) + - [Simple Moving Average](./pkg/indicator/sma.go) + - [Ehler's Super Smoother Filter](./pkg/indicator/ssf.go) + - [Stochastic Oscillator](./pkg/indicator/stoch.go) + - [SuperTrend](./pkg/indicator/supertrend.go) + - [Triple Exponential Moving Average](./pkg/indicator/tema.go) + - [Tillson T3 Moving Average](./pkg/indicator/till.go) + - [Triangular Moving Average](./pkg/indicator/tma.go) + - [Variable Index Dynamic Average](./pkg/indicator/vidya.go) + - [Volatility Indicator](./pkg/indicator/volatility.go) + - [Volume Weighted Average Price](./pkg/indicator/vwap.go) + - [Zero Lag Exponential Moving Average](./pkg/indicator/zlema.go) + - 更多... + +## 截圖 + +![BBGO 儀表板](assets/screenshots/dashboard.jpeg) + +![BBGO 回測報告](assets/screenshots/backtest-report.jpg) + +## 內建策略 + +| 策略 | 描述 | 交易類型 | 是否支援回測 | +|-------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------|------------------| +| grid | 第一代網格策略，提供更多的靈活性，但您需要準備庫存。 | maker | | +| grid2 | 第二代網格策略，可以將您的報價資產轉換成網格，支持基礎+報價模式。 | maker | | +| bollgrid | 實現了一個基本的網格策略，內置布林通道 (bollinger band)。 | maker | | +| xmaker | 跨交易所市場製造策略，它在另一邊對您的庫存風險進行對沖。 | maker | 不 | +| xnav | 這個策略幫助您記錄當前的淨資產價值。 | tool | 不 | +| xalign | 這個策略自動對齊您的餘額位置。 | tool | 不 | +| xfunding | 一種資金費率策略。 | funding | 不 | +| autoborrow | 這個策略使用保證金借入資產，幫助您保持最小餘額。 | tool | 不 | +| pivotshort | 這個策略找到支點低點並在價格突破前一低點時進行交易。 | long/short | | +| schedule | 這個策略定期以固定數量買賣，您可以將其用作單一的DCA，或補充像BNB這樣的費用資產。 | tool | +| irr | 這個策略基於預測的回報率開倉。 | long/short | | +| bollmaker | 這個策略持有長期多空倉位，在兩邊下單，並使用布林通道 (bollinger band) 控制倉位大小。| maker | | +| wall | 這個策略在訂單簿上創建一堵牆（大量訂單）。 | maker | 不 | +| scmaker | 這個市場製造策略是為穩定幣市場設計的，如USDC/USDT。 | maker | | +| drift | | long/short | | +| rsicross | 這個策略在快速 RSI 越過慢速 RSI 時開啟多倉，這是使用 v2 指標的演示策略。 | long/short | | +| marketcap | 這個策略實現了一個基於市值資本化重新平衡投資組合的策略。 | rebalance | 不 | +| supertrend | 這個策略使用 DEMA 和超級趨勢指標開啟多空倉位。 | long/short | | +| trendtrader | 這個策略基於趨勢線突破開啟多空倉位。 | long/short | | +| elliottwave | | long/short | | +| ewoDgtrd | | long/short | | +| fixedmaker | | maker | | +| factoryzoo | | long/short | | +| fmaker | | maker | | +| linregmaker | 一個基於線性回歸的市場製造商。 | maker | | +| convert | 轉換策略是一個幫助您將特定資產轉換為目標資產的工具。 | tool | 不 | + +## 已支援交易所 + +- Binance Spot Exchange (以及 binance.us) +- OKEx Spot Exchange +- Kucoin Spot Exchange +- MAX Spot Exchange (台灣交易所) +- Bitget Exchange +- Bybit Exchange + +## 文件 + +- [參考文件](doc/README.md) + +## 要求 + +* Go SDK 1.20 +* Linux / MacOS / Windows (WSL) +* 在您註冊賬戶後獲取您的交易所 API 密鑰和密碼（您可以選擇一個或多個交易所）： + - MAX: https://max.maicoin.com/signup?r=c7982718 + - Binance: https://accounts.binance.com/en/register?ref=38192708 + - OKEx: https://www.okex.com/join/2412712?src=from:ios-share + - Kucoin: https://www.kucoin.com/ucenter/signup?rcode=r3KX2D4 + +這個項目由一小群人維護和支持。如果您想支持這個項目，請使用上面提供的鏈接和推薦碼在交易所註冊。 + +## 安裝 + +### 從 binary 安裝 + +以下 script 將幫助你設置文件和 dotenv 文件： + +```sh +# 針對 Binance 交易所的網格交易策略 +bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/setup-grid.sh) binance + +# 針對 MAX 交易所的網格交易策略 +bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/setup-grid.sh) max + +# 針對 Binance 交易所的布林格網格交易策略 +bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/setup-bollgrid.sh) binance + +# 針對 MAX 交易所的布林格網格交易策略 +bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/setup-bollgrid.sh) max +``` + +如果您已經在某處有配置，則可能適合您的是僅下載腳本： + +```sh +bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/download.sh) +``` + +或者參考[發布頁面](https://github.com/c9s/bbgo/releases)並手動下載。 + +自 v2 起，我們添加了一個新的浮點實現 dnum，以支持更高精度的小數。要下載和設置，請參考[Dnum安裝](doc/topics/dnum-binary.md) + +### 一鍵Linode StackScript + +StackScript 允許您一鍵部署一個輕量級實體與 bbgo。 + +- BBGO grid on Binance +- BBGO grid USDT/TWD on MAX +- BBGO grid USDC/TWD on MAX +- BBGO grid LINK/TWD on MAX +- BBGO grid USDC/USDT on MAX +- BBGO grid on MAX +- BBGO bollmaker on Binance + +### 從程式碼構建 +參見[從程式碼構建](./doc/build-from-source.md) + +## 配置 + +添加您的 dotenv 文件： + +```sh +# 針對 Binance 交易所 +BINANCE_API_KEY= +BINANCE_API_SECRET= + +# 如果您想使用 binance.us，將此更改為1 +BINANCE_US=0 + +# 針對 MAX 交易所 +MAX_API_KEY= +MAX_API_SECRET= + +# 針對 OKEx 交易所 +OKEX_API_KEY= +OKEX_API_SECRET= +OKEX_API_PASSPHRASE + +# 針對 Kucoin 交易所 +KUCOIN_API_KEY= +KUCOIN_API_SECRET= +KUCOIN_API_PASSPHRASE= +KUCOIN_API_KEY_VERSION=2 + +# 針對 Bybit 交易所 +BYBIT_API_KEY= +BYBIT_API_SECRET= +``` + +準備您的dotenv文件 `.env.local` 和 BBGO yaml 配置文件 `bbgo.yaml`。 + +要檢查可用的環境變量，請參見[環境變量](./doc/configuration/envvars.md) + +最小的 bbgo.yaml 可以通過以下方式生成： + +```sh +curl -o bbgo.yaml https://raw.githubusercontent.com/c9s/bbgo/main/config/minimal.yaml +``` + +要運行策略 + +```sh +bbgo run +``` + +要啟動帶有前端儀表板的 bbgo + +```sh +bbgo run --enable-webserver +``` + +如果您想切換到另一個 dotenv 文件，您可以添加 `--dotenv` 選項或 `--config` : + +```sh +bbgo sync --dotenv .env.dev --config config/grid.yaml --session binance +``` + +要查詢轉賬歷史 + +```sh +bbgo transfer-history --session max --asset USDT --since "2019-01-01" +``` + + + +## 進階配置 + +### 與 Binance 同步系統時間 + + BBGO 提供了用於 UNIX 系統 / 子系統的腳本，以便與 Binance 同步日期。需要事先安裝 jq 和 bc。在 Ubuntu 中安裝相依套件，嘗試以下命令： + +```bash +sudo apt install -y bc jq +``` + +要同步日期，嘗試 + +```bash +sudo ./scripts/sync_time.sh +``` + +您還可以將腳本添加到 crontab 中，這樣系統時間就可以定期與 Binance 同步 + +### Testnet (Paper Trading) + +目前僅支持 [Binance Test Network](https://testnet.binance.vision) + +```bash +export PAPER_TRADE=1 +export DISABLE_MARKET_CACHE=1 # 測試網路支援的市場遠少於主網路 +``` + +### 通知 + +- [設定 Telegram 通知](./doc/configuration/telegram.md) +- [設定 Slack 通知](./doc/configuration/slack.md) + +### 同步交易資料 + +預設情況下， BBGO 不會從交易所同步您的交易資料，因此很難正確計算您的盈虧。 + +通過將交易和訂單同步到本地資料庫，您可以獲得一些好處，如盈虧計算、回測和資產計算。 + +您只能使用一個資料庫驅動程序 MySQL 或 SQLite 來存儲您的交易資料。 + +**注意**：SQLite 不完全支援，我們建議您使用 MySQL 而不是 SQLite。 + +配置 MySQL 資料庫 +要使用 MySQL 資料庫進行資料同步，首先您需要安裝 MySQL 服務器： +#### Configure MySQL Database + +```sh +# Ubuntu Linux +sudo apt-get install -y mysql-server + +# 對於更新的 Ubuntu Linux +sudo apt install -y mysql-server +``` + +或者[在 docker 中執行它](https://hub.docker.com/_/mysql) + +創建您的 mysql 資料庫： + +Create your mysql database: + +```sh +mysql -uroot -e "CREATE DATABASE bbgo CHARSET utf8" +``` + +然後將這些環境變數放入您的 `.env.local` 文件中： + +```sh +DB_DRIVER=mysql +DB_DSN="user:password@tcp(127.0.0.1:3306)/bbgo" +``` + +#### Configure Sqlite3 Database + +配置 Sqlite3 資料庫 +要使用 SQLite3 而不是 MySQL，只需將這些環境變數放入您的 `.env.local` 文件中： + +```sh +DB_DRIVER=sqlite3 +DB_DSN=bbgo.sqlite3 +``` + +## 同步您自己的交易資料 + +一旦您配置了資料庫，您就可以從交易所同步您自己的交易資料。 + +參見[配置私人交易資料同步](./doc/configuration/sync.md) + +## 使用 Redis 在 BBGO session 之間保持持久性 + +要使用 Redis，首先您需要安裝您的 Redis 服務器 + +```sh +# 對於 Ubuntu/Debian Linux +sudo apt-get install -y redis + +# 對於更新的 Ubuntu/Debian Linux +sudo apt install -y redis +``` + +在您的 `bbgo.yaml` 中設定以下環境變數： + +```yaml +persistence: + redis: + host: 127.0.0.1 # 指向您的 Redis 服務器的 IP 地址或主機名，如果與 BBGO 相同則為 127.0.0.1 + port: 6379 # Redis 服務器的端口，預設為 6379 + db: 0 # 使用的 DB 號碼。如果其他應用程序也在使用 Redis，您可以設置為另一個 DB 以避免衝突 +``` + +## 內建策略 + +查看策略目錄 [strategy](pkg/strategy) 以獲得所有內置策略： + +- `pricealert` 策略演示如何使用通知系統 [pricealert](pkg/strategy/pricealert)。參見[文件](./doc/strategy/pricealert.md). +- `buyandhold` 策略演示如何訂閱 kline 事件並提交市場訂單 [buyandhold](pkg/strategy/pricedrop) +- `bollgrid` 策略實現了一個基本的網格策略，使用內置的布林通道指標 [bollgrid](pkg/strategy/bollgrid) +- `grid` 策略實現了固定價格帶網格策略 [grid](pkg/strategy/grid)。參見[文件](./doc/strategy/grid.md). +- `supertrend` 策略使用 Supertrend 指標作為趨勢，並使用 DEMA 指標作為噪聲 +過濾器 [supertrend](pkg/strategy/supertrend)。參見[文件](./doc/strategy/supertrend.md). +- `support` 策略使用具有高交易量的 K 線作為支撐 [support](pkg/strategy/support). 參見[文件](./doc/strategy/support.md). +- `flashcrash` 策略實現了一個捕捉閃崩的策略 [flashcrash](pkg/strategy/flashcrash) +- `marketcap`策略實現了一個基於市場資本化重新平衡投資組合的策略 [marketcap](pkg/strategy/marketcap). 參見[文件](./doc/strategy/marketcap.md). +- `pivotshort` - 以做空為重點的策略。 +- `irr` - 回報率策略。 +- `drift` - 漂移策略。 +- `grid2` - 第二代網格策略。 + +要運行這些內置策略，只需修改配置文件以使配置適合您，例如，如果您想運行 `buyandhold` 策略 + +```sh +vim config/buyandhold.yaml + +# 使用配置運行 bbgo +bbgo run --config config/buyandhold.yaml +``` + +## 回測 + +參考[回測](./doc/topics/back-testing.md) + +## 添加策略 + +參見[開發策略](./doc/topics/developing-strategy.md) + +## 開發您自己的私人策略 + +創建您的 go 包，使用 `go mod`` 初始化存儲庫，並添加 bbgo 作為依賴： + +```sh +go mod init +go get github.com/c9s/bbgo@main +``` + +建立您的 go 套件，使用 go mod 初始化存儲庫，並添加 bbgo 作為依賴： + +```sh +vim strategy.go +``` + +您可以從 獲取策略骨架。 現在添加您的配置 + +```sh +mkdir config +(cd config && curl -o bbgo.yaml https://raw.githubusercontent.com/c9s/bbgo/main/config/minimal.yaml) +``` + +將您的策略包路徑添加到配置文件 `config/bbgo.yaml` + +```yaml +--- +build: + dir: build + imports: + - github.com/your_id/your_swing + targets: + - name: swing-amd64-linux + os: linux + arch: amd64 + - name: swing-amd64-darwin + os: darwin + arch: amd64 +``` + +運行 `bbgo run` 命令，bbgo 將編譯一個導入您策略的包裝 binary 文件： + +```sh +dotenv -f .env.local -- bbgo run --config config/bbgo.yaml +``` + +或者您可以通過以下方式構建您自己的包裝 binary 文件 + +```shell +bbgo build --config config/bbgo.yaml +``` + +參考 +- +- +- +- +- + +## 命令用法 + +### 向特定交易所 session 提交訂單 + +```shell +bbgo submit-order --session=okex --symbol=OKBUSDT --side=buy --price=10.0 --quantity=1 +``` + +### 列出特定交易所 session 的未平倉訂單 + +```sh +bbgo list-orders open --session=okex --symbol=OKBUSDT +bbgo list-orders open --session=max --symbol=MAXUSDT +bbgo list-orders open --session=binance --symbol=BNBUSDT +``` + +### 取消一個未平倉訂單 + +```shell +# 對於 okex，order id 和 symbol 都是必需的 +bbgo cancel-order --session=okex --order-id=318223238325248000 --symbol=OKBUSDT + +# 對於 max，您只需要提供您的 order id +bbgo cancel-order --session=max --order-id=1234566 +``` + +### 除錯用戶資料流 + +```shell +bbgo userdatastream --session okex +bbgo userdatastream --session max +bbgo userdatastream --session binance +``` + +## 動態注入 + +為了最小化策略代碼，bbgo 支持動態依賴注入。 + +在執行您的策略之前，如果 bbgo 發現使用 bbgo 組件的嵌入字段，則會將組件注入到您的策略對象中。例如： + +```go +type Strategy struct { + Symbol string `json:"symbol" + Market types.Market +} +``` + +支援的組件（目前僅限單一交易所策略） + +- `*bbgo.ExchangeSession` +- `bbgo.OrderExecutor` + +如果您的策略中有 `Symbol string` 字段，您的策略將被檢測為基於符號的策略，然後以下類型可以自動注入： + +- `types.Market` + +## 策略執行階段 + +1. 從配置文件加載配置。 +1. 分配並初始化交易所 session 。 +1. 將交易所 session 添加到環境（資料層）。 +1. 使用給定的環境初始化交易者對象（邏輯層）。 +1. 交易者初始化環境並啟動交易所連接。 +1. 依次調用 strategy.Run() 方法。 + +## 交易所 API 範例 + +請查看範例 [examples](examples) + +初始化 MAX API: + +```go +key := os.Getenv("MAX_API_KEY") +secret := os.Getenv("MAX_API_SECRET") + +maxRest := maxapi.NewRestClient(maxapi.ProductionAPIURL) +maxRest.Auth(key, secret) +``` + +創建用戶資料流以獲取訂單簿（深度） + +```go +stream := max.NewStream(key, secret) +stream.Subscribe(types.BookChannel, symbol, types.SubscribeOptions{}) + +streambook := types.NewStreamBook(symbol) +streambook.BindStream(stream) +``` + +## 部署 + +- [Helm Chart](./doc/deployment/helm-chart.md) +- 裸機或 VPS + +## 開發 + +- [添加新交易所](./doc/development/adding-new-exchange.md) +- [遷移](./doc/development/migration.md) + +### 設置您的本地存儲庫 + +1. 點擊 GitHub 儲存庫的 "Fork" 按鈕。 +1. 將你分叉的儲存庫複製到 `$GOPATH/github.com/c9s/bbgo`。 +1. 更改目錄到 `$GOPATH/github.com/c9s/bbgo`。 +1. 創建一個分支並開始你的開發。 +1. 測試你的更改。 +1. 將你的更改推送到你的分叉。 +1. 發送一個拉取請求。 + +### 測試桌面應用 + +對於 webview + +```sh +make embed && go run -tags web ./cmd/bbgo-webview +``` + +對於 lorca + +```sh +make embed && go run -tags web ./cmd/bbgo-lorca +``` + +## 常見問題 + +### 什麼是倉位 ? + +- 基礎貨幣 & 報價貨幣 +- 如何計算平均成本？ + +### 尋找新策略？ + +你可以寫一篇關於 BBGO 的文章，主題不限，750-1500 字以換取，我可以為你實現策略（取決於複雜性和努力程度）。如果你有興趣，可以在 telegram 或 twitter 私訊我，我們可以討論。 + +### 添加新的加密貨幣交易所支持？ + +如果你希望 BBGO 支持一個目前 BBGO 未包含的新加密貨幣交易所，我們可以為你實現。成本是 10 ETH。如果你對此感興趣，請在 telegram 私訊我。 + +## 社群 + +- Telegram +- Telegram (台灣社群) +- Twitter + +## 貢獻 + +參見[貢獻](./CONTRIBUTING.md) + +### 歡迎[抖內](https://opencollective.com/bbgo#backer) + + + +## BBGO 代幣經濟 + +為了支持 BBGO 的開發，我們創建了一個獎勵池來支持貢獻者，通過贈送 $BBG 代幣。查看詳情在 [$BBG 合約頁面](contracts/README.md) 和我們的[官方網站](https://bbgo.finance) + +## 支持者 + +- GitBook + +## 授權 + +AGPL 授權 \ No newline at end of file From 90ae2330c84168d8c8f06b303f43f4d9c9ca3d26 Mon Sep 17 00:00:00 2001 From: ShihChi Huang Date: Tue, 9 Jan 2024 21:30:56 -0800 Subject: [PATCH 412/422] doc: fix frontend path --- doc/build-from-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/build-from-source.md b/doc/build-from-source.md index d8057ee48a..2d66dac0a1 100644 --- a/doc/build-from-source.md +++ b/doc/build-from-source.md @@ -97,7 +97,7 @@ go run ./cmd/bbgo run You can also use the makefile to build bbgo: ```shell -cd frontend && yarn install +cd apps/frontend && yarn install make bbgo ``` From 1dedd32f4263284a9e9a292225d701fc6bd90791 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 10 Jan 2024 13:56:17 +0800 Subject: [PATCH 413/422] pkg/exchange: support unsubscribe and resubscribe --- pkg/exchange/okex/parse.go | 9 ++++-- pkg/exchange/okex/stream.go | 41 +++++++++++++++++++++++++++- pkg/exchange/okex/stream_test.go | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/okex/parse.go b/pkg/exchange/okex/parse.go index 193518474c..c09e21c15c 100644 --- a/pkg/exchange/okex/parse.go +++ b/pkg/exchange/okex/parse.go @@ -101,8 +101,10 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { type WsEventType string const ( - WsEventTypeLogin = "login" - WsEventTypeError = "error" + WsEventTypeLogin = "login" + WsEventTypeError = "error" + WsEventTypeSubscribe = "subscribe" + WsEventTypeUnsubscribe = "unsubscribe" ) type WebSocketEvent struct { @@ -122,6 +124,9 @@ func (w *WebSocketEvent) IsValid() error { case WsEventTypeError: return fmt.Errorf("websocket request error, code: %s, msg: %s", w.Code, w.Message) + case WsEventTypeSubscribe, WsEventTypeUnsubscribe: + return nil + case WsEventTypeLogin: // Actually, this code is unnecessary because the events are either `Subscribe` or `Unsubscribe`, But to avoid bugs // in the exchange, we still check. diff --git a/pkg/exchange/okex/stream.go b/pkg/exchange/okex/stream.go index c27fedbfa4..96b955c73d 100644 --- a/pkg/exchange/okex/stream.go +++ b/pkg/exchange/okex/stream.go @@ -2,6 +2,7 @@ package okex import ( "context" + "fmt" "golang.org/x/time/rate" "strconv" "time" @@ -15,7 +16,7 @@ var ( ) type WebsocketOp struct { - Op string `json:"op"` + Op WsEventType `json:"op"` Args interface{} `json:"args"` } @@ -60,6 +61,44 @@ func NewStream(client *okexapi.RestClient) *Stream { return stream } +func (s *Stream) syncSubscriptions(opType WsEventType) error { + if opType != WsEventTypeUnsubscribe && opType != WsEventTypeSubscribe { + return fmt.Errorf("unexpected subscription type: %v", opType) + } + + logger := log.WithField("opType", opType) + var topics []WebsocketSubscription + for _, subscription := range s.Subscriptions { + topic, err := convertSubscription(subscription) + if err != nil { + logger.WithError(err).Errorf("convert error, subscription: %+v", subscription) + return err + } + + topics = append(topics, topic) + } + + logger.Infof("%s channels: %+v", opType, topics) + if err := s.Conn.WriteJSON(WebsocketOp{ + Op: opType, + Args: topics, + }); err != nil { + logger.WithError(err).Error("failed to send request") + return err + } + + return nil +} + +func (s *Stream) Unsubscribe() { + // errors are handled in the syncSubscriptions, so they are skipped here. + _ = s.syncSubscriptions(WsEventTypeUnsubscribe) + s.Resubscribe(func(old []types.Subscription) (new []types.Subscription, err error) { + // clear the subscriptions + return []types.Subscription{}, nil + }) +} + func (s *Stream) handleConnect() { if s.PublicOnly { var subs []WebsocketSubscription diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go index cf0125dedb..a832767cf6 100644 --- a/pkg/exchange/okex/stream_test.go +++ b/pkg/exchange/okex/stream_test.go @@ -5,6 +5,7 @@ import ( "os" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" @@ -93,4 +94,50 @@ func TestStream(t *testing.T) { c := make(chan struct{}) <-c }) + + t.Run("Subscribe/Unsubscribe test", func(t *testing.T) { + s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ + Depth: types.DepthLevel50, + }) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBookSnapshot(func(book types.SliceOrderBook) { + t.Log("got snapshot", book) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", book) + }) + + <-time.After(5 * time.Second) + + s.Unsubscribe() + c := make(chan struct{}) + <-c + }) + + t.Run("Resubscribe test", func(t *testing.T) { + s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ + Depth: types.DepthLevel50, + }) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnBookSnapshot(func(book types.SliceOrderBook) { + t.Log("got snapshot", book) + }) + s.OnBookUpdate(func(book types.SliceOrderBook) { + t.Log("got update", book) + }) + + <-time.After(5 * time.Second) + + s.Resubscribe(func(old []types.Subscription) (new []types.Subscription, err error) { + return old, nil + }) + c := make(chan struct{}) + <-c + }) } From a7aa34c396d15c5ff0d47f305cf66b0e5ffff79e Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 10 Jan 2024 14:02:03 +0800 Subject: [PATCH 414/422] pkg/exchange: add comment --- pkg/exchange/okex/convert.go | 4 ++++ pkg/exchange/okex/stream_test.go | 6 +++--- pkg/types/stream.go | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index f1889bb0af..0551ef684a 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -97,6 +97,10 @@ func convertSubscription(s types.Subscription) (WebsocketSubscription, error) { }, nil case types.BookChannel: + if s.Options.Depth != types.DepthLevel400 { + return WebsocketSubscription{}, fmt.Errorf("%s depth not supported", s.Options.Depth) + } + return WebsocketSubscription{ Channel: ChannelBooks, InstrumentID: toLocalSymbol(s.Symbol), diff --git a/pkg/exchange/okex/stream_test.go b/pkg/exchange/okex/stream_test.go index a832767cf6..7f85973adb 100644 --- a/pkg/exchange/okex/stream_test.go +++ b/pkg/exchange/okex/stream_test.go @@ -48,7 +48,7 @@ func TestStream(t *testing.T) { t.Run("book test", func(t *testing.T) { s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ - Depth: types.DepthLevel50, + Depth: types.DepthLevel400, }) s.SetPublicOnly() err := s.Connect(context.Background()) @@ -97,7 +97,7 @@ func TestStream(t *testing.T) { t.Run("Subscribe/Unsubscribe test", func(t *testing.T) { s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ - Depth: types.DepthLevel50, + Depth: types.DepthLevel400, }) s.SetPublicOnly() err := s.Connect(context.Background()) @@ -119,7 +119,7 @@ func TestStream(t *testing.T) { t.Run("Resubscribe test", func(t *testing.T) { s.Subscribe(types.BookChannel, "BTCUSDT", types.SubscribeOptions{ - Depth: types.DepthLevel50, + Depth: types.DepthLevel400, }) s.SetPublicOnly() err := s.Connect(context.Background()) diff --git a/pkg/types/stream.go b/pkg/types/stream.go index f8aa209e13..aaa9efab5d 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -547,6 +547,7 @@ const ( DepthLevel20 Depth = "20" DepthLevel50 Depth = "50" DepthLevel200 Depth = "200" + DepthLevel400 Depth = "400" ) type Speed string From 6e661c805ac62ec70d9852173d7109f33261822f Mon Sep 17 00:00:00 2001 From: "chiahung.lin" Date: Wed, 10 Jan 2024 14:37:07 +0800 Subject: [PATCH 415/422] fix --- config/dca2.yaml | 3 ++- pkg/{types => strategy/common}/callbacks.go | 17 +++++++++-------- pkg/strategy/dca2/profit_stats.go | 18 +----------------- pkg/strategy/dca2/strategy.go | 15 ++++++++++++++- pkg/types/persistence_ttl.go | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 27 deletions(-) rename pkg/{types => strategy/common}/callbacks.go (51%) create mode 100644 pkg/types/persistence_ttl.go diff --git a/config/dca2.yaml b/config/dca2.yaml index 894afc6342..e14c2c9637 100644 --- a/config/dca2.yaml +++ b/config/dca2.yaml @@ -22,9 +22,10 @@ exchangeStrategies: - on: max dca2: symbol: ETHUSDT - short: false quoteInvestment: "200" maxOrderCount: 5 priceDeviation: "0.01" takeProfitRatio: "0.002" coolDownInterval: 180 + recoverWhenStart: true + keepOrdersWhenShutdown: true diff --git a/pkg/types/callbacks.go b/pkg/strategy/common/callbacks.go similarity index 51% rename from pkg/types/callbacks.go rename to pkg/strategy/common/callbacks.go index 01a4af82bf..3df6c875f4 100644 --- a/pkg/types/callbacks.go +++ b/pkg/strategy/common/callbacks.go @@ -1,36 +1,37 @@ -package types +package common -type CommonCallback struct { +//go:generate callbackgen -type StatusCallbacks +type StatusCallbacks struct { readyCallbacks []func() closedCallbacks []func() errorCallbacks []func(error) } -func (c *CommonCallback) OnReady(cb func()) { +func (c *StatusCallbacks) OnReady(cb func()) { c.readyCallbacks = append(c.readyCallbacks, cb) } -func (c *CommonCallback) EmitReady() { +func (c *StatusCallbacks) EmitReady() { for _, cb := range c.readyCallbacks { cb() } } -func (c *CommonCallback) OnClosed(cb func()) { +func (c *StatusCallbacks) OnClosed(cb func()) { c.closedCallbacks = append(c.closedCallbacks, cb) } -func (c *CommonCallback) EmitClosed() { +func (c *StatusCallbacks) EmitClosed() { for _, cb := range c.closedCallbacks { cb() } } -func (c *CommonCallback) OnError(cb func(err error)) { +func (c *StatusCallbacks) OnError(cb func(err error)) { c.errorCallbacks = append(c.errorCallbacks, cb) } -func (c *CommonCallback) EmitError(err error) { +func (c *StatusCallbacks) EmitError(err error) { for _, cb := range c.errorCallbacks { cb(err) } diff --git a/pkg/strategy/dca2/profit_stats.go b/pkg/strategy/dca2/profit_stats.go index a468e65cd2..2bde24197c 100644 --- a/pkg/strategy/dca2/profit_stats.go +++ b/pkg/strategy/dca2/profit_stats.go @@ -3,27 +3,11 @@ package dca2 import ( "fmt" "strings" - "time" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) -type PersistenceTTL struct { - ttl time.Duration -} - -func (p *PersistenceTTL) SetTTL(ttl time.Duration) { - if ttl.Nanoseconds() <= 0 { - return - } - p.ttl = ttl -} - -func (p *PersistenceTTL) Expiration() time.Duration { - return p.ttl -} - type ProfitStats struct { Symbol string `json:"symbol"` Market types.Market `json:"market,omitempty"` @@ -37,7 +21,7 @@ type ProfitStats struct { TotalProfit fixedpoint.Value `json:"totalProfit,omitempty"` TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` - PersistenceTTL + types.PersistenceTTL } func newProfitStats(market types.Market, quoteInvestment fixedpoint.Value) *ProfitStats { diff --git a/pkg/strategy/dca2/strategy.go b/pkg/strategy/dca2/strategy.go index d355b5f1fc..1717a1e5b3 100644 --- a/pkg/strategy/dca2/strategy.go +++ b/pkg/strategy/dca2/strategy.go @@ -10,6 +10,7 @@ import ( "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/strategy/common" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" "github.com/prometheus/client_golang/prometheus" @@ -69,7 +70,7 @@ type Strategy struct { state State // callbacks - types.CommonCallback + common.StatusCallbacks positionCallbacks []func(*types.Position) profitCallbacks []func(*ProfitStats) } @@ -297,11 +298,22 @@ func (s *Strategy) CalculateProfitOfCurrentRound(ctx context.Context) error { // query the trades of this round for _, order := range orders { + if order.OrderID > s.ProfitStats.FromOrderID { + s.ProfitStats.FromOrderID = order.OrderID + } + + // skip not this strategy order + if order.GroupID != s.OrderGroupID { + continue + } + if order.ExecutedQuantity.Sign() == 0 { // skip no trade orders continue } + s.logger.Infof("[DCA] calculate profit stats from order: %s", order.String()) + trades, err := queryService.QueryOrderTrades(ctx, types.OrderQuery{ Symbol: order.Symbol, OrderID: strconv.FormatUint(order.OrderID, 10), @@ -312,6 +324,7 @@ func (s *Strategy) CalculateProfitOfCurrentRound(ctx context.Context) error { } for _, trade := range trades { + s.logger.Infof("[DCA] calculate profit stats from trade: %s", trade.String()) s.ProfitStats.AddTrade(trade) } } diff --git a/pkg/types/persistence_ttl.go b/pkg/types/persistence_ttl.go new file mode 100644 index 0000000000..1b056ca710 --- /dev/null +++ b/pkg/types/persistence_ttl.go @@ -0,0 +1,18 @@ +package types + +import "time" + +type PersistenceTTL struct { + ttl time.Duration +} + +func (p *PersistenceTTL) SetTTL(ttl time.Duration) { + if ttl.Nanoseconds() <= 0 { + return + } + p.ttl = ttl +} + +func (p *PersistenceTTL) Expiration() time.Duration { + return p.ttl +} From 260eef3b0c0e73ebf92b707d515ca12fd0e73448 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 10 Jan 2024 10:31:27 +0800 Subject: [PATCH 416/422] pkg/exchange: generate place order request by requestgen --- pkg/exchange/okex/exchange.go | 72 +++-- pkg/exchange/okex/okexapi/client_test.go | 14 +- .../okex/okexapi/place_order_request.go | 67 ++++ .../okexapi/place_order_request_accessors.go | 151 --------- .../okexapi/place_order_request_requestgen.go | 305 ++++++++++++++++++ pkg/exchange/okex/okexapi/trade.go | 70 ---- 6 files changed, 421 insertions(+), 258 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/place_order_request.go delete mode 100644 pkg/exchange/okex/okexapi/place_order_request_accessors.go create mode 100644 pkg/exchange/okex/okexapi/place_order_request_requestgen.go diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 9d3eac7fbc..d3ca2e9397 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -28,6 +28,7 @@ var ( queryTickerLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) queryTickersLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) queryAccountLimiter = rate.NewLimiter(rate.Every(200*time.Millisecond), 5) + placeOrderLimiter = rate.NewLimiter(rate.Every(30*time.Millisecond), 30) ) const ID = "okex" @@ -198,63 +199,68 @@ func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (*types.Order, error) { orderReq := e.client.NewPlaceOrderRequest() - orderType, err := toLocalOrderType(order.Type) - if err != nil { - return nil, err - } - orderReq.InstrumentID(toLocalSymbol(order.Symbol)) orderReq.Side(toLocalSideType(order.Side)) - - if order.Market.Symbol != "" { - orderReq.Quantity(order.Market.FormatQuantity(order.Quantity)) - } else { - // TODO report error - orderReq.Quantity(order.Quantity.FormatString(8)) - } + orderReq.Size(order.Market.FormatQuantity(order.Quantity)) // set price field for limit orders switch order.Type { case types.OrderTypeStopLimit, types.OrderTypeLimit: - if order.Market.Symbol != "" { - orderReq.Price(order.Market.FormatPrice(order.Price)) + orderReq.Price(order.Market.FormatPrice(order.Price)) + case types.OrderTypeMarket: + // Because our order.Quantity unit is base coin, so we indicate the target currency to Base. + if order.Side == types.SideTypeBuy { + orderReq.Size(order.Market.FormatQuantity(order.Quantity)) + orderReq.TargetCurrency(okexapi.TargetCurrencyBase) } else { - // TODO report error - orderReq.Price(order.Price.FormatString(8)) + orderReq.Size(order.Market.FormatQuantity(order.Quantity)) + orderReq.TargetCurrency(okexapi.TargetCurrencyQuote) } } + orderType, err := toLocalOrderType(order.Type) + if err != nil { + return nil, err + } + switch order.TimeInForce { - case "FOK": + case types.TimeInForceFOK: orderReq.OrderType(okexapi.OrderTypeFOK) - case "IOC": + case types.TimeInForceIOC: orderReq.OrderType(okexapi.OrderTypeIOC) default: orderReq.OrderType(orderType) } - orderHead, err := orderReq.Do(ctx) + if err := placeOrderLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("place order rate limiter wait error: %w", err) + } + + _, err = strconv.ParseInt(order.ClientOrderID, 10, 64) if err != nil { - return nil, err + return nil, fmt.Errorf("client order id should be numberic: %s, err: %w", order.ClientOrderID, err) } + orderReq.ClientOrderID(order.ClientOrderID) - orderID, err := strconv.ParseInt(orderHead.OrderID, 10, 64) + orders, err := orderReq.Do(ctx) if err != nil { return nil, err } - return &types.Order{ - SubmitOrder: order, - Exchange: types.ExchangeOKEx, - OrderID: uint64(orderID), - Status: types.OrderStatusNew, - ExecutedQuantity: fixedpoint.Zero, - IsWorking: true, - CreationTime: types.Time(time.Now()), - UpdateTime: types.Time(time.Now()), - IsMargin: false, - IsIsolated: false, - }, nil + if len(orders) != 1 { + return nil, fmt.Errorf("unexpected length of order response: %v", orders) + } + + orderRes, err := e.QueryOrder(ctx, types.OrderQuery{ + Symbol: order.Symbol, + OrderID: orders[0].OrderID, + ClientOrderID: orders[0].ClientOrderID, + }) + if err != nil { + return nil, fmt.Errorf("failed to query order by id: %s, clientOrderId: %s, err: %w", orders[0].OrderID, orders[0].ClientOrderID, err) + } + + return orderRes, nil // TODO: move this to batch place orders interface /* diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 218dcfd674..91fe26395f 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -91,15 +91,21 @@ func TestClient_PlaceOrderRequest(t *testing.T) { order, err := req. InstrumentID("BTC-USDT"). - TradeMode("cash"). - Side(SideTypeBuy). + TradeMode(TradeModeCash). + Side(SideTypeSell). OrderType(OrderTypeLimit). - Price("15000"). - Quantity("0.0001"). + TargetCurrency(TargetCurrencyBase). + Price("48000"). + Size("0.001"). Do(ctx) assert.NoError(t, err) assert.NotEmpty(t, order) t.Logf("place order: %+v", order) + + c := client.NewGetOrderDetailsRequest().OrderID(order[0].OrderID).InstrumentID("BTC-USDT") + res, err := c.Do(ctx) + assert.NoError(t, err) + t.Log(res) } func TestClient_GetPendingOrderRequest(t *testing.T) { diff --git a/pkg/exchange/okex/okexapi/place_order_request.go b/pkg/exchange/okex/okexapi/place_order_request.go new file mode 100644 index 0000000000..c45a3d0eaa --- /dev/null +++ b/pkg/exchange/okex/okexapi/place_order_request.go @@ -0,0 +1,67 @@ +package okexapi + +import "github.com/c9s/requestgen" + +type TradeMode string + +const ( + TradeModeCash TradeMode = "cash" + TradeModeIsolated TradeMode = "isolated" + TradeModeCross TradeMode = "cross" +) + +type TargetCurrency string + +const ( + TargetCurrencyBase TargetCurrency = "base_ccy" + TargetCurrencyQuote TargetCurrency = "quote_ccy" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type OrderResponse struct { + OrderID string `json:"ordId"` + ClientOrderID string `json:"clOrdId"` + Tag string `json:"tag"` + Code string `json:"sCode"` + Message string `json:"sMsg"` +} + +//go:generate PostRequest -url "/api/v5/trade/order" -type PlaceOrderRequest -responseDataType []OrderResponse +type PlaceOrderRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentID string `param:"instId"` + + // tdMode + // margin mode: "cross", "isolated" + // non-margin mode cash + tradeMode TradeMode `param:"tdMode" validValues:"cross,isolated,cash"` + + // A combination of case-sensitive alphanumerics, all numbers, or all letters of up to 32 characters. + clientOrderID *string `param:"clOrdId"` + + // A combination of case-sensitive alphanumerics, all numbers, or all letters of up to 8 characters. + tag *string `param:"tag"` + + // "buy" or "sell" + side SideType `param:"side" validValues:"buy,sell"` + + orderType OrderType `param:"ordType"` + + size string `param:"sz"` + + // price + price *string `param:"px"` + + // Whether the target currency uses the quote or base currency. + // base_ccy: Base currency ,quote_ccy: Quote currency + // Only applicable to SPOT Market Orders + // Default is quote_ccy for buy, base_ccy for sell + targetCurrency *TargetCurrency `param:"tgtCcy" validValues:"quote_ccy,base_ccy"` +} + +func (c *RestClient) NewPlaceOrderRequest() *PlaceOrderRequest { + return &PlaceOrderRequest{client: c} +} diff --git a/pkg/exchange/okex/okexapi/place_order_request_accessors.go b/pkg/exchange/okex/okexapi/place_order_request_accessors.go deleted file mode 100644 index b272cee113..0000000000 --- a/pkg/exchange/okex/okexapi/place_order_request_accessors.go +++ /dev/null @@ -1,151 +0,0 @@ -// Code generated by "requestgen -type PlaceOrderRequest"; DO NOT EDIT. - -package okexapi - -import ( - "encoding/json" - "fmt" - "net/url" -) - -func (p *PlaceOrderRequest) InstrumentID(instrumentID string) *PlaceOrderRequest { - p.instrumentID = instrumentID - return p -} - -func (p *PlaceOrderRequest) TradeMode(tradeMode string) *PlaceOrderRequest { - p.tradeMode = tradeMode - return p -} - -func (p *PlaceOrderRequest) ClientOrderID(clientOrderID string) *PlaceOrderRequest { - p.clientOrderID = &clientOrderID - return p -} - -func (p *PlaceOrderRequest) Tag(tag string) *PlaceOrderRequest { - p.tag = &tag - return p -} - -func (p *PlaceOrderRequest) Side(side SideType) *PlaceOrderRequest { - p.side = side - return p -} - -func (p *PlaceOrderRequest) OrderType(orderType OrderType) *PlaceOrderRequest { - p.orderType = orderType - return p -} - -func (p *PlaceOrderRequest) Quantity(quantity string) *PlaceOrderRequest { - p.quantity = quantity - return p -} - -func (p *PlaceOrderRequest) Price(price string) *PlaceOrderRequest { - p.price = &price - return p -} - -func (p *PlaceOrderRequest) GetParameters() (map[string]interface{}, error) { - var params = map[string]interface{}{} - - // check instrumentID field -> json key instId - instrumentID := p.instrumentID - - // assign parameter of instrumentID - params["instId"] = instrumentID - - // check tradeMode field -> json key tdMode - tradeMode := p.tradeMode - - switch tradeMode { - case "cross", "isolated", "cash": - params["tdMode"] = tradeMode - - default: - return params, fmt.Errorf("tdMode value %v is invalid", tradeMode) - - } - - // assign parameter of tradeMode - params["tdMode"] = tradeMode - - // check clientOrderID field -> json key clOrdId - if p.clientOrderID != nil { - clientOrderID := *p.clientOrderID - - // assign parameter of clientOrderID - params["clOrdId"] = clientOrderID - } - - // check tag field -> json key tag - if p.tag != nil { - tag := *p.tag - - // assign parameter of tag - params["tag"] = tag - } - - // check side field -> json key side - side := p.side - - switch side { - case "buy", "sell": - params["side"] = side - - default: - return params, fmt.Errorf("side value %v is invalid", side) - - } - - // assign parameter of side - params["side"] = side - - // check orderType field -> json key ordType - orderType := p.orderType - - // assign parameter of orderType - params["ordType"] = orderType - - // check quantity field -> json key sz - quantity := p.quantity - - // assign parameter of quantity - params["sz"] = quantity - - // check price field -> json key px - if p.price != nil { - price := *p.price - - // assign parameter of price - params["px"] = price - } - - return params, nil -} - -func (p *PlaceOrderRequest) GetParametersQuery() (url.Values, error) { - query := url.Values{} - - params, err := p.GetParameters() - if err != nil { - return query, err - } - - for k, v := range params { - query.Add(k, fmt.Sprintf("%v", v)) - } - - return query, nil -} - -func (p *PlaceOrderRequest) GetParametersJSON() ([]byte, error) { - params, err := p.GetParameters() - if err != nil { - return nil, err - } - - return json.Marshal(params) -} diff --git a/pkg/exchange/okex/okexapi/place_order_request_requestgen.go b/pkg/exchange/okex/okexapi/place_order_request_requestgen.go new file mode 100644 index 0000000000..3303d59612 --- /dev/null +++ b/pkg/exchange/okex/okexapi/place_order_request_requestgen.go @@ -0,0 +1,305 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v5/trade/order -type PlaceOrderRequest -responseDataType []OrderResponse"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (r *PlaceOrderRequest) InstrumentID(instrumentID string) *PlaceOrderRequest { + r.instrumentID = instrumentID + return r +} + +func (r *PlaceOrderRequest) TradeMode(tradeMode TradeMode) *PlaceOrderRequest { + r.tradeMode = tradeMode + return r +} + +func (r *PlaceOrderRequest) ClientOrderID(clientOrderID string) *PlaceOrderRequest { + r.clientOrderID = &clientOrderID + return r +} + +func (r *PlaceOrderRequest) Tag(tag string) *PlaceOrderRequest { + r.tag = &tag + return r +} + +func (r *PlaceOrderRequest) Side(side SideType) *PlaceOrderRequest { + r.side = side + return r +} + +func (r *PlaceOrderRequest) OrderType(orderType OrderType) *PlaceOrderRequest { + r.orderType = orderType + return r +} + +func (r *PlaceOrderRequest) Size(size string) *PlaceOrderRequest { + r.size = size + return r +} + +func (r *PlaceOrderRequest) Price(price string) *PlaceOrderRequest { + r.price = &price + return r +} + +func (r *PlaceOrderRequest) TargetCurrency(targetCurrency TargetCurrency) *PlaceOrderRequest { + r.targetCurrency = &targetCurrency + return r +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (r *PlaceOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (r *PlaceOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check instrumentID field -> json key instId + instrumentID := r.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + // check tradeMode field -> json key tdMode + tradeMode := r.tradeMode + + // TEMPLATE check-valid-values + switch tradeMode { + case "cross", "isolated", "cash": + params["tdMode"] = tradeMode + + default: + return nil, fmt.Errorf("tdMode value %v is invalid", tradeMode) + + } + // END TEMPLATE check-valid-values + + // assign parameter of tradeMode + params["tdMode"] = tradeMode + // check clientOrderID field -> json key clOrdId + if r.clientOrderID != nil { + clientOrderID := *r.clientOrderID + + // assign parameter of clientOrderID + params["clOrdId"] = clientOrderID + } else { + } + // check tag field -> json key tag + if r.tag != nil { + tag := *r.tag + + // assign parameter of tag + params["tag"] = tag + } else { + } + // check side field -> json key side + side := r.side + + // TEMPLATE check-valid-values + switch side { + case "buy", "sell": + params["side"] = side + + default: + return nil, fmt.Errorf("side value %v is invalid", side) + + } + // END TEMPLATE check-valid-values + + // assign parameter of side + params["side"] = side + // check orderType field -> json key ordType + orderType := r.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeMarket, OrderTypeLimit, OrderTypePostOnly, OrderTypeFOK, OrderTypeIOC: + params["ordType"] = orderType + + default: + return nil, fmt.Errorf("ordType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["ordType"] = orderType + // check size field -> json key sz + size := r.size + + // assign parameter of size + params["sz"] = size + // check price field -> json key px + if r.price != nil { + price := *r.price + + // assign parameter of price + params["px"] = price + } else { + } + // check targetCurrency field -> json key tgtCcy + if r.targetCurrency != nil { + targetCurrency := *r.targetCurrency + + // TEMPLATE check-valid-values + switch targetCurrency { + case "quote_ccy", "base_ccy": + params["tgtCcy"] = targetCurrency + + default: + return nil, fmt.Errorf("tgtCcy value %v is invalid", targetCurrency) + + } + // END TEMPLATE check-valid-values + + // assign parameter of targetCurrency + params["tgtCcy"] = targetCurrency + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (r *PlaceOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := r.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if r.isVarSlice(_v) { + r.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (r *PlaceOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := r.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (r *PlaceOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (r *PlaceOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (r *PlaceOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (r *PlaceOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (r *PlaceOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := r.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (r *PlaceOrderRequest) GetPath() string { + return "/api/v5/trade/order" +} + +// Do generates the request object and send the request object to the API endpoint +func (r *PlaceOrderRequest) Do(ctx context.Context) ([]OrderResponse, error) { + + params, err := r.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + var apiURL string + + apiURL = r.GetPath() + + req, err := r.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := r.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []OrderResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index f42553c517..09f03e360f 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -11,20 +11,6 @@ import ( "github.com/pkg/errors" ) -type OrderResponse struct { - OrderID string `json:"ordId"` - ClientOrderID string `json:"clOrdId"` - Tag string `json:"tag"` - Code string `json:"sCode"` - Message string `json:"sMsg"` -} - -func (c *RestClient) NewPlaceOrderRequest() *PlaceOrderRequest { - return &PlaceOrderRequest{ - client: c, - } -} - func (c *RestClient) NewBatchPlaceOrderRequest() *BatchPlaceOrderRequest { return &BatchPlaceOrderRequest{ client: c, @@ -61,67 +47,11 @@ func (c *RestClient) NewGetTransactionDetailsRequest() *GetTransactionDetailsReq } } -//go:generate requestgen -type PlaceOrderRequest -type PlaceOrderRequest struct { - client *RestClient - - instrumentID string `param:"instId"` - - // tdMode - // margin mode: "cross", "isolated" - // non-margin mode cash - tradeMode string `param:"tdMode" validValues:"cross,isolated,cash"` - - // A combination of case-sensitive alphanumerics, all numbers, or all letters of up to 32 characters. - clientOrderID *string `param:"clOrdId"` - - // A combination of case-sensitive alphanumerics, all numbers, or all letters of up to 8 characters. - tag *string `param:"tag"` - - // "buy" or "sell" - side SideType `param:"side" validValues:"buy,sell"` - - orderType OrderType `param:"ordType"` - - quantity string `param:"sz"` - - // price - price *string `param:"px"` -} - func (r *PlaceOrderRequest) Parameters() map[string]interface{} { params, _ := r.GetParameters() return params } -func (r *PlaceOrderRequest) Do(ctx context.Context) (*OrderResponse, error) { - payload := r.Parameters() - req, err := r.client.NewAuthenticatedRequest(ctx, "POST", "/api/v5/trade/order", nil, payload) - if err != nil { - return nil, err - } - - response, err := r.client.SendRequest(req) - if err != nil { - return nil, err - } - - var apiResponse APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err - } - var data []OrderResponse - if err := json.Unmarshal(apiResponse.Data, &data); err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, errors.New("order create error") - } - - return &data[0], nil -} - //go:generate requestgen -type CancelOrderRequest type CancelOrderRequest struct { client *RestClient From 373242d3064413464745c6ff91ba5173fff239f0 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 10 Jan 2024 16:39:52 +0800 Subject: [PATCH 417/422] pkg/exchange: generate cancel order by requestgen --- pkg/exchange/okex/exchange.go | 18 +- .../okex/okexapi/cancel_order_request.go | 21 ++ .../okexapi/cancel_order_request_accessors.go | 76 ------- .../cancel_order_request_requestgen.go | 195 ++++++++++++++++++ pkg/exchange/okex/okexapi/client_test.go | 62 ++++++ pkg/exchange/okex/okexapi/trade.go | 57 +---- 6 files changed, 295 insertions(+), 134 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/cancel_order_request.go delete mode 100644 pkg/exchange/okex/okexapi/cancel_order_request_accessors.go create mode 100644 pkg/exchange/okex/okexapi/cancel_order_request_requestgen.go diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index d3ca2e9397..0dcb7955c2 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -24,11 +24,12 @@ var ( marketDataLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 5) orderRateLimiter = rate.NewLimiter(rate.Every(300*time.Millisecond), 5) - queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - queryTickerLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - queryTickersLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - queryAccountLimiter = rate.NewLimiter(rate.Every(200*time.Millisecond), 5) - placeOrderLimiter = rate.NewLimiter(rate.Every(30*time.Millisecond), 30) + queryMarketLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryTickerLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryTickersLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) + queryAccountLimiter = rate.NewLimiter(rate.Every(200*time.Millisecond), 5) + placeOrderLimiter = rate.NewLimiter(rate.Every(30*time.Millisecond), 30) + batchCancelOrderLimiter = rate.NewLimiter(rate.Every(5*time.Millisecond), 200) ) const ID = "okex" @@ -321,11 +322,18 @@ func (e *Exchange) CancelOrders(ctx context.Context, orders ...types.Order) erro req.InstrumentID(toLocalSymbol(order.Symbol)) req.OrderID(strconv.FormatUint(order.OrderID, 10)) if len(order.ClientOrderID) > 0 { + _, err := strconv.ParseInt(order.ClientOrderID, 10, 64) + if err != nil { + return fmt.Errorf("client order id should be numberic: %s, err: %w", order.ClientOrderID, err) + } req.ClientOrderID(order.ClientOrderID) } reqs = append(reqs, req) } + if err := batchCancelOrderLimiter.Wait(ctx); err != nil { + return fmt.Errorf("batch cancel order rate limiter wait error: %w", err) + } batchReq := e.client.NewBatchCancelOrderRequest() batchReq.Add(reqs...) _, err := batchReq.Do(ctx) diff --git a/pkg/exchange/okex/okexapi/cancel_order_request.go b/pkg/exchange/okex/okexapi/cancel_order_request.go new file mode 100644 index 0000000000..3b5cd40175 --- /dev/null +++ b/pkg/exchange/okex/okexapi/cancel_order_request.go @@ -0,0 +1,21 @@ +package okexapi + +import "github.com/c9s/requestgen" + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +//go:generate PostRequest -url "/api/v5/trade/cancel-order" -type CancelOrderRequest -responseDataType []OrderResponse +type CancelOrderRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentID string `param:"instId"` + orderID *string `param:"ordId"` + clientOrderID *string `param:"clOrdId"` +} + +func (c *RestClient) NewCancelOrderRequest() *CancelOrderRequest { + return &CancelOrderRequest{ + client: c, + } +} diff --git a/pkg/exchange/okex/okexapi/cancel_order_request_accessors.go b/pkg/exchange/okex/okexapi/cancel_order_request_accessors.go deleted file mode 100644 index aaaf3060ba..0000000000 --- a/pkg/exchange/okex/okexapi/cancel_order_request_accessors.go +++ /dev/null @@ -1,76 +0,0 @@ -// Code generated by "requestgen -type CancelOrderRequest"; DO NOT EDIT. - -package okexapi - -import ( - "encoding/json" - "fmt" - "net/url" -) - -func (c *CancelOrderRequest) InstrumentID(instrumentID string) *CancelOrderRequest { - c.instrumentID = instrumentID - return c -} - -func (c *CancelOrderRequest) OrderID(orderID string) *CancelOrderRequest { - c.orderID = &orderID - return c -} - -func (c *CancelOrderRequest) ClientOrderID(clientOrderID string) *CancelOrderRequest { - c.clientOrderID = &clientOrderID - return c -} - -func (c *CancelOrderRequest) GetParameters() (map[string]interface{}, error) { - var params = map[string]interface{}{} - - // check instrumentID field -> json key instId - instrumentID := c.instrumentID - - // assign parameter of instrumentID - params["instId"] = instrumentID - - // check orderID field -> json key ordId - if c.orderID != nil { - orderID := *c.orderID - - // assign parameter of orderID - params["ordId"] = orderID - } - - // check clientOrderID field -> json key clOrdId - if c.clientOrderID != nil { - clientOrderID := *c.clientOrderID - - // assign parameter of clientOrderID - params["clOrdId"] = clientOrderID - } - - return params, nil -} - -func (c *CancelOrderRequest) GetParametersQuery() (url.Values, error) { - query := url.Values{} - - params, err := c.GetParameters() - if err != nil { - return query, err - } - - for k, v := range params { - query.Add(k, fmt.Sprintf("%v", v)) - } - - return query, nil -} - -func (c *CancelOrderRequest) GetParametersJSON() ([]byte, error) { - params, err := c.GetParameters() - if err != nil { - return nil, err - } - - return json.Marshal(params) -} diff --git a/pkg/exchange/okex/okexapi/cancel_order_request_requestgen.go b/pkg/exchange/okex/okexapi/cancel_order_request_requestgen.go new file mode 100644 index 0000000000..ad550a6e23 --- /dev/null +++ b/pkg/exchange/okex/okexapi/cancel_order_request_requestgen.go @@ -0,0 +1,195 @@ +// Code generated by "requestgen -method POST -responseType .APIResponse -responseDataField Data -url /api/v5/trade/cancel-order -type CancelOrderRequest -responseDataType []OrderResponse"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" +) + +func (c *CancelOrderRequest) InstrumentID(instrumentID string) *CancelOrderRequest { + c.instrumentID = instrumentID + return c +} + +func (c *CancelOrderRequest) OrderID(orderID string) *CancelOrderRequest { + c.orderID = &orderID + return c +} + +func (c *CancelOrderRequest) ClientOrderID(clientOrderID string) *CancelOrderRequest { + c.clientOrderID = &clientOrderID + return c +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (c *CancelOrderRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (c *CancelOrderRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + // check instrumentID field -> json key instId + instrumentID := c.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + // check orderID field -> json key ordId + if c.orderID != nil { + orderID := *c.orderID + + // assign parameter of orderID + params["ordId"] = orderID + } else { + } + // check clientOrderID field -> json key clOrdId + if c.clientOrderID != nil { + clientOrderID := *c.clientOrderID + + // assign parameter of clientOrderID + params["clOrdId"] = clientOrderID + } else { + } + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (c *CancelOrderRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := c.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if c.isVarSlice(_v) { + c.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (c *CancelOrderRequest) GetParametersJSON() ([]byte, error) { + params, err := c.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (c *CancelOrderRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (c *CancelOrderRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (c *CancelOrderRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (c *CancelOrderRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (c *CancelOrderRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := c.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (c *CancelOrderRequest) GetPath() string { + return "/api/v5/trade/cancel-order" +} + +// Do generates the request object and send the request object to the API endpoint +func (c *CancelOrderRequest) Do(ctx context.Context) ([]OrderResponse, error) { + + params, err := c.GetParameters() + if err != nil { + return nil, err + } + query := url.Values{} + + var apiURL string + + apiURL = c.GetPath() + + req, err := c.client.NewAuthenticatedRequest(ctx, "POST", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := c.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []OrderResponse + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 91fe26395f..bcc0150272 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -2,10 +2,12 @@ package okexapi import ( "context" + "fmt" "os" "strconv" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/c9s/bbgo/pkg/testutil" @@ -108,6 +110,66 @@ func TestClient_PlaceOrderRequest(t *testing.T) { t.Log(res) } +func TestClient_CancelOrderRequest(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + req := client.NewPlaceOrderRequest() + clientId := fmt.Sprintf("%d", uuid.New().ID()) + + order, err := req. + InstrumentID("BTC-USDT"). + TradeMode(TradeModeCash). + Side(SideTypeSell). + OrderType(OrderTypeLimit). + TargetCurrency(TargetCurrencyBase). + ClientOrderID(clientId). + Price("48000"). + Size("0.001"). + Do(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, order) + t.Logf("place order: %+v", order) + + c := client.NewGetOrderDetailsRequest().ClientOrderID(clientId).InstrumentID("BTC-USDT") + res, err := c.Do(ctx) + assert.NoError(t, err) + t.Log(res) + + cancelResp, err := client.NewCancelOrderRequest().ClientOrderID(clientId).InstrumentID("BTC-USDT").Do(ctx) + assert.NoError(t, err) + t.Log(cancelResp) +} + +func TestClient_BatchCancelOrderRequest(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + req := client.NewPlaceOrderRequest() + clientId := fmt.Sprintf("%d", uuid.New().ID()) + + order, err := req. + InstrumentID("BTC-USDT"). + TradeMode(TradeModeCash). + Side(SideTypeSell). + OrderType(OrderTypeLimit). + TargetCurrency(TargetCurrencyBase). + ClientOrderID(clientId). + Price("48000"). + Size("0.001"). + Do(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, order) + t.Logf("place order: %+v", order) + + c := client.NewGetOrderDetailsRequest().ClientOrderID(clientId).InstrumentID("BTC-USDT") + res, err := c.Do(ctx) + assert.NoError(t, err) + t.Log(res) + + cancelResp, err := client.NewBatchCancelOrderRequest().Add(&CancelOrderRequest{instrumentID: "BTC-USDT", clientOrderID: &clientId}).Do(ctx) + assert.NoError(t, err) + t.Log(cancelResp) +} + func TestClient_GetPendingOrderRequest(t *testing.T) { client := getTestClientOrSkip(t) ctx := context.Background() diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index 09f03e360f..19d5c497f0 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -17,12 +17,6 @@ func (c *RestClient) NewBatchPlaceOrderRequest() *BatchPlaceOrderRequest { } } -func (c *RestClient) NewCancelOrderRequest() *CancelOrderRequest { - return &CancelOrderRequest{ - client: c, - } -} - func (c *RestClient) NewBatchCancelOrderRequest() *BatchCancelOrderRequest { return &BatchCancelOrderRequest{ client: c, @@ -52,52 +46,6 @@ func (r *PlaceOrderRequest) Parameters() map[string]interface{} { return params } -//go:generate requestgen -type CancelOrderRequest -type CancelOrderRequest struct { - client *RestClient - - instrumentID string `param:"instId"` - orderID *string `param:"ordId"` - clientOrderID *string `param:"clOrdId"` -} - -func (r *CancelOrderRequest) Parameters() map[string]interface{} { - payload, _ := r.GetParameters() - return payload -} - -func (r *CancelOrderRequest) Do(ctx context.Context) ([]OrderResponse, error) { - payload, err := r.GetParameters() - if err != nil { - return nil, err - } - - if r.clientOrderID == nil && r.orderID != nil { - return nil, errors.New("either orderID or clientOrderID is required for canceling order") - } - - req, err := r.client.NewAuthenticatedRequest(ctx, "POST", "/api/v5/trade/cancel-order", nil, payload) - if err != nil { - return nil, err - } - - response, err := r.client.SendRequest(req) - if err != nil { - return nil, err - } - - var apiResponse APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err - } - var data []OrderResponse - if err := json.Unmarshal(apiResponse.Data, &data); err != nil { - return nil, err - } - - return data, nil -} - type BatchCancelOrderRequest struct { client *RestClient @@ -113,7 +61,10 @@ func (r *BatchCancelOrderRequest) Do(ctx context.Context) ([]OrderResponse, erro var parameterList []map[string]interface{} for _, req := range r.reqs { - params := req.Parameters() + params, err := req.GetParameters() + if err != nil { + return nil, err + } parameterList = append(parameterList, params) } From 905148a34f47cebb35660813a9664463c2b72f54 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 26 Jun 2021 21:03:03 +0800 Subject: [PATCH 418/422] maxapi: use fastjson parser pool --- pkg/exchange/max/maxapi/public_parser.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/exchange/max/maxapi/public_parser.go b/pkg/exchange/max/maxapi/public_parser.go index d96799f50e..369fc1bd87 100644 --- a/pkg/exchange/max/maxapi/public_parser.go +++ b/pkg/exchange/max/maxapi/public_parser.go @@ -16,10 +16,13 @@ var ErrIncorrectBookEntryElementLength = errors.New("incorrect book entry elemen const Buy = 1 const Sell = -1 +var parserPool fastjson.ParserPool + // ParseMessage accepts the raw messages from max public websocket channels and parses them into market data // Return types: *BookEvent, *PublicTradeEvent, *SubscriptionEvent, *ErrorEvent func ParseMessage(payload []byte) (interface{}, error) { - parser := fastjson.Parser{} + parser := parserPool.Get() + val, err := parser.ParseBytes(payload) if err != nil { return nil, errors.Wrap(err, "failed to parse payload: "+string(payload)) From 68be0badca2565595d057ba3f5c5b5c36b2d041b Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 28 Jun 2021 13:11:26 +0800 Subject: [PATCH 419/422] max: improve depth parsing speed --- pkg/exchange/max/maxapi/public_parser.go | 60 +++++++++++++++--------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/pkg/exchange/max/maxapi/public_parser.go b/pkg/exchange/max/maxapi/public_parser.go index 369fc1bd87..231748e971 100644 --- a/pkg/exchange/max/maxapi/public_parser.go +++ b/pkg/exchange/max/maxapi/public_parser.go @@ -168,8 +168,8 @@ type BookEvent struct { Market string `json:"M"` Channel string `json:"c"` Timestamp int64 `json:"t"` // Millisecond timestamp - Bids []BookEntry - Asks []BookEntry + Bids types.PriceVolumeSlice + Asks types.PriceVolumeSlice } func (e *BookEvent) Time() time.Time { @@ -179,25 +179,8 @@ func (e *BookEvent) Time() time.Time { func (e *BookEvent) OrderBook() (snapshot types.SliceOrderBook, err error) { snapshot.Symbol = strings.ToUpper(e.Market) snapshot.Time = e.Time() - - for _, bid := range e.Bids { - pv, err := bid.PriceVolumePair() - if err != nil { - return snapshot, err - } - - snapshot.Bids = append(snapshot.Bids, pv) - } - - for _, ask := range e.Asks { - pv, err := ask.PriceVolumePair() - if err != nil { - return snapshot, err - } - - snapshot.Asks = append(snapshot.Asks, pv) - } - + snapshot.Bids = e.Bids + snapshot.Asks = e.Asks return snapshot, nil } @@ -236,12 +219,12 @@ func parseBookEvent(val *fastjson.Value) (*BookEvent, error) { t := time.Unix(0, event.Timestamp*int64(time.Millisecond)) var err error - event.Asks, err = parseBookEntries(val.GetArray("a"), Sell, t) + event.Asks, err = parseBookEntries2(val.GetArray("a")) if err != nil { return nil, err } - event.Bids, err = parseBookEntries(val.GetArray("b"), Buy, t) + event.Bids, err = parseBookEntries2(val.GetArray("b")) if err != nil { return nil, err } @@ -270,6 +253,37 @@ func (e *BookEntry) PriceVolumePair() (pv types.PriceVolume, err error) { return pv, err } +// parseBookEntries2 parses JSON struct like `[["233330", "0.33"], ....]` +func parseBookEntries2(vals []*fastjson.Value) (entries types.PriceVolumeSlice, err error) { + for _, entry := range vals { + pv, err := entry.Array() + if err != nil { + return nil, err + } + + if len(pv) < 2 { + return nil, ErrIncorrectBookEntryElementLength + } + + price, err := fixedpoint.NewFromString(string(pv[0].GetStringBytes())) + if err != nil { + return nil, err + } + + volume, err := fixedpoint.NewFromString(string(pv[1].GetStringBytes())) + if err != nil { + return nil, err + } + + entries = append(entries, types.PriceVolume{ + Price: price, + Volume: volume, + }) + } + + return entries, err +} + // parseBookEntries parses JSON struct like `[["233330", "0.33"], ....]` func parseBookEntries(vals []*fastjson.Value, side int, t time.Time) (entries []BookEntry, err error) { for _, entry := range vals { From c01be14c705b216919cad4773353e7b54254940c Mon Sep 17 00:00:00 2001 From: c9s Date: Mon, 28 Jun 2021 13:13:32 +0800 Subject: [PATCH 420/422] max: remove unused var --- pkg/exchange/max/maxapi/public_parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exchange/max/maxapi/public_parser.go b/pkg/exchange/max/maxapi/public_parser.go index 231748e971..e6dcb2ac88 100644 --- a/pkg/exchange/max/maxapi/public_parser.go +++ b/pkg/exchange/max/maxapi/public_parser.go @@ -216,7 +216,7 @@ func parseBookEvent(val *fastjson.Value) (*BookEvent, error) { Timestamp: val.GetInt64("T"), } - t := time.Unix(0, event.Timestamp*int64(time.Millisecond)) + // t := time.Unix(0, event.Timestamp*int64(time.Millisecond)) var err error event.Asks, err = parseBookEntries2(val.GetArray("a")) From b352ae855f991628affbaa54b556b2929c167a11 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 11 Jan 2024 16:41:42 +0800 Subject: [PATCH 421/422] pkg/exchange: add query open orders --- pkg/exchange/okex/convert.go | 43 +++ pkg/exchange/okex/convert_test.go | 84 +++++ pkg/exchange/okex/exchange.go | 51 ++- pkg/exchange/okex/okexapi/client.go | 4 + pkg/exchange/okex/okexapi/client_test.go | 35 +- .../okex/okexapi/get_open_orders_request.go | 112 ++++++ .../get_open_orders_request_requestgen.go | 321 ++++++++++++++++++ pkg/exchange/okex/okexapi/trade.go | 93 +---- 8 files changed, 627 insertions(+), 116 deletions(-) create mode 100644 pkg/exchange/okex/convert_test.go create mode 100644 pkg/exchange/okex/okexapi/get_open_orders_request.go create mode 100644 pkg/exchange/okex/okexapi/get_open_orders_request_requestgen.go diff --git a/pkg/exchange/okex/convert.go b/pkg/exchange/okex/convert.go index 0551ef684a..126370893b 100644 --- a/pkg/exchange/okex/convert.go +++ b/pkg/exchange/okex/convert.go @@ -149,6 +149,49 @@ func toGlobalTrades(orderDetails []okexapi.OrderDetails) ([]types.Trade, error) return trades, nil } +func openOrderToGlobal(order *okexapi.OpenOrder) (*types.Order, error) { + side := toGlobalSide(order.Side) + + orderType, err := toGlobalOrderType(order.OrderType) + if err != nil { + return nil, err + } + + timeInForce := types.TimeInForceGTC + switch order.OrderType { + case okexapi.OrderTypeFOK: + timeInForce = types.TimeInForceFOK + case okexapi.OrderTypeIOC: + timeInForce = types.TimeInForceIOC + } + + orderStatus, err := toGlobalOrderStatus(order.State) + if err != nil { + return nil, err + } + + return &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: order.ClientOrderId, + Symbol: toGlobalSymbol(order.InstrumentID), + Side: side, + Type: orderType, + Price: order.Price, + Quantity: order.Size, + TimeInForce: timeInForce, + }, + Exchange: types.ExchangeOKEx, + OrderID: uint64(order.OrderId), + UUID: strconv.FormatInt(int64(order.OrderId), 10), + Status: orderStatus, + OriginalStatus: string(order.State), + ExecutedQuantity: order.AccumulatedFillSize, + IsWorking: order.State.IsWorking(), + CreationTime: types.Time(order.CreatedTime), + UpdateTime: types.Time(order.UpdatedTime), + }, nil +} + func toGlobalOrders(orderDetails []okexapi.OrderDetails) ([]types.Order, error) { var orders []types.Order var err error diff --git a/pkg/exchange/okex/convert_test.go b/pkg/exchange/okex/convert_test.go new file mode 100644 index 0000000000..42a44f8c68 --- /dev/null +++ b/pkg/exchange/okex/convert_test.go @@ -0,0 +1,84 @@ +package okex + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/okex/okexapi" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func Test_openOrderToGlobal(t *testing.T) { + var ( + assert = assert.New(t) + + orderId = 665576973905014786 + // {"accFillSz":"0","algoClOrdId":"","algoId":"","attachAlgoClOrdId":"","attachAlgoOrds":[],"avgPx":"","cTime":"1704957916401","cancelSource":"","cancelSourceReason":"","category":"normal","ccy":"","clOrdId":"","fee":"0","feeCcy":"USDT","fillPx":"","fillSz":"0","fillTime":"","instId":"BTC-USDT","instType":"SPOT","lever":"","ordId":"665576973905014786","ordType":"limit","pnl":"0","posSide":"net","px":"48174.5","pxType":"","pxUsd":"","pxVol":"","quickMgnType":"","rebate":"0","rebateCcy":"BTC","reduceOnly":"false","side":"sell","slOrdPx":"","slTriggerPx":"","slTriggerPxType":"","source":"","state":"live","stpId":"","stpMode":"","sz":"0.00001","tag":"","tdMode":"cash","tgtCcy":"","tpOrdPx":"","tpTriggerPx":"","tpTriggerPxType":"","tradeId":"","uTime":"1704957916401"} + openOrder = &okexapi.OpenOrder{ + AccumulatedFillSize: fixedpoint.NewFromFloat(0), + AvgPrice: fixedpoint.NewFromFloat(0), + CreatedTime: types.NewMillisecondTimestampFromInt(1704957916401), + Category: "normal", + Currency: "BTC", + ClientOrderId: "", + Fee: fixedpoint.Zero, + FeeCurrency: "USDT", + FillTime: types.NewMillisecondTimestampFromInt(0), + InstrumentID: "BTC-USDT", + InstrumentType: okexapi.InstrumentTypeSpot, + OrderId: types.StrInt64(orderId), + OrderType: okexapi.OrderTypeLimit, + Price: fixedpoint.NewFromFloat(48174.5), + Side: okexapi.SideTypeBuy, + State: okexapi.OrderStateLive, + Size: fixedpoint.NewFromFloat(0.00001), + UpdatedTime: types.NewMillisecondTimestampFromInt(1704957916401), + } + expOrder = &types.Order{ + SubmitOrder: types.SubmitOrder{ + ClientOrderID: openOrder.ClientOrderId, + Symbol: toGlobalSymbol(openOrder.InstrumentID), + Side: types.SideTypeBuy, + Type: types.OrderTypeLimit, + Quantity: fixedpoint.NewFromFloat(0.00001), + Price: fixedpoint.NewFromFloat(48174.5), + AveragePrice: fixedpoint.Zero, + StopPrice: fixedpoint.Zero, + TimeInForce: types.TimeInForceGTC, + }, + Exchange: types.ExchangeOKEx, + OrderID: uint64(orderId), + UUID: fmt.Sprintf("%d", orderId), + Status: types.OrderStatusNew, + OriginalStatus: string(okexapi.OrderStateLive), + ExecutedQuantity: fixedpoint.Zero, + IsWorking: true, + CreationTime: types.Time(types.NewMillisecondTimestampFromInt(1704957916401).Time()), + UpdateTime: types.Time(types.NewMillisecondTimestampFromInt(1704957916401).Time()), + } + ) + + t.Run("succeeds", func(t *testing.T) { + order, err := openOrderToGlobal(openOrder) + assert.NoError(err) + assert.Equal(expOrder, order) + }) + + t.Run("unexpected order status", func(t *testing.T) { + newOrder := *openOrder + newOrder.State = "xxx" + _, err := openOrderToGlobal(&newOrder) + assert.ErrorContains(err, "xxx") + }) + + t.Run("unexpected order type", func(t *testing.T) { + newOrder := *openOrder + newOrder.OrderType = "xxx" + _, err := openOrderToGlobal(&newOrder) + assert.ErrorContains(err, "xxx") + }) + +} diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 0dcb7955c2..4b6381a585 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -30,15 +30,15 @@ var ( queryAccountLimiter = rate.NewLimiter(rate.Every(200*time.Millisecond), 5) placeOrderLimiter = rate.NewLimiter(rate.Every(30*time.Millisecond), 30) batchCancelOrderLimiter = rate.NewLimiter(rate.Every(5*time.Millisecond), 200) + queryOpenOrderLimiter = rate.NewLimiter(rate.Every(30*time.Millisecond), 30) ) -const ID = "okex" +const ( + ID = "okex" -// PlatformToken is the platform currency of OKEx, pre-allocate static string here -const PlatformToken = "OKB" + // PlatformToken is the platform currency of OKEx, pre-allocate static string here + PlatformToken = "OKB" -const ( - // Constant For query limit defaultQueryLimit = 100 ) @@ -295,15 +295,46 @@ func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (*t */ } +// QueryOpenOrders retrieves the pending orders. The data returned is ordered by createdTime, and we utilized the +// `After` parameter to acquire all orders. func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders []types.Order, err error) { instrumentID := toLocalSymbol(symbol) - req := e.client.NewGetPendingOrderRequest().InstrumentType(okexapi.InstrumentTypeSpot).InstrumentID(instrumentID) - orderDetails, err := req.Do(ctx) - if err != nil { - return orders, err + + nextCursor := int64(0) + for { + if err := queryOpenOrderLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("query open orders rate limiter wait error: %w", err) + } + + req := e.client.NewGetOpenOrdersRequest(). + InstrumentID(instrumentID). + After(strconv.FormatInt(nextCursor, 10)) + openOrders, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query open orders: %w", err) + } + + for _, o := range openOrders { + o, err := openOrderToGlobal(&o) + if err != nil { + return nil, fmt.Errorf("failed to convert order, err: %v", err) + } + + orders = append(orders, *o) + } + + orderLen := len(openOrders) + // a defensive programming to ensure the length of order response is expected. + if orderLen > defaultQueryLimit { + return nil, fmt.Errorf("unexpected open orders length %d", orderLen) + } + + if orderLen < defaultQueryLimit { + break + } + nextCursor = int64(openOrders[orderLen-1].OrderId) } - orders, err = toGlobalOrders(orderDetails) return orders, err } diff --git a/pkg/exchange/okex/okexapi/client.go b/pkg/exchange/okex/okexapi/client.go index 85b00b8072..425339b0f6 100644 --- a/pkg/exchange/okex/okexapi/client.go +++ b/pkg/exchange/okex/okexapi/client.go @@ -58,6 +58,10 @@ const ( OrderStateFilled OrderState = "filled" ) +func (o OrderState) IsWorking() bool { + return o == OrderStateLive || o == OrderStatePartiallyFilled +} + type RestClient struct { requestgen.BaseAPIClient diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index bcc0150272..71a75f87c4 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -140,6 +140,26 @@ func TestClient_CancelOrderRequest(t *testing.T) { t.Log(cancelResp) } +func TestClient_OpenOrdersRequest(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + + orders := []OpenOrder{} + beforeId := int64(0) + for { + c := client.NewGetOpenOrdersRequest().InstrumentID("BTC-USDT").Limit("1").After(fmt.Sprintf("%d", beforeId)) + res, err := c.Do(ctx) + assert.NoError(t, err) + if len(res) != 1 { + break + } + orders = append(orders, res...) + beforeId = int64(res[0].OrderId) + } + + t.Log(orders) +} + func TestClient_BatchCancelOrderRequest(t *testing.T) { client := getTestClientOrSkip(t) ctx := context.Background() @@ -170,21 +190,6 @@ func TestClient_BatchCancelOrderRequest(t *testing.T) { t.Log(cancelResp) } -func TestClient_GetPendingOrderRequest(t *testing.T) { - client := getTestClientOrSkip(t) - ctx := context.Background() - req := client.NewGetPendingOrderRequest() - odr_type := []string{string(OrderTypeLimit), string(OrderTypeIOC)} - - pending_order, err := req. - InstrumentID("BTC-USDT"). - OrderTypes(odr_type). - Do(ctx) - assert.NoError(t, err) - assert.NotEmpty(t, pending_order) - t.Logf("pending order: %+v", pending_order) -} - func TestClient_GetOrderDetailsRequest(t *testing.T) { client := getTestClientOrSkip(t) ctx := context.Background() diff --git a/pkg/exchange/okex/okexapi/get_open_orders_request.go b/pkg/exchange/okex/okexapi/get_open_orders_request.go new file mode 100644 index 0000000000..70995c1b38 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_open_orders_request.go @@ -0,0 +1,112 @@ +package okexapi + +import ( + "time" + + "github.com/c9s/requestgen" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +type OpenOrder struct { + AccumulatedFillSize fixedpoint.Value `json:"accFillSz"` + // If none is filled, it will return "". + AvgPrice fixedpoint.Value `json:"avgPx"` + CreatedTime types.MillisecondTimestamp `json:"cTime"` + Category string `json:"category"` + ClientOrderId string `json:"clOrdId"` + Fee fixedpoint.Value `json:"fee"` + FeeCurrency string `json:"feeCcy"` + // Last filled time + FillTime types.MillisecondTimestamp `json:"fillTime"` + InstrumentID string `json:"instId"` + InstrumentType InstrumentType `json:"instType"` + OrderId types.StrInt64 `json:"ordId"` + OrderType OrderType `json:"ordType"` + Price fixedpoint.Value `json:"px"` + Side SideType `json:"side"` + State OrderState `json:"state"` + Size fixedpoint.Value `json:"sz"` + TargetCurrency string `json:"tgtCcy"` + UpdatedTime types.MillisecondTimestamp `json:"uTime"` + + // Margin currency + // Only applicable to cross MARGIN orders in Single-currency margin. + Currency string `json:"ccy"` + TradeId string `json:"tradeId"` + // Last filled price + FillPrice fixedpoint.Value `json:"fillPx"` + // Last filled quantity + FillSize fixedpoint.Value `json:"fillSz"` + // Leverage, from 0.01 to 125. + // Only applicable to MARGIN/FUTURES/SWAP + Lever string `json:"lever"` + // Profit and loss, Applicable to orders which have a trade and aim to close position. It always is 0 in other conditions + Pnl fixedpoint.Value `json:"pnl"` + PositionSide string `json:"posSide"` + // Options price in USDOnly applicable to options; return "" for other instrument types + PriceUsd fixedpoint.Value `json:"pxUsd"` + // Implied volatility of the options orderOnly applicable to options; return "" for other instrument types + PriceVol fixedpoint.Value `json:"pxVol"` + // Price type of options + PriceType string `json:"pxType"` + // Rebate amount, only applicable to spot and margin, the reward of placing orders from the platform (rebate) + // given to user who has reached the specified trading level. If there is no rebate, this field is "". + Rebate fixedpoint.Value `json:"rebate"` + RebateCcy string `json:"rebateCcy"` + // Client-supplied Algo ID when placing order attaching TP/SL. + AttachAlgoClOrdId string `json:"attachAlgoClOrdId"` + SlOrdPx fixedpoint.Value `json:"slOrdPx"` + SlTriggerPx fixedpoint.Value `json:"slTriggerPx"` + SlTriggerPxType string `json:"slTriggerPxType"` + AttachAlgoOrds []interface{} `json:"attachAlgoOrds"` + Source string `json:"source"` + // Self trade prevention ID. Return "" if self trade prevention is not applicable + StpId string `json:"stpId"` + // Self trade prevention mode. Return "" if self trade prevention is not applicable + StpMode string `json:"stpMode"` + Tag string `json:"tag"` + TradeMode string `json:"tdMode"` + TpOrdPx fixedpoint.Value `json:"tpOrdPx"` + TpTriggerPx fixedpoint.Value `json:"tpTriggerPx"` + TpTriggerPxType string `json:"tpTriggerPxType"` + ReduceOnly string `json:"reduceOnly"` + QuickMgnType string `json:"quickMgnType"` + AlgoClOrdId string `json:"algoClOrdId"` + AlgoId string `json:"algoId"` +} + +//go:generate GetRequest -url "/api/v5/trade/orders-pending" -type GetOpenOrdersRequest -responseDataType []OpenOrder +type GetOpenOrdersRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentID *string `param:"instId,query"` + + instrumentType InstrumentType `param:"instType,query"` + + orderType *OrderType `param:"ordType,query"` + + state *OrderState `param:"state,query"` + category *string `param:"category,query"` + // Pagination of data to return records earlier than the requested ordId + after *string `param:"after,query"` + // Pagination of data to return records newer than the requested ordId + before *string `param:"before,query"` + // Filter with a begin timestamp. Unix timestamp format in milliseconds, e.g. 1597026383085 + begin *time.Time `param:"begin,query"` + + // Filter with an end timestamp. Unix timestamp format in milliseconds, e.g. 1597026383085 + end *time.Time `param:"end,query"` + limit *string `param:"limit,query"` +} + +func (c *RestClient) NewGetOpenOrdersRequest() *GetOpenOrdersRequest { + return &GetOpenOrdersRequest{ + client: c, + instrumentType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_open_orders_request_requestgen.go b/pkg/exchange/okex/okexapi/get_open_orders_request_requestgen.go new file mode 100644 index 0000000000..c97836a323 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_open_orders_request_requestgen.go @@ -0,0 +1,321 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/orders-pending -type GetOpenOrdersRequest -responseDataType []OpenOrder"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "regexp" + "time" +) + +func (g *GetOpenOrdersRequest) InstrumentID(instrumentID string) *GetOpenOrdersRequest { + g.instrumentID = &instrumentID + return g +} + +func (g *GetOpenOrdersRequest) InstrumentType(instrumentType InstrumentType) *GetOpenOrdersRequest { + g.instrumentType = instrumentType + return g +} + +func (g *GetOpenOrdersRequest) OrderType(orderType OrderType) *GetOpenOrdersRequest { + g.orderType = &orderType + return g +} + +func (g *GetOpenOrdersRequest) State(state OrderState) *GetOpenOrdersRequest { + g.state = &state + return g +} + +func (g *GetOpenOrdersRequest) Category(category string) *GetOpenOrdersRequest { + g.category = &category + return g +} + +func (g *GetOpenOrdersRequest) After(after string) *GetOpenOrdersRequest { + g.after = &after + return g +} + +func (g *GetOpenOrdersRequest) Before(before string) *GetOpenOrdersRequest { + g.before = &before + return g +} + +func (g *GetOpenOrdersRequest) Begin(begin time.Time) *GetOpenOrdersRequest { + g.begin = &begin + return g +} + +func (g *GetOpenOrdersRequest) End(end time.Time) *GetOpenOrdersRequest { + g.end = &end + return g +} + +func (g *GetOpenOrdersRequest) Limit(limit string) *GetOpenOrdersRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetOpenOrdersRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instrumentID field -> json key instId + if g.instrumentID != nil { + instrumentID := *g.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + } else { + } + // check instrumentType field -> json key instType + instrumentType := g.instrumentType + + // TEMPLATE check-valid-values + switch instrumentType { + case InstrumentTypeSpot, InstrumentTypeSwap, InstrumentTypeFutures, InstrumentTypeOption, InstrumentTypeMARGIN: + params["instType"] = instrumentType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instrumentType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instrumentType + params["instType"] = instrumentType + // check orderType field -> json key ordType + if g.orderType != nil { + orderType := *g.orderType + + // TEMPLATE check-valid-values + switch orderType { + case OrderTypeMarket, OrderTypeLimit, OrderTypePostOnly, OrderTypeFOK, OrderTypeIOC: + params["ordType"] = orderType + + default: + return nil, fmt.Errorf("ordType value %v is invalid", orderType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of orderType + params["ordType"] = orderType + } else { + } + // check state field -> json key state + if g.state != nil { + state := *g.state + + // TEMPLATE check-valid-values + switch state { + case OrderStateCanceled, OrderStateLive, OrderStatePartiallyFilled, OrderStateFilled: + params["state"] = state + + default: + return nil, fmt.Errorf("state value %v is invalid", state) + + } + // END TEMPLATE check-valid-values + + // assign parameter of state + params["state"] = state + } else { + } + // check category field -> json key category + if g.category != nil { + category := *g.category + + // assign parameter of category + params["category"] = category + } else { + } + // check after field -> json key after + if g.after != nil { + after := *g.after + + // assign parameter of after + params["after"] = after + } else { + } + // check before field -> json key before + if g.before != nil { + before := *g.before + + // assign parameter of before + params["before"] = before + } else { + } + // check begin field -> json key begin + if g.begin != nil { + begin := *g.begin + + // assign parameter of begin + params["begin"] = begin + } else { + } + // check end field -> json key end + if g.end != nil { + end := *g.end + + // assign parameter of end + params["end"] = end + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetOpenOrdersRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetOpenOrdersRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetOpenOrdersRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetOpenOrdersRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetOpenOrdersRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetOpenOrdersRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetOpenOrdersRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetOpenOrdersRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetOpenOrdersRequest) GetPath() string { + return "/api/v5/trade/orders-pending" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetOpenOrdersRequest) Do(ctx context.Context) ([]OpenOrder, error) { + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + + type responseValidator interface { + Validate() error + } + validator, ok := interface{}(apiResponse).(responseValidator) + if ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []OpenOrder + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/trade.go b/pkg/exchange/okex/okexapi/trade.go index 19d5c497f0..f9440c80bd 100644 --- a/pkg/exchange/okex/okexapi/trade.go +++ b/pkg/exchange/okex/okexapi/trade.go @@ -4,11 +4,11 @@ import ( "context" "encoding/json" "net/url" - "strings" + + "github.com/pkg/errors" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" - "github.com/pkg/errors" ) func (c *RestClient) NewBatchPlaceOrderRequest() *BatchPlaceOrderRequest { @@ -29,12 +29,6 @@ func (c *RestClient) NewGetOrderDetailsRequest() *GetOrderDetailsRequest { } } -func (c *RestClient) NewGetPendingOrderRequest() *GetPendingOrderRequest { - return &GetPendingOrderRequest{ - client: c, - } -} - func (c *RestClient) NewGetTransactionDetailsRequest() *GetTransactionDetailsRequest { return &GetTransactionDetailsRequest{ client: c, @@ -249,89 +243,6 @@ func (r *GetOrderDetailsRequest) Do(ctx context.Context) (*OrderDetails, error) return &data[0], nil } -type GetPendingOrderRequest struct { - client *RestClient - - instId *string - - instType *InstrumentType - - orderTypes []string - - state *OrderState -} - -func (r *GetPendingOrderRequest) InstrumentID(instId string) *GetPendingOrderRequest { - r.instId = &instId - return r -} - -func (r *GetPendingOrderRequest) InstrumentType(instType InstrumentType) *GetPendingOrderRequest { - r.instType = &instType - return r -} - -func (r *GetPendingOrderRequest) State(state OrderState) *GetPendingOrderRequest { - r.state = &state - return r -} - -func (r *GetPendingOrderRequest) OrderTypes(orderTypes []string) *GetPendingOrderRequest { - r.orderTypes = orderTypes - return r -} - -func (r *GetPendingOrderRequest) AddOrderTypes(orderTypes ...string) *GetPendingOrderRequest { - r.orderTypes = append(r.orderTypes, orderTypes...) - return r -} - -func (r *GetPendingOrderRequest) Parameters() map[string]interface{} { - var payload = map[string]interface{}{} - - if r.instId != nil { - payload["instId"] = r.instId - } - - if r.instType != nil { - payload["instType"] = r.instType - } - - if r.state != nil { - payload["state"] = r.state - } - - if len(r.orderTypes) > 0 { - payload["ordType"] = strings.Join(r.orderTypes, ",") - } - - return payload -} - -func (r *GetPendingOrderRequest) Do(ctx context.Context) ([]OrderDetails, error) { - payload := r.Parameters() - req, err := r.client.NewAuthenticatedRequest(ctx, "GET", "/api/v5/trade/orders-pending", nil, payload) - if err != nil { - return nil, err - } - - response, err := r.client.SendRequest(req) - if err != nil { - return nil, err - } - - var apiResponse APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err - } - var data []OrderDetails - if err := json.Unmarshal(apiResponse.Data, &data); err != nil { - return nil, err - } - - return data, nil -} - type GetTransactionDetailsRequest struct { client *RestClient From 228bfba5253ce3c487ae1ed07db9f68f06089e13 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 11 Jan 2024 16:42:06 +0800 Subject: [PATCH 422/422] pkg/fixedpoint: support "" on fixedpoint.Value.unmarshalJson --- pkg/fixedpoint/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fixedpoint/convert.go b/pkg/fixedpoint/convert.go index 9a8bf36fe2..d000603a81 100644 --- a/pkg/fixedpoint/convert.go +++ b/pkg/fixedpoint/convert.go @@ -296,7 +296,7 @@ func (v *Value) UnmarshalJSON(data []byte) error { *v = Zero return nil } - if len(data) == 0 { + if len(data) == 0 || bytes.Equal(data, []byte{'"', '"'}) { *v = Zero return nil }