database: sqlite3 issue fix

	database: upgrade gosqllite3 version for increasing variable amount limit
	types: update kline starttime/endtime field to prevent sqlite3
	time parsing issue.

	fix c9s#215

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/magefile/mage v1.11.0 // indirect
 	github.com/magiconair/properties v1.8.4 // indirect
 	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
+	github.com/mattn/go-sqlite3 v1.14.9 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/pelletier/go-toml v1.8.1 // indirect
 	github.com/pkg/errors v0.9.1

go.sum

@@ -246,6 +246,8 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
+github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

pkg/backtest/exchange.go

@@ -266,7 +266,7 @@ func (e Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker
 
 	kline := matching.LastKLine
 	return &types.Ticker{
-		Time:   kline.EndTime,
+		Time:   kline.EndTime.Time(),
 		Volume: kline.Volume,
 		Last:   kline.Close,
 		Open:   kline.Open,

pkg/backtest/matching.go

@@ -399,7 +399,7 @@ func (m *SimplePriceMatching) SellToPrice(price fixedpoint.Value) (closedOrders
 }
 
 func (m *SimplePriceMatching) processKLine(kline types.KLine) {
-	m.CurrentTime = kline.EndTime
+	m.CurrentTime = kline.EndTime.Time()
 	m.LastKLine = kline
 
 	switch kline.Direction() {

pkg/exchange/batch/batch.go

@@ -128,7 +128,7 @@ func (e KLineBatchQuery) Query(ctx context.Context, symbol string, interval type
 				}
 
 				//The issue is in FTX, prev endtime = next start time , so if add 1 ms , it would query forever.
-				currentTime = kline.StartTime
+				currentTime = kline.StartTime.Time()
 				tryQueryKlineTimes = 0
 			}
 

pkg/exchange/binance/exchange.go

@@ -919,8 +919,8 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
 			Exchange:                 types.ExchangeBinance,
 			Symbol:                   symbol,
 			Interval:                 interval,
-			StartTime:                time.Unix(0, k.OpenTime*int64(time.Millisecond)),
-			EndTime:                  time.Unix(0, k.CloseTime*int64(time.Millisecond)),
+			StartTime:                types.NewTimeFromUnix(0, k.OpenTime*int64(time.Millisecond)),
+			EndTime:                  types.NewTimeFromUnix(0, k.CloseTime*int64(time.Millisecond)),
 			Open:                     util.MustParseFloat(k.Open),
 			Close:                    util.MustParseFloat(k.Close),
 			High:                     util.MustParseFloat(k.High),
@@ -1070,7 +1070,7 @@ func (e *Exchange) BatchQueryKLines(ctx context.Context, symbol string, interval
 			}
 
 			allKLines = append(allKLines, kline)
-			startTime = kline.EndTime
+			startTime = kline.EndTime.Time()
 		}
 	}
 

pkg/exchange/binance/parse.go

@@ -521,8 +521,8 @@ func (k *KLine) KLine() types.KLine {
 		Exchange:                 types.ExchangeBinance,
 		Symbol:                   k.Symbol,
 		Interval:                 types.Interval(k.Interval),
-		StartTime:                time.Unix(0, k.StartTime*int64(time.Millisecond)),
-		EndTime:                  time.Unix(0, k.EndTime*int64(time.Millisecond)),
+		StartTime:                types.NewTimeFromUnix(0, k.StartTime*int64(time.Millisecond)),
+		EndTime:                  types.NewTimeFromUnix(0, k.EndTime*int64(time.Millisecond)),
 		Open:                     k.Open.Float64(),
 		Close:                    k.Close.Float64(),
 		High:                     k.High.Float64(),

pkg/exchange/binance/stream_test.go

@@ -49,8 +49,8 @@ func Test_Batch(t *testing.T) {
 		assert.True(t, nowMinTime.Unix() > lastmaxtime.Unix())
 		assert.True(t, nowMaxTime.Unix() > lastmaxtime.Unix())
 
-		lastmintime = nowMinTime
-		lastmaxtime = nowMaxTime
+		lastmintime = nowMinTime.Time()
+		lastmaxtime = nowMaxTime.Time()
 		assert.True(t, lastmintime.Unix() <= lastmaxtime.Unix())
 
 	}

pkg/exchange/ftx/convert.go

@@ -141,8 +141,8 @@ func toGlobalKLine(symbol string, interval types.Interval, h Candle) (types.KLin
 	return types.KLine{
 		Exchange:  types.ExchangeFTX,
 		Symbol:    toGlobalSymbol(symbol),
-		StartTime: h.StartTime.Time,
-		EndTime:   h.StartTime.Add(interval.Duration()),
+		StartTime: types.Time(h.StartTime.Time),
+		EndTime:   types.Time(h.StartTime.Add(interval.Duration())),
 		Interval:  interval,
 		Open:      h.Open,
 		Close:     h.Close,

pkg/exchange/ftx/exchange.go

@@ -260,7 +260,7 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
 		for _, line := range lines {
 
 			if line.StartTime.Unix() < currentEnd.Unix() {
-				currentEnd = line.StartTime
+				currentEnd = line.StartTime.Time()
 			}
 
 			if line.StartTime.Unix() > since.Unix() {

pkg/exchange/ftx/stream.go

@@ -145,7 +145,7 @@ func (s *Stream) pollKLines(ctx context.Context) {
 			s.EmitKLine(klines[0])
 			s.EmitKLineClosed(klines[0])
 			s.EmitKLine(klines[1])
-			lastClosed = klines[0].StartTime
+			lastClosed = klines[0].StartTime.Time()
 		}
 	}
 
@@ -175,7 +175,7 @@ func (s *Stream) pollKLines(ctx context.Context) {
 					if lastClosed.Unix() < klines[0].StartTime.Unix() {
 						s.EmitKLine(klines[0])
 						s.EmitKLineClosed(klines[0])
-						lastClosed = klines[0].StartTime
+						lastClosed = klines[0].StartTime.Time()
 					}
 					s.EmitKLine(klines[1])
 				}

pkg/exchange/ftx/stream_test.go

@@ -80,8 +80,8 @@ func Test_Batch(t *testing.T) {
 			assert.True(t, nowMinTime.Unix() > lastmaxtime.Unix())
 			assert.True(t, nowMaxTime.Unix() > lastmaxtime.Unix())
 		}
-		lastmintime = nowMinTime
-		lastmaxtime = nowMaxTime
+		lastmintime = nowMinTime.Time()
+		lastmaxtime = nowMaxTime.Time()
 		assert.True(t, lastmintime.Unix() <= lastmaxtime.Unix())
 
 	}

pkg/exchange/max/maxapi/public.go

@@ -216,8 +216,8 @@ func (k KLine) KLine() types.KLine {
 		Exchange:  types.ExchangeMax,
 		Symbol:    strings.ToUpper(k.Symbol), // global symbol
 		Interval:  types.Interval(k.Interval),
-		StartTime: k.StartTime,
-		EndTime:   k.EndTime,
+		StartTime: types.Time(k.StartTime),
+		EndTime:   types.Time(k.EndTime),
 		Open:      k.Open,
 		Close:     k.Close,
 		High:      k.High,

pkg/exchange/max/maxapi/public_parser.go

@@ -118,8 +118,8 @@ type KLinePayload struct {
 
 func (k KLinePayload) KLine() types.KLine {
 	return types.KLine{
-		StartTime:      time.Unix(0, k.StartTime*int64(time.Millisecond)),
-		EndTime:        time.Unix(0, k.EndTime*int64(time.Millisecond)),
+		StartTime:      types.Time(time.Unix(0, k.StartTime*int64(time.Millisecond))),
+		EndTime:        types.Time(time.Unix(0, k.EndTime*int64(time.Millisecond))),
 		Symbol:         k.Market,
 		Interval:       types.Interval(k.Resolution),
 		Open:           util.MustParseFloat(k.Open),

pkg/exchange/okex/exchange.go

@@ -302,8 +302,8 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
 			Closed:      true,
 			Volume:      candle.Volume.Float64(),
 			QuoteVolume: candle.VolumeInCurrency.Float64(),
-			StartTime:   candle.Time,
-			EndTime:     candle.Time.Add(interval.Duration() - time.Millisecond),
+			StartTime:   types.Time(candle.Time),
+			EndTime:     types.Time(candle.Time.Add(interval.Duration() - time.Millisecond)),
 		})
 	}
 

pkg/exchange/okex/parse.go

@@ -199,8 +199,8 @@ func (c *Candle) KLine() types.KLine {
 		Close:       c.Close.Float64(),
 		Volume:      c.Volume.Float64(),
 		QuoteVolume: c.VolumeInCurrency.Float64(),
-		StartTime:   c.StartTime,
-		EndTime:     endTime,
+		StartTime:   types.Time(c.StartTime),
+		EndTime:     types.Time(endTime),
 	}
 }
 

pkg/indicator/ad.go

@@ -49,7 +49,7 @@ func (inc *AD) calculateAndUpdate(kLines []types.KLine) {
 
 		inc.update(k)
 		inc.EmitUpdate(inc.Last())
-		inc.EndTime = kLines[i].EndTime
+		inc.EndTime = kLines[i].EndTime.Time()
 	}
 
 }

pkg/indicator/boll.go

@@ -105,7 +105,7 @@ func (inc *BOLL) calculateAndUpdate(kLines []types.KLine) {
 	inc.DownBand.Push(downBand)
 
 	// update end time
-	inc.EndTime = kLines[index].EndTime
+	inc.EndTime = kLines[index].EndTime.Time()
 
 	// log.Infof("update boll: sma=%f, up=%f, down=%f", sma, upBand, downBand)
 

pkg/indicator/ewma.go

@@ -81,7 +81,7 @@ func (inc *EWMA) calculateAndUpdate(allKLines []types.KLine) {
 		var k = allKLines[i]
 		var ewma = priceF(k)*multiplier + (1-multiplier)*inc.Values[i-1]
 		inc.Values.Push(ewma)
-		inc.LastOpenTime = k.StartTime
+		inc.LastOpenTime = k.StartTime.Time()
 		inc.EmitUpdate(ewma)
 	}
 

pkg/indicator/macd.go

@@ -76,7 +76,7 @@ func (inc *MACD) calculateAndUpdate(kLines []types.KLine) {
 	for i, kLine := range kLines {
 		inc.update(kLine, priceF)
 		inc.EmitUpdate(inc.Values[len(inc.Values)-1])
-		inc.EndTime = kLines[i].EndTime
+		inc.EndTime = kLines[i].EndTime.Time()
 	}
 
 }

pkg/indicator/obv.go

@@ -59,7 +59,7 @@ func (inc *OBV) calculateAndUpdate(kLines []types.KLine) {
 
 		inc.update(k, priceF)
 		inc.EmitUpdate(inc.Last())
-		inc.EndTime = kLines[i].EndTime
+		inc.EndTime = kLines[i].EndTime.Time()
 	}
 
 }

pkg/indicator/sma.go

@@ -55,7 +55,7 @@ func (inc *SMA) calculateAndUpdate(kLines []types.KLine) {
 		inc.Values = inc.Values[MaxNumOfSMATruncateSize-1:]
 	}
 
-	inc.EndTime = kLines[index].EndTime
+	inc.EndTime = kLines[index].EndTime.Time()
 
 	inc.EmitUpdate(sma)
 }

pkg/indicator/stoch.go

@@ -66,7 +66,7 @@ func (inc *STOCH) calculateAndUpdate(kLines []types.KLine) {
 
 		inc.update(k)
 		inc.EmitUpdate(inc.LastK(), inc.LastD())
-		inc.EndTime = kLines[i].EndTime
+		inc.EndTime = kLines[i].EndTime.Time()
 	}
 }
 

pkg/indicator/stoch_test.go

@@ -25,7 +25,7 @@ func TestSTOCH_update(t *testing.T) {
 
 	buildKLines := func(open, high, low, close []float64) (kLines []types.KLine) {
 		for i := range high {
-			kLines = append(kLines, types.KLine{Open: open[i], High: high[i], Low: low[i], Close: close[i], EndTime: time.Now()})
+			kLines = append(kLines, types.KLine{Open: open[i], High: high[i], Low: low[i], Close: close[i], EndTime: types.Time(time.Now())})
 		}
 		return kLines
 	}

pkg/indicator/vwap.go

@@ -92,7 +92,7 @@ func (inc *VWAP) calculateAndUpdate(kLines []types.KLine) {
 		inc.Values.Push(vwap)
 		inc.EmitUpdate(vwap)
 
-		inc.EndTime = kLines[i].EndTime
+		inc.EndTime = kLines[i].EndTime.Time()
 	}
 }
 

pkg/service/backtest.go

@@ -65,7 +65,7 @@ func (s *BacktestService) Verify(symbols []string, startTime time.Time, endTime
 					if prevKLine.StartTime.Unix() == k.StartTime.Unix() {
 						s._deleteDuplicatedKLine(k)
 						log.Errorf("found kline data duplicated at time: %s kline: %+v , deleted it", k.StartTime, k)
-					} else if prevKLine.StartTime.Add(interval.Duration()) != k.StartTime {
+					} else if prevKLine.StartTime.Time().Add(interval.Duration()).Unix() != k.StartTime.Time().Unix() {
 						corruptCnt++
 						log.Errorf("found kline data corrupted at time: %s kline: %+v", k.StartTime, k)
 						log.Errorf("between %d and %d",
@@ -339,9 +339,9 @@ func (s *BacktestService) SyncExist(ctx context.Context, exchange types.Exchange
 	for k := range klineC {
 		if nowStartTime.Add(interval.Duration()).Unix() < k.StartTime.Unix() {
 			log.Infof("syncing %s interval %s syncing %s ~ %s ", symbol, interval, nowStartTime, k.EndTime)
-			s.Sync(ctx, exchange, symbol, nowStartTime.Add(interval.Duration()), k.EndTime.Add(-1*interval.Duration()), interval)
+			s.Sync(ctx, exchange, symbol, nowStartTime.Add(interval.Duration()), k.EndTime.Time().Add(-1*interval.Duration()), interval)
 		}
-		nowStartTime = k.StartTime
+		nowStartTime = k.StartTime.Time()
 	}
 
 	if err := <-errC; err != nil {

pkg/types/kline.go

@@ -50,8 +50,8 @@ type KLine struct {
 
 	Symbol string `json:"symbol" db:"symbol"`
 
-	StartTime time.Time `json:"startTime" db:"start_time"`
-	EndTime   time.Time `json:"endTime" db:"end_time"`
+	StartTime Time `json:"startTime" db:"start_time"`
+	EndTime   Time `json:"endTime" db:"end_time"`
 
 	Interval Interval `json:"interval" db:"interval"`
 
@@ -69,11 +69,11 @@ type KLine struct {
 	Closed         bool   `json:"closed" db:"closed"`
 }
 
-func (k KLine) GetStartTime() time.Time {
+func (k KLine) GetStartTime() Time {
 	return k.StartTime
 }
 
-func (k KLine) GetEndTime() time.Time {
+func (k KLine) GetEndTime() Time {
 	return k.EndTime
 }
 
@@ -179,11 +179,10 @@ func (k KLine) Color() string {
 	return GrayColor
 }
 
-
 func (k KLine) String() string {
 	return fmt.Sprintf("%s %s %s %s O: %.4f H: %.4f L: %.4f C: %.4f CHG: %.4f MAXCHG: %.4f V: %.4f QV: %.2f TBBV: %.2f",
 		k.Exchange.String(),
-		k.StartTime.Format("2006-01-02 15:04"),
+		k.StartTime.Time().Format("2006-01-02 15:04"),
 		k.Symbol, k.Interval, k.Open, k.High, k.Low, k.Close, k.GetChange(), k.GetMaxChange(), k.Volume, k.QuoteVolume, k.TakerBuyBaseAssetVolume)
 }
 

pkg/types/time.go

@@ -96,6 +96,22 @@ func (t Time) Time() time.Time {
 	return time.Time(t)
 }
 
+func (t Time) Unix() int64 {
+	return time.Time(t).Unix()
+}
+
+func (t Time) After(time2 time.Time) bool {
+	return time.Time(t).After(time2)
+}
+
+func (t Time) Before(time2 time.Time) bool {
+	return time.Time(t).Before(time2)
+}
+
+func NewTimeFromUnix(sec int64, nsec int64) Time {
+	return Time(time.Unix(sec, nsec))
+}
+
 // Value implements the driver.Valuer interface
 // see http://jmoiron.net/blog/built-in-interfaces/
 func (t Time) Value() (driver.Value, error) {