Fix order ID

go.mod

@@ -5,11 +5,12 @@ go 1.14
 require (
 	contrib.go.opencensus.io/exporter/stackdriver v0.13.4 // indirect
 	github.com/GeertJohan/go.rice v1.0.0
-	github.com/dfuse-io/binary v0.0.0-20210125232659-d265783d8b7c
+	github.com/dfuse-io/binary v0.0.0-20210215145406-f5ccab52bf49
+	github.com/dfuse-io/dfuse-solana v0.0.0-20210215142939-a2baa3503463 // indirect
 	github.com/dfuse-io/logging v0.0.0-20210109005628-b97a57253f70
 	github.com/fatih/color v1.7.0
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/google/go-cmp v0.4.1
+	github.com/google/go-cmp v0.5.1
 	github.com/gorilla/rpc v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/klauspost/compress v1.11.3
@@ -33,6 +34,5 @@ require (
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
-	golang.org/x/tools v0.0.0-20200601175630-2caf76543d99 // indirect
-	google.golang.org/api v0.15.0
+	google.golang.org/api v0.29.0
 )

programs/serum/init_test.go

@@ -14,8 +14,69 @@
 
 package serum
 
-import "github.com/dfuse-io/logging"
+import (
+	"encoding/hex"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/dfuse-io/logging"
+	"github.com/klauspost/compress/zstd"
+	"github.com/stretchr/testify/require"
+)
 
 func init() {
 	logging.TestingOverride()
 }
+
+func writeCompressedFile(t *testing.T, filename string, data []byte) {
+	writeFile(t, filename, zstEncoder.EncodeAll(data, nil))
+}
+
+func readCompressedFile(t *testing.T, file string) []byte {
+	data := readFile(t, file)
+
+	out, err := zstDecoder.DecodeAll(data, nil)
+	require.NoError(t, err)
+
+	return out
+}
+
+func readHexFile(t *testing.T, file string) []byte {
+	data := readFile(t, file)
+
+	out, err := hex.DecodeString(string(data))
+	require.NoError(t, err)
+
+	return out
+}
+
+var zstEncoder, _ = zstd.NewWriter(nil)
+var zstDecoder, _ = zstd.NewReader(nil)
+
+func readFile(t *testing.T, file string) []byte {
+	data, err := ioutil.ReadFile(file)
+	require.NoError(t, err)
+
+	return data
+}
+
+func writeFile(t *testing.T, filename string, data []byte) {
+	require.NoError(t, ioutil.WriteFile(filename, data, os.ModePerm), "unable to write file %s", filename)
+}
+
+func writeJSONFile(t *testing.T, filename string, v interface{}) {
+	out, err := json.MarshalIndent(v, "", "  ")
+	require.NoError(t, err)
+
+	require.NoError(t, ioutil.WriteFile(filename, out, os.ModePerm), "unable to write JSON file %s", filename)
+}
+
+func readJSONFile(t *testing.T, file string, v interface{}) {
+	data, err := ioutil.ReadFile(file)
+	require.NoError(t, err)
+
+	require.NoError(t, json.Unmarshal(data, v))
+	return
+}

programs/serum/market.go

@@ -61,7 +61,7 @@ func (m *MarketMeta) PriceNumberToLots(price *big.Int) *big.Float {
 }
 
 type OpenOrdersMeta struct {
-	OpenOrdersV2 OpenOrdersV2
+	OpenOrders OpenOrders
 }
 
 type Order struct {

programs/serum/math.go

@@ -52,19 +52,6 @@ func divideBnToNumber(numerator, denomiator *big.Float) *big.Float {
 	return F().Quo(numerator, denomiator)
 }
 
-func GetPrice(orderID string) (uint64, error) {
-	d, err := hex.DecodeString(orderID)
-	if err != nil {
-		return 0, fmt.Errorf("unable to decode order ID: %w", err)
-	}
-
-	if len(d) < 8 {
-		return 0, fmt.Errorf("order ID too short expecting at least 8 bytes got %d", len(d))
-	}
-
-	return binary.BigEndian.Uint64(d[:8]), nil
-}
-
 func GetSeqNum(orderId string, side Side) (uint64, error) {
 	d, err := hex.DecodeString(orderId)
 	if err != nil {

programs/serum/math_test.go

@@ -7,21 +7,18 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func Test_GetPrice(t *testing.T) {
-	price, err := GetPrice("0000000000" +
-		"000eedffffffffffa78933")
+func Test_GetSeqNum(t *testing.T) {
+	seqNum, err := GetSeqNum("0000000000000eedffffffffffa78933", SideBid)
 	require.NoError(t, err)
-	assert.Equal(t, uint64(3821), price)
-}
+	assert.Equal(t, uint64(5797580), seqNum)
 
-func Test_GetSeqNum(t *testing.T) {
-	price, err := GetSeqNum("0000000000000eedffffffffffa78933", SideBid)
+	seqNum, err = GetSeqNum("0000000000000eed00000000005876cc", SideAsk)
 	require.NoError(t, err)
-	assert.Equal(t, uint64(5797580), price)
+	assert.Equal(t, uint64(5797580), seqNum)
 
-	price, err = GetSeqNum("0000000000000eed00000000005876cc", SideAsk)
+	seqNum, err = GetSeqNum("0000000000000840ffffffffffacdefd", SideBid)
 	require.NoError(t, err)
-	assert.Equal(t, uint64(5797580), price)
+	assert.Equal(t, uint64(5447938), seqNum)
 }
 
 func Test_PriceLotsToNumber(t *testing.T) {

programs/serum/queue.go

@@ -302,7 +302,7 @@ type Event struct {
 	NativeQtyReleased uint64 // the amount you should release (free to settle)
 	NativeQtyPaid     uint64 // The amount out of your account
 	NativeFeeOrRebate uint64 // maker etc...
-	OrderID           bin.Uint128
+	OrderID           OrderID
 	Owner             solana.PublicKey // OpenOrder Account address NOT trader
 	ClientOrderID     uint64
 }

programs/serum/queue_test.go

@@ -16,88 +16,28 @@ package serum
 
 import (
 	"context"
-	"encoding/binary"
-	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
 	bin "github.com/dfuse-io/binary"
 	"github.com/dfuse-io/solana-go"
 	"github.com/dfuse-io/solana-go/diff"
 	"github.com/dfuse-io/solana-go/rpc"
-	"github.com/klauspost/compress/zstd"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestDecoder_ScanEvenQueue(t *testing.T) {
-	t.Skip("broken test, don't have the courage to fix it right now")
-	// market -> 7xLk17EQQ5KLDLDe44wCmupJKJjTGd8hs3eSVVhCx932 (SOL/USDT)
-	// Base SOL -> So11111111111111111111111111111111111111112
-	baseLotSize := uint64(100000000)
-	baseDecimal := uint64(9)
-	// Quote USDT -> BQcdHdAQW1hczDbBi9hiegXAR7A98Q9jx3X3iBBBDiq4
-	quoteLotSize := uint64(100)
-	quoteDecimal := uint64(6)
-
-	data, err := ioutil.ReadFile("./testdata/serum-sol-usdt-event-queue.hex")
-	require.NoError(t, err)
-
-	byteData, err := hex.DecodeString(string(data))
-	require.NoError(t, err)
-
-	offset := 5 + 8 + 8 + 8 + 8
-
-	decoder := bin.NewDecoder(byteData[offset:])
-
-	i := 0
-	for {
-		e := &Event{}
-		err := decoder.Decode(e)
-		require.NoError(t, err)
-
-		if e.Flag.IsFill() {
-			orderID := make([]byte, 16)
-			binary.BigEndian.PutUint64(orderID[:], e.OrderID.Lo)
-			binary.BigEndian.PutUint64(orderID[8:], e.OrderID.Hi)
-
-			p, err := GetPrice(hex.EncodeToString(orderID))
-			require.NoError(t, err)
-
-			pf := PriceLotsToNumber(p, baseLotSize, quoteLotSize, baseDecimal, quoteDecimal)
-			fmt.Printf("Index: %d: Amount Released: %d, Amount Out: %d, Price: %d Price as num %s\n", i, e.NativeQtyReleased, e.NativeQtyPaid, p, pf.String())
-		}
-		i++
-	}
-}
-
-func TestEventQueue_Decoder_withRollOver(t *testing.T) {
-	eqCnt, err := ioutil.ReadFile("./testdata/event-queue-rollover.log")
-	require.NoError(t, err)
-
-	data, err := hex.DecodeString(string(eqCnt))
-	require.NoError(t, err)
-
-	var q *EventQueue
-	err = bin.NewDecoder(data).Decode(&q)
-	require.NoError(t, err)
-
-	cnt, err := json.MarshalIndent(q, "", "  ")
-	require.NoError(t, err)
-
-	fmt.Println(string(cnt))
-}
-
 func TestDecoder_EventQueue_Diff(t *testing.T) {
+	//t.Skip("diff event queue test")
+
 	oldDataFile := "testdata/serum-event-queue-old.bin.zst"
 	newDataFile := "testdata/serum-event-queue-new.bin.zst"
 
-	// olDataJSONFile := strings.ReplaceAll(oldDataFile, ".bin.zst", ".json")
-	// newDataJSONFile := strings.ReplaceAll(newDataFile, ".bin.zst", ".json")
+	olDataJSONFile := strings.ReplaceAll(oldDataFile, ".bin.zst", ".json")
+	newDataJSONFile := strings.ReplaceAll(newDataFile, ".bin.zst", ".json")
 
 	if os.Getenv("TESTDATA_UPDATE") == "true" {
 		client := rpc.NewClient("http://api.mainnet-beta.solana.com:80/rpc")
@@ -108,19 +48,19 @@ func TestDecoder_EventQueue_Diff(t *testing.T) {
 		require.NoError(t, err)
 		writeCompressedFile(t, oldDataFile, info.Value.Data)
 
-		// oldQueue := &EventQueue{}
-		// require.NoError(t, oldQueue.Decode(info.Value.Data))
-		// writeJSONFile(t, olDataJSONFile, oldQueue)
+		oldQueue := &EventQueue{}
+		require.NoError(t, oldQueue.Decode(info.Value.Data))
+		writeJSONFile(t, olDataJSONFile, oldQueue)
 
 		time.Sleep(900 * time.Millisecond)
 
 		info, err = client.GetAccountInfo(ctx, account)
 		require.NoError(t, err)
 		writeCompressedFile(t, newDataFile, info.Value.Data)
 
-		// newQueue := &EventQueue{}
-		// require.NoError(t, newQueue.Decode(info.Value.Data))
-		// writeJSONFile(t, newDataJSONFile, newQueue)
+		newQueue := &EventQueue{}
+		require.NoError(t, newQueue.Decode(info.Value.Data))
+		writeJSONFile(t, newDataJSONFile, newQueue)
 	}
 
 	oldQueue := &EventQueue{}
@@ -201,8 +141,8 @@ func TestDecoder_EventQueue_DiffManual(t *testing.T) {
 		Count:        13,
 		SeqNum:       25,
 		Events: []*Event{
-			{OrderID: bin.Uint128{Lo: 1}},
-			{OrderID: bin.Uint128{Lo: 2}},
+			{OrderID: OrderID(bin.Uint128{Lo: 1})},
+			{OrderID: OrderID(bin.Uint128{Lo: 2})},
 		},
 		EndPadding: [7]byte{},
 	}
@@ -212,9 +152,9 @@ func TestDecoder_EventQueue_DiffManual(t *testing.T) {
 		Count:  13,
 		SeqNum: 25,
 		Events: []*Event{
-			{OrderID: bin.Uint128{Lo: 1}},
-			{OrderID: bin.Uint128{Lo: 4}},
-			{OrderID: bin.Uint128{Lo: 5}},
+			{OrderID: OrderID(bin.Uint128{Lo: 1})},
+			{OrderID: OrderID(bin.Uint128{Lo: 4})},
+			{OrderID: OrderID(bin.Uint128{Lo: 5})},
 		},
 	}
 
@@ -229,46 +169,3 @@ func TestDecoder_EventQueue_DiffManual(t *testing.T) {
 		}
 	}))
 }
-
-func writeCompressedFile(t *testing.T, filename string, data []byte) {
-	require.NoError(t, ioutil.WriteFile(filename, zstEncoder.EncodeAll(data, nil), os.ModePerm), "unable to write compressed file %s", filename)
-}
-
-func readCompressedFile(t *testing.T, file string) []byte {
-	data, err := ioutil.ReadFile(file)
-	require.NoError(t, err)
-
-	out, err := zstDecoder.DecodeAll(data, nil)
-	require.NoError(t, err)
-
-	return out
-}
-
-var zstEncoder, _ = zstd.NewWriter(nil)
-var zstDecoder, _ = zstd.NewReader(nil)
-
-func writeFile(t *testing.T, filename string, data []byte) {
-	require.NoError(t, ioutil.WriteFile(filename, data, os.ModePerm), "unable to write file %s", filename)
-}
-
-func readFile(t *testing.T, file string) []byte {
-	data, err := ioutil.ReadFile(file)
-	require.NoError(t, err)
-
-	return data
-}
-
-func writeJSONFile(t *testing.T, filename string, v interface{}) {
-	out, err := json.MarshalIndent(v, "", "  ")
-	require.NoError(t, err)
-
-	require.NoError(t, ioutil.WriteFile(filename, out, os.ModePerm), "unable to write JSON file %s", filename)
-}
-
-func readJSONFile(t *testing.T, file string, v interface{}) {
-	data, err := ioutil.ReadFile(file)
-	require.NoError(t, err)
-
-	require.NoError(t, json.Unmarshal(data, v))
-	return
-}

programs/serum/rpc.go

@@ -54,7 +54,7 @@ func FetchOpenOrders(ctx context.Context, rpcCli *rpc.Client, openOrdersAddr sol
 
 	openOrdersMeta := &OpenOrdersMeta{}
 
-	if err := openOrdersMeta.OpenOrdersV2.Decode(acctInfo.Value.Data); err != nil {
+	if err := openOrdersMeta.OpenOrders.Decode(acctInfo.Value.Data); err != nil {
 		return nil, fmt.Errorf("decoding market v2: %w", err)
 	}
 

programs/serum/rpc_test.go

@@ -48,7 +48,7 @@ func TestFetchMarket(t *testing.T) {
 	openOrders, err := FetchOpenOrders(ctx, client, openOrderAdd)
 	require.NoError(t, err)
 
-	cnt, err := json.MarshalIndent(openOrders.OpenOrdersV2, "", " ")
+	cnt, err := json.MarshalIndent(openOrders.OpenOrders, "", " ")
 
 	require.NoError(t, err)