binary encoding and some changes to nson benchmarks.

binary/README.md

@@ -0,0 +1,18 @@
+# The simplest binary encoding for Nostr events
+
+Some benchmarks:
+
+goos: linux
+goarch: amd64
+pkg: github.com/nbd-wtf/go-nostr/binary
+cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics
+BenchmarkBinaryEncoding/easyjson.Marshal-4                 24488             53274 ns/op           35191 B/op        102 allocs/op
+BenchmarkBinaryEncoding/binary.Marshal-4                    5066            218284 ns/op         1282116 B/op         88 allocs/op
+BenchmarkBinaryEncoding/binary.MarshalBinary-4              5743            191603 ns/op         1277763 B/op         37 allocs/op
+BenchmarkBinaryDecoding/easyjson.Unmarshal-4               32701             38647 ns/op           45832 B/op        124 allocs/op
+BenchmarkBinaryDecoding/binary.Unmarshal-4                 85705             14249 ns/op           25488 B/op        141 allocs/op
+BenchmarkBinaryDecoding/binary.UnmarshalBinary-4          213438              5451 ns/op           16784 B/op         39 allocs/op
+BenchmarkBinaryDecoding/easyjson.Unmarshal+sig-4             307           3971993 ns/op          131639 B/op        404 allocs/op
+BenchmarkBinaryDecoding/binary.Unmarshal+sig-4               310           3924042 ns/op          111277 B/op        421 allocs/op
+PASS
+ok      github.com/nbd-wtf/go-nostr/binary      11.444s

binary/binary.go

@@ -0,0 +1,76 @@
+package binary
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func UnmarshalBinary(data []byte, evt *Event) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("failed to decode leaner: %v", r)
+		}
+	}()
+
+	copy(evt.ID[:], data[0:32])
+	copy(evt.PubKey[:], data[32:64])
+	copy(evt.Sig[:], data[64:128])
+	evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
+	evt.Kind = binary.BigEndian.Uint16(data[132:134])
+	contentLength := int(binary.BigEndian.Uint16(data[134:136]))
+	evt.Content = string(data[136 : 136+contentLength])
+
+	curr := 136 + contentLength
+	ntags := int(data[curr])
+	evt.Tags = make(nostr.Tags, ntags)
+
+	for t := range evt.Tags {
+		curr = curr + 1
+		nItems := int(data[curr])
+		tag := make(nostr.Tag, nItems)
+		for i := range tag {
+			curr = curr + 1
+			itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
+			itemStart := curr + 2
+			itemEnd := itemStart + itemSize
+			item := string(data[itemStart:itemEnd])
+			tag[i] = item
+			curr = itemEnd
+		}
+		evt.Tags[t] = tag
+	}
+
+	return err
+}
+
+func MarshalBinary(evt *Event) []byte {
+	content := []byte(evt.Content)
+	buf := make([]byte, 32+32+64+4+2+2+len(content)+65536 /* blergh */)
+	copy(buf[0:32], evt.ID[:])
+	copy(buf[32:64], evt.PubKey[:])
+	copy(buf[64:128], evt.Sig[:])
+	binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
+	binary.BigEndian.PutUint16(buf[132:134], evt.Kind)
+	binary.BigEndian.PutUint16(buf[134:136], uint16(len(content)))
+	copy(buf[136:], content)
+
+	curr := 136 + len(content)
+	buf[curr] = uint8(len(evt.Tags))
+	for _, tag := range evt.Tags {
+		curr++
+		buf[curr] = uint8(len(tag))
+		for _, item := range tag {
+			curr++
+			itemb := []byte(item)
+			itemSize := len(itemb)
+			binary.BigEndian.PutUint16(buf[curr:curr+2], uint16(itemSize))
+			itemEnd := curr + 2 + itemSize
+			copy(buf[curr+2:itemEnd], itemb)
+			curr = itemEnd
+		}
+	}
+	buf = buf[0 : curr+1]
+	return buf
+}

binary/event.go

@@ -0,0 +1,44 @@
+package binary
+
+import (
+	"encoding/hex"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type Event struct {
+	PubKey    [32]byte
+	Sig       [64]byte
+	ID        [32]byte
+	Kind      uint16
+	CreatedAt nostr.Timestamp
+	Content   string
+	Tags      nostr.Tags
+}
+
+func BinaryEvent(evt *nostr.Event) *Event {
+	bevt := Event{
+		Tags:      evt.Tags,
+		Content:   evt.Content,
+		Kind:      uint16(evt.Kind),
+		CreatedAt: evt.CreatedAt,
+	}
+
+	hex.Decode(bevt.ID[:], []byte(evt.ID))
+	hex.Decode(bevt.PubKey[:], []byte(evt.PubKey))
+	hex.Decode(bevt.Sig[:], []byte(evt.Sig))
+
+	return &bevt
+}
+
+func (bevt *Event) ToNormalEvent() *nostr.Event {
+	return &nostr.Event{
+		Tags:      bevt.Tags,
+		Content:   bevt.Content,
+		Kind:      int(bevt.Kind),
+		CreatedAt: bevt.CreatedAt,
+		ID:        hex.EncodeToString(bevt.ID[:]),
+		PubKey:    hex.EncodeToString(bevt.PubKey[:]),
+		Sig:       hex.EncodeToString(bevt.Sig[:]),
+	}
+}

binary/hybrid.go

@@ -0,0 +1,79 @@
+package binary
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func Unmarshal(data []byte, evt *nostr.Event) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("failed to decode leaner: %v", r)
+		}
+	}()
+
+	evt.ID = hex.EncodeToString(data[0:32])
+	evt.PubKey = hex.EncodeToString(data[32:64])
+	evt.Sig = hex.EncodeToString(data[64:128])
+	evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
+	evt.Kind = int(binary.BigEndian.Uint16(data[132:134]))
+	contentLength := int(binary.BigEndian.Uint16(data[134:136]))
+	evt.Content = string(data[136 : 136+contentLength])
+
+	curr := 136 + contentLength
+	ntags := int(data[curr])
+	evt.Tags = make(nostr.Tags, ntags)
+
+	for t := range evt.Tags {
+		curr = curr + 1
+		nItems := int(data[curr])
+		tag := make(nostr.Tag, nItems)
+		for i := range tag {
+			curr = curr + 1
+			itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
+			itemStart := curr + 2
+			itemEnd := itemStart + itemSize
+			item := string(data[itemStart:itemEnd])
+			tag[i] = item
+			curr = itemEnd
+		}
+		evt.Tags[t] = tag
+	}
+
+	return err
+}
+
+func Marshal(evt *nostr.Event) ([]byte, error) {
+	content := []byte(evt.Content)
+	buf := make([]byte, 32+32+64+4+2+2+len(content)+65536 /* blergh */)
+
+	hex.Decode(buf[0:32], []byte(evt.ID))
+	hex.Decode(buf[32:64], []byte(evt.PubKey))
+	hex.Decode(buf[64:128], []byte(evt.Sig))
+
+	binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
+	binary.BigEndian.PutUint16(buf[132:134], uint16(evt.Kind))
+	binary.BigEndian.PutUint16(buf[134:136], uint16(len(content)))
+	copy(buf[136:], content)
+
+	curr := 136 + len(content)
+	buf[curr] = uint8(len(evt.Tags))
+	for _, tag := range evt.Tags {
+		curr++
+		buf[curr] = uint8(len(tag))
+		for _, item := range tag {
+			curr++
+			itemb := []byte(item)
+			itemSize := len(itemb)
+			binary.BigEndian.PutUint16(buf[curr:curr+2], uint16(itemSize))
+			itemEnd := curr + 2 + itemSize
+			copy(buf[curr+2:itemEnd], itemb)
+			curr = itemEnd
+		}
+	}
+	buf = buf[0 : curr+1]
+	return buf, nil
+}

nson/README.md

@@ -35,3 +35,23 @@ It's explained better in the NIP proposal linked above, but the idea is that we
 a special JSON attribute called `"nson"`, and then the reader can just pull the strings directly from the JSON blob
 without having to parse the full JSON syntax. Also for fields of static size we don't even need that. This is only
 possible because Nostr events have a static and strict format.
+
+## Update: comparison with `easyjson`
+
+Another comparison, using the `easyjson` library that is already built in `go-nostr`, shows that the performance gains
+are only of 2x (the standard library JSON encoding is just too slow).
+
+```
+goos: linux
+goarch: amd64
+pkg: github.com/nbd-wtf/go-nostr/nson
+cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics
+BenchmarkNSONEncoding/easyjson.Marshal-4                   21511             54849 ns/op
+BenchmarkNSONEncoding/nson.Marshal-4                        4810            297624 ns/op
+BenchmarkNSONDecoding/easyjson.Unmarshal-4                 25196             46652 ns/op
+BenchmarkNSONDecoding/nson.Unmarshal-4                     61117             22933 ns/op
+BenchmarkNSONDecoding/easyjson.Unmarshal+sig-4               303           4110988 ns/op
+BenchmarkNSONDecoding/nson.Unmarshal+sig-4                   296           3881435 ns/op
+PASS
+ok      github.com/nbd-wtf/go-nostr/nson
+```

nson/nson_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/mailru/easyjson"
 	"github.com/nbd-wtf/go-nostr"
 )
 
@@ -153,10 +154,10 @@ func BenchmarkNSONEncoding(b *testing.B) {
 		events[i] = evt
 	}
 
-	b.Run("json.Marshal", func(b *testing.B) {
+	b.Run("easyjson.Marshal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, evt := range events {
-				json.Marshal(evt)
+				easyjson.Marshal(evt)
 			}
 		}
 	})
@@ -168,32 +169,22 @@ func BenchmarkNSONEncoding(b *testing.B) {
 			}
 		}
 	})
-
-	b.Run("nson.Marshal to bytes", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			for _, evt := range events {
-				MarshalBytes(evt)
-			}
-		}
-	})
 }
 
 func BenchmarkNSONDecoding(b *testing.B) {
 	events := make([]string, len(normalEvents))
-	eventsB := make([][]byte, len(normalEvents))
 	for i, jevt := range normalEvents {
 		evt := &nostr.Event{}
 		json.Unmarshal([]byte(jevt), evt)
 		nevt, _ := Marshal(evt)
 		events[i] = nevt
-		eventsB[i] = []byte(nevt)
 	}
 
-	b.Run("json.Unmarshal", func(b *testing.B) {
+	b.Run("easyjson.Unmarshal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, nevt := range events {
 				evt := &nostr.Event{}
-				err := json.Unmarshal([]byte(nevt), evt)
+				err := easyjson.Unmarshal([]byte(nevt), evt)
 				if err != nil {
 					b.Fatalf("failed to unmarshal: %s", err)
 				}
@@ -213,23 +204,11 @@ func BenchmarkNSONDecoding(b *testing.B) {
 		}
 	})
 
-	b.Run("nson.Unmarshal from bytes", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			for _, nevt := range eventsB {
-				evt := &nostr.Event{}
-				err := UnmarshalBytes(nevt, evt)
-				if err != nil {
-					b.Fatalf("failed to unmarshal: %s", err)
-				}
-			}
-		}
-	})
-
-	b.Run("json.Unmarshal + sig verification", func(b *testing.B) {
+	b.Run("easyjson.Unmarshal+sig", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, nevt := range events {
 				evt := &nostr.Event{}
-				err := json.Unmarshal([]byte(nevt), evt)
+				err := easyjson.Unmarshal([]byte(nevt), evt)
 				if err != nil {
 					b.Fatalf("failed to unmarshal: %s", err)
 				}
@@ -238,7 +217,7 @@ func BenchmarkNSONDecoding(b *testing.B) {
 		}
 	})
 
-	b.Run("nson.Unmarshal + sig verification", func(b *testing.B) {
+	b.Run("nson.Unmarshal+sig", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, nevt := range events {
 				evt := &nostr.Event{}