Add ability to serialize Nil pointers

All pointers have a nil/not nil prefix, it tries to be backwards compatible by
ignoring if such prefix is missing. It is also a 5 byte payload to avoid of
serialized messages to have those 5 bytes by change

Author: nytzuga

codec/reflectcodec/type_codec.go

@@ -19,6 +19,12 @@ import (
 // DefaultTagName that enables serialization.
 const DefaultTagName = "serialize"
 
+var (
+	MagicBytesPtrNil        = []byte{0xff, 0x32, 0x31, 0x36, 0x98}
+	MagicBytesPtr           = []byte{0xc3, 0xa1, 0x00, 0x33, 0x25}
+	magicBytesPtrPrefixSize = len(MagicBytesPtrNil)
+)
+
 var (
 	_ codec.Codec = (*genericCodec)(nil)
 
@@ -85,7 +91,13 @@ func (c *genericCodec) Size(value interface{}) (int, error) {
 		return 0, errMarshalNil // can't marshal nil
 	}
 
-	size, _, err := c.size(reflect.ValueOf(value))
+	metaValue := reflect.ValueOf(value)
+	switch valueKind := metaValue.Kind(); valueKind {
+	case reflect.Ptr:
+		metaValue = metaValue.Elem()
+	}
+
+	size, _, err := c.size(metaValue)
 	return size, err
 }
 
@@ -114,10 +126,13 @@ func (c *genericCodec) size(value reflect.Value) (int, bool, error) {
 		return wrappers.StringLen(value.String()), false, nil
 	case reflect.Ptr:
 		if value.IsNil() {
-			// Can't marshal nil pointers (but nil slices are fine)
-			return 0, false, errMarshalNil
+			return magicBytesPtrPrefixSize, false, nil
 		}
-		return c.size(value.Elem())
+		size, _, err := c.size(value.Elem())
+		if err != nil {
+			return 0, false, err
+		}
+		return magicBytesPtrPrefixSize + size, false, nil
 
 	case reflect.Interface:
 		if value.IsNil() {
@@ -279,7 +294,13 @@ func (c *genericCodec) MarshalInto(value interface{}, p *wrappers.Packer) error
 		return errMarshalNil // can't marshal nil
 	}
 
-	return c.marshal(reflect.ValueOf(value), p, c.maxSliceLen)
+	metaValue := reflect.ValueOf(value)
+	switch valueKind := metaValue.Kind(); valueKind {
+	case reflect.Ptr:
+		metaValue = metaValue.Elem()
+	}
+
+	return c.marshal(metaValue, p, c.maxSliceLen)
 }
 
 // marshal writes the byte representation of [value] to [p]
@@ -318,8 +339,13 @@ func (c *genericCodec) marshal(value reflect.Value, p *wrappers.Packer, maxSlice
 		p.PackBool(value.Bool())
 		return p.Err
 	case reflect.Ptr:
-		if value.IsNil() { // Can't marshal nil (except nil slices)
-			return errMarshalNil
+		if value.IsNil() {
+			p.PackFixedBytes(MagicBytesPtrNil)
+			return p.Err
+		}
+		p.PackFixedBytes(MagicBytesPtr)
+		if p.Err != nil {
+			return p.Err
 		}
 		return c.marshal(value.Elem(), p, c.maxSliceLen)
 	case reflect.Interface:
@@ -643,6 +669,10 @@ func (c *genericCodec) unmarshal(p *wrappers.Packer, value reflect.Value, maxSli
 		// Create a new pointer to a new value of the underlying type
 		v := reflect.New(t)
 		// Fill the value
+		if bytes.Equal(MagicBytesPtrNil, p.UnpackIfMatches(MagicBytesPtrNil, MagicBytesPtr)) {
+			// Nil
+			return nil
+		}
 		if err := c.unmarshal(p, v.Elem(), c.maxSliceLen); err != nil {
 			return fmt.Errorf("couldn't unmarshal pointer: %w", err)
 		}

codec/test_codec.go

@@ -23,6 +23,8 @@ var (
 		TestBigArray,
 		TestPointerToStruct,
 		TestSliceOfStruct,
+		TestStructWithPtr,
+		TestStructWithPtrNil,
 		TestInterface,
 		TestSliceOfInterface,
 		TestArrayOfInterface,
@@ -63,7 +65,8 @@ type Foo interface {
 }
 
 type MyInnerStruct struct {
-	Str string `serialize:"true"`
+	Str               string `serialize:"true"`
+	NumberNotProvided *int32 `serialize:"true":121`
 }
 
 func (*MyInnerStruct) Foo() int {
@@ -86,6 +89,11 @@ type MyInnerStruct3 struct {
 	F   Foo           `serialize:"true"`
 }
 
+type MyStructWithPtr struct {
+	N1 *int32 `serialize:"true"`
+	N2 *int64 `serialize:"true"`
+}
+
 type myStruct struct {
 	InnerStruct  MyInnerStruct               `serialize:"true"`
 	InnerStruct2 *MyInnerStruct              `serialize:"true"`
@@ -145,21 +153,23 @@ func TestStruct(codec GeneralCodec, t testing.TB) {
 	myMap7["key"] = "value"
 	myMap7[int32(1)] = int32(2)
 
+	number := int32(8)
+
 	myStructInstance := myStruct{
-		InnerStruct:  MyInnerStruct{"hello"},
-		InnerStruct2: &MyInnerStruct{"yello"},
+		InnerStruct:  MyInnerStruct{"hello", nil},
+		InnerStruct2: &MyInnerStruct{"yello", nil},
 		Member1:      1,
 		Member2:      2,
 		MySlice:      []byte{1, 2, 3, 4},
 		MySlice2:     []string{"one", "two", "three"},
-		MySlice3:     []MyInnerStruct{{"abc"}, {"ab"}, {"c"}},
+		MySlice3:     []MyInnerStruct{{"abc", nil}, {"ab", &number}, {"c", nil}},
 		MySlice4:     []*MyInnerStruct2{{true}, {}},
 		MySlice5:     []Foo{&MyInnerStruct2{true}, &MyInnerStruct2{}},
 		MyArray:      [4]byte{5, 6, 7, 8},
 		MyArray2:     [5]string{"four", "five", "six", "seven"},
-		MyArray3:     [3]MyInnerStruct{{"d"}, {"e"}, {"f"}},
+		MyArray3:     [3]MyInnerStruct{{"d", nil}, {"e", nil}, {"f", nil}},
 		MyArray4:     [2]*MyInnerStruct2{{}, {true}},
-		MyInterface:  &MyInnerStruct{"yeet"},
+		MyInterface:  &MyInnerStruct{"yeet", &number},
 		InnerStruct3: MyInnerStruct3{
 			Str: "str",
 			M1: MyInnerStruct{
@@ -414,20 +424,70 @@ func TestPointerToStruct(codec GeneralCodec, t testing.TB) {
 	require.Equal(myPtr, myPtrUnmarshaled)
 }
 
+func TestStructWithPtr(codec GeneralCodec, t testing.TB) {
+	require := require.New(t)
+	n1 := int32(5)
+	n2 := int64(10)
+	struct1 := MyStructWithPtr{
+		N1: &n1,
+		N2: &n2,
+	}
+
+	require.NoError(codec.RegisterType(&MyStructWithPtr{}))
+	manager := NewDefaultManager()
+	require.NoError(manager.RegisterCodec(0, codec))
+
+	bytes, err := manager.Marshal(0, struct1)
+	require.NoError(err)
+
+	bytesLen, err := manager.Size(0, struct1)
+	require.NoError(err)
+	require.Len(bytes, bytesLen)
+
+	var struct1Unmarshaled MyStructWithPtr
+	version, err := manager.Unmarshal(bytes, &struct1Unmarshaled)
+	require.NoError(err)
+	require.Zero(version)
+	require.Equal(struct1, struct1Unmarshaled)
+}
+
+func TestStructWithPtrNil(codec GeneralCodec, t testing.TB) {
+	require := require.New(t)
+	struct1 := MyStructWithPtr{}
+
+	require.NoError(codec.RegisterType(&MyStructWithPtr{}))
+	manager := NewDefaultManager()
+	require.NoError(manager.RegisterCodec(0, codec))
+
+	bytes, err := manager.Marshal(0, struct1)
+	require.NoError(err)
+
+	bytesLen, err := manager.Size(0, struct1)
+	require.NoError(err)
+	require.Len(bytes, bytesLen)
+
+	var struct1Unmarshaled MyStructWithPtr
+	version, err := manager.Unmarshal(bytes, &struct1Unmarshaled)
+	require.NoError(err)
+	require.Zero(version)
+	require.Equal(struct1, struct1Unmarshaled)
+}
+
 // Test marshalling a slice of structs
 func TestSliceOfStruct(codec GeneralCodec, t testing.TB) {
 	require := require.New(t)
-
+	n1 := int32(-1)
+	n2 := int32(0xff)
 	mySlice := []MyInnerStruct3{
 		{
 			Str: "One",
-			M1:  MyInnerStruct{"Two"},
-			F:   &MyInnerStruct{"Three"},
+			M1:  MyInnerStruct{"Two", &n1},
+			F:   &MyInnerStruct{"Three", &n2},
 		},
 		{
 			Str: "Four",
-			M1:  MyInnerStruct{"Five"},
-			F:   &MyInnerStruct{"Six"},
+			M1:  MyInnerStruct{"Five", nil},
+			F:   &MyInnerStruct{"Six", nil},
 		},
 	}
 	require.NoError(codec.RegisterType(&MyInnerStruct{}))

utils/wrappers/packing.go

@@ -4,6 +4,7 @@
 package wrappers
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"math"
@@ -179,6 +180,23 @@ func (p *Packer) PackFixedBytes(bytes []byte) {
 	p.Offset += len(bytes)
 }
 
+func (p *Packer) UnpackIfMatches(values ...[]byte) []byte {
+	totalLength := len(p.Bytes)
+	for _, value := range values {
+		length := len(value)
+		if totalLength-p.Offset < length {
+			continue
+		}
+
+		if bytes.Equal(value, p.Bytes[p.Offset:p.Offset+length]) {
+			p.Offset += length
+			return value
+		}
+	}
+
+	return nil
+}
+
 // UnpackFixedBytes unpack a byte slice, with no length descriptor from the byte
 // array
 func (p *Packer) UnpackFixedBytes(size int) []byte {