rlp: add "tail" struct tag

Author: fjl

rlp/decode.go

@@ -63,11 +63,12 @@ type Decoder interface {
 // must contain an element for each decoded field. Decode returns an
 // error if there are too few or too many elements.
 //
-// The decoding of struct fields honours one particular struct tag,
-// "nil". This tag applies to pointer-typed fields and changes the
+// The decoding of struct fields honours two struct tags, "tail" and
+// "nil". For an explanation of "tail", see the example.
+// The "nil" tag applies to pointer-typed fields and changes the
 // decoding rules for the field such that input values of size zero
-// decode as a nil pointer. This tag can be useful when decoding recursive
-// types.
+// decode as a nil pointer. This tag can be useful when decoding
+// recursive types.
 //
 //     type StructWithEmptyOK struct {
 //         Foo *[20]byte `rlp:"nil"`
@@ -190,7 +191,7 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) {
 	case kind == reflect.String:
 		return decodeString, nil
 	case kind == reflect.Slice || kind == reflect.Array:
-		return makeListDecoder(typ)
+		return makeListDecoder(typ, tags)
 	case kind == reflect.Struct:
 		return makeStructDecoder(typ)
 	case kind == reflect.Ptr:
@@ -264,7 +265,7 @@ func decodeBigInt(s *Stream, val reflect.Value) error {
 	return nil
 }
 
-func makeListDecoder(typ reflect.Type) (decoder, error) {
+func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) {
 	etype := typ.Elem()
 	if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) {
 		if typ.Kind() == reflect.Array {
@@ -277,15 +278,26 @@ func makeListDecoder(typ reflect.Type) (decoder, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	isArray := typ.Kind() == reflect.Array
-	return func(s *Stream, val reflect.Value) error {
-		if isArray {
+	var dec decoder
+	switch {
+	case typ.Kind() == reflect.Array:
+		dec = func(s *Stream, val reflect.Value) error {
 			return decodeListArray(s, val, etypeinfo.decoder)
-		} else {
+		}
+	case tag.tail:
+		// A slice with "tail" tag can occur as the last field
+		// of a struct and is upposed to swallow all remaining
+		// list elements. The struct decoder already called s.List,
+		// proceed directly to decoding the elements.
+		dec = func(s *Stream, val reflect.Value) error {
+			return decodeSliceElems(s, val, etypeinfo.decoder)
+		}
+	default:
+		dec = func(s *Stream, val reflect.Value) error {
 			return decodeListSlice(s, val, etypeinfo.decoder)
 		}
-	}, nil
+	}
+	return dec, nil
 }
 
 func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error {
@@ -297,7 +309,13 @@ func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error {
 		val.Set(reflect.MakeSlice(val.Type(), 0, 0))
 		return s.ListEnd()
 	}
+	if err := decodeSliceElems(s, val, elemdec); err != nil {
+		return err
+	}
+	return s.ListEnd()
+}
 
+func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error {
 	i := 0
 	for ; ; i++ {
 		// grow slice if necessary
@@ -323,12 +341,11 @@ func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error {
 	if i < val.Len() {
 		val.SetLen(i)
 	}
-	return s.ListEnd()
+	return nil
 }
 
 func decodeListArray(s *Stream, val reflect.Value, elemdec decoder) error {
-	_, err := s.List()
-	if err != nil {
+	if _, err := s.List(); err != nil {
 		return wrapStreamError(err, val.Type())
 	}
 	vlen := val.Len()
@@ -398,11 +415,11 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) {
 		return nil, err
 	}
 	dec := func(s *Stream, val reflect.Value) (err error) {
-		if _, err = s.List(); err != nil {
+		if _, err := s.List(); err != nil {
 			return wrapStreamError(err, typ)
 		}
 		for _, f := range fields {
-			err = f.info.decoder(s, val.Field(f.index))
+			err := f.info.decoder(s, val.Field(f.index))
 			if err == EOL {
 				return &decodeError{msg: "too few elements", typ: typ}
 			} else if err != nil {

rlp/decode_tail_test.go

@@ -0,0 +1,33 @@
+package rlp
+
+import (
+	"bytes"
+	"fmt"
+)
+
+type structWithTail struct {
+	A, B uint
+	C    []uint `rlp:"tail"`
+}
+
+func ExampleDecode_structTagTail() {
+	// In this example, the "tail" struct tag is used to decode lists of
+	// differing length into a struct.
+	var val structWithTail
+
+	err := Decode(bytes.NewReader([]byte{0xC4, 0x01, 0x02, 0x03, 0x04}), &val)
+	fmt.Printf("with 4 elements: err=%v val=%v\n", err, val)
+
+	err = Decode(bytes.NewReader([]byte{0xC6, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), &val)
+	fmt.Printf("with 6 elements: err=%v val=%v\n", err, val)
+
+	// Note that at least two list elements must be present to
+	// fill fields A and B:
+	err = Decode(bytes.NewReader([]byte{0xC1, 0x01}), &val)
+	fmt.Printf("with 1 element: err=%q\n", err)
+
+	// Output:
+	// with 4 elements: err=<nil> val={1 2 [3 4]}
+	// with 6 elements: err=<nil> val={1 2 [3 4 5 6]}
+	// with 1 element: err="rlp: too few elements for rlp.structWithTail"
+}

rlp/decode_test.go

@@ -312,6 +312,26 @@ type recstruct struct {
 	Child *recstruct `rlp:"nil"`
 }
 
+type invalidTail1 struct {
+	A uint `rlp:"tail"`
+	B string
+}
+
+type invalidTail2 struct {
+	A uint
+	B string `rlp:"tail"`
+}
+
+type tailRaw struct {
+	A    uint
+	Tail []RawValue `rlp:"tail"`
+}
+
+type tailUint struct {
+	A    uint
+	Tail []uint `rlp:"tail"`
+}
+
 var (
 	veryBigInt = big.NewInt(0).Add(
 		big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16),
@@ -437,6 +457,38 @@ var decodeTests = []decodeTest{
 		ptr:   new(recstruct),
 		error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I",
 	},
+	{
+		input: "C0",
+		ptr:   new(invalidTail1),
+		error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail1.A (must be on last field)",
+	},
+	{
+		input: "C0",
+		ptr:   new(invalidTail2),
+		error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail2.B (field type is not slice)",
+	},
+	{
+		input: "C50102C20102",
+		ptr:   new(tailUint),
+		error: "rlp: expected input string or byte for uint, decoding into (rlp.tailUint).Tail[1]",
+	},
+
+	// struct tag "tail"
+	{
+		input: "C3010203",
+		ptr:   new(tailRaw),
+		value: tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}},
+	},
+	{
+		input: "C20102",
+		ptr:   new(tailRaw),
+		value: tailRaw{A: 1, Tail: []RawValue{unhex("02")}},
+	},
+	{
+		input: "C101",
+		ptr:   new(tailRaw),
+		value: tailRaw{A: 1, Tail: []RawValue{}},
+	},
 
 	// RawValue
 	{input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))},

rlp/encode.go

@@ -345,7 +345,7 @@ var (
 )
 
 // makeWriter creates a writer function for the given type.
-func makeWriter(typ reflect.Type) (writer, error) {
+func makeWriter(typ reflect.Type, ts tags) (writer, error) {
 	kind := typ.Kind()
 	switch {
 	case typ == rawValueType:
@@ -371,7 +371,7 @@ func makeWriter(typ reflect.Type) (writer, error) {
 	case kind == reflect.Array && isByte(typ.Elem()):
 		return writeByteArray, nil
 	case kind == reflect.Slice || kind == reflect.Array:
-		return makeSliceWriter(typ)
+		return makeSliceWriter(typ, ts)
 	case kind == reflect.Struct:
 		return makeStructWriter(typ)
 	case kind == reflect.Ptr:
@@ -507,20 +507,21 @@ func writeInterface(val reflect.Value, w *encbuf) error {
 	return ti.writer(eval, w)
 }
 
-func makeSliceWriter(typ reflect.Type) (writer, error) {
+func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) {
 	etypeinfo, err := cachedTypeInfo1(typ.Elem(), tags{})
 	if err != nil {
 		return nil, err
 	}
 	writer := func(val reflect.Value, w *encbuf) error {
-		lh := w.list()
+		if !ts.tail {
+			defer w.listEnd(w.list())
+		}
 		vlen := val.Len()
 		for i := 0; i < vlen; i++ {
 			if err := etypeinfo.writer(val.Index(i), w); err != nil {
 				return err
 			}
 		}
-		w.listEnd(lh)
 		return nil
 	}
 	return writer, nil

rlp/encode_test.go

@@ -214,6 +214,10 @@ var encTests = []encTest{
 	{val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"},
 	{val: &recstruct{5, nil}, output: "C205C0"},
 	{val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"},
+	{val: &tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, output: "C3010203"},
+	{val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"},
+	{val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"},
+	{val: &tailRaw{A: 1, Tail: nil}, output: "C101"},
 
 	// nil
 	{val: (*uint)(nil), output: "80"},

rlp/typecache.go

@@ -17,7 +17,9 @@
 package rlp
 
 import (
+	"fmt"
 	"reflect"
+	"strings"
 	"sync"
 )
 
@@ -33,7 +35,13 @@ type typeinfo struct {
 
 // represents struct tags
 type tags struct {
+	// rlp:"nil" controls whether empty input results in a nil pointer.
 	nilOK bool
+
+	// rlp:"tail" controls whether this field swallows additional list
+	// elements. It can only be set for the last field, which must be
+	// of slice type.
+	tail bool
 }
 
 type typekey struct {
@@ -89,7 +97,10 @@ type field struct {
 func structFields(typ reflect.Type) (fields []field, err error) {
 	for i := 0; i < typ.NumField(); i++ {
 		if f := typ.Field(i); f.PkgPath == "" { // exported
-			tags := parseStructTag(f.Tag.Get("rlp"))
+			tags, err := parseStructTag(typ, i)
+			if err != nil {
+				return nil, err
+			}
 			info, err := cachedTypeInfo1(f.Type, tags)
 			if err != nil {
 				return nil, err
@@ -100,16 +111,35 @@ func structFields(typ reflect.Type) (fields []field, err error) {
 	return fields, nil
 }
 
-func parseStructTag(tag string) tags {
-	return tags{nilOK: tag == "nil"}
+func parseStructTag(typ reflect.Type, fi int) (tags, error) {
+	f := typ.Field(fi)
+	var ts tags
+	for _, t := range strings.Split(f.Tag.Get("rlp"), ",") {
+		switch t = strings.TrimSpace(t); t {
+		case "":
+		case "nil":
+			ts.nilOK = true
+		case "tail":
+			ts.tail = true
+			if fi != typ.NumField()-1 {
+				return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (must be on last field)`, typ, f.Name)
+			}
+			if f.Type.Kind() != reflect.Slice {
+				return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (field type is not slice)`, typ, f.Name)
+			}
+		default:
+			return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name)
+		}
+	}
+	return ts, nil
 }
 
 func genTypeInfo(typ reflect.Type, tags tags) (info *typeinfo, err error) {
 	info = new(typeinfo)
 	if info.decoder, err = makeDecoder(typ, tags); err != nil {
 		return nil, err
 	}
-	if info.writer, err = makeWriter(typ); err != nil {
+	if info.writer, err = makeWriter(typ, tags); err != nil {
 		return nil, err
 	}
 	return info, nil