[FAB-4104] Proto translator dynamic field comp

The proto translation framework introduced in FAB-4100 needs to be
applied to messages with dynamic fields.  Dynamic fields are fields
whose message may need to be unmarshaled differently depending on
context.

For instance, different ConfigValue messages may resolve the same field
to different types, depending on the context in which the ConfigValue is
evaluated.

This CR adds a dynamic proto msg handler to the proto translation
framework.

Change-Id: I4bf809721f9af37778aeb3df3d706bfda1ed4e7b
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

common/tools/protolator/api.go

@@ -27,7 +27,7 @@ import (
 // opaque byte fields are represented as their expanded proto contents) and back once again
 // to standard proto messages.
 //
-// There are currently two different types of interfaces available for protos to implement:
+// There are currently three different types of interfaces available for protos to implement:
 //
 // 1. StaticallyOpaque*FieldProto:  These interfaces should be implemented by protos which have
 // opaque byte fields whose marshaled type is known at compile time.  This is mostly true
@@ -40,6 +40,13 @@ import (
 // For example, the Payload.data field depends upon the Payload.Header.ChannelHeader.type field,
 // which is along a statically marshaled path.
 //
+// 3. Dynamic*FieldProto: These interfaces are for messages which contain other messages whose
+// attributes cannot be determined at compile time.  For example, a ConfigValue message may evaluate
+// the map field values["MSP"] successfully in an organization context, but not at all in a channel
+// context.  Because go is not a dynamic language, this dynamic behavior must be simulated by
+// wrapping the underlying proto message in another type which can be configured at runtime with
+// different contextual behavior. (See tests for examples)
+//
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
 // StaticallyOpaqueFieldProto should be implemented by protos which have bytes fields which
@@ -107,3 +114,44 @@ type VariablyOpaqueSliceFieldProto interface {
 	// type for the field name.
 	VariablyOpaqueSliceFieldProto(name string, index int) (proto.Message, error)
 }
+
+// DynamicFieldProto should be implemented by protos which have nested fields whose attributes
+// (such as their opaque types) cannot be determined until runtime
+type DynamicFieldProto interface {
+	// DynamicFields returns the field names which are dynamic
+	DynamicFields() []string
+
+	// DynamicFieldProto returns a newly allocated dynamic message, decorating an underlying
+	// proto message with the runtime determined function
+	DynamicFieldProto(name string, underlying proto.Message) (proto.Message, error)
+}
+
+// DynamicMapFieldProto should be implemented by protos which have maps to messages whose attributes
+// (such as their opaque types) cannot be determined until runtime
+type DynamicMapFieldProto interface {
+	// DynamicFields returns the field names which are dynamic
+	DynamicMapFields() []string
+
+	// DynamicMapFieldProto returns a newly allocated dynamic message, decorating an underlying
+	// proto message with the runtime determined function
+	DynamicMapFieldProto(name string, key string, underlying proto.Message) (proto.Message, error)
+}
+
+// DynamicSliceFieldProto should be implemented by protos which have slices of messages whose attributes
+// (such as their opaque types) cannot be determined until runtime
+type DynamicSliceFieldProto interface {
+	// DynamicFields returns the field names which are dynamic
+	DynamicSliceFields() []string
+
+	// DynamicSliceFieldProto returns a newly allocated dynamic message, decorating an underlying
+	// proto message with the runtime determined function
+	DynamicSliceFieldProto(name string, index int, underlying proto.Message) (proto.Message, error)
+}
+
+// DecoreatedProto should be implemented by the dynamic wrappers applied by the Dynamic*FieldProto interfaces
+// This is necessary for the proto system to unmarshal, because it discovers proto message type by reflection
+// (Rather than by interface definition as it probably should ( https://github.com/golang/protobuf/issues/291 )
+type DecoratedProto interface {
+	// Underlying returns the underlying proto message which is being dynamically decorated
+	Underlying() proto.Message
+}

common/tools/protolator/dynamic.go

@@ -0,0 +1,149 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protolator
+
+import (
+	"reflect"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func dynamicFrom(dynamicMsg func(underlying proto.Message) (proto.Message, error), value interface{}, destType reflect.Type) (reflect.Value, error) {
+	tree := value.(map[string]interface{}) // Safe, already checked
+	uMsg := reflect.New(destType.Elem())
+	nMsg, err := dynamicMsg(uMsg.Interface().(proto.Message)) // Safe, already checked
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	if err := recursivelyPopulateMessageFromTree(tree, nMsg); err != nil {
+		return reflect.Value{}, err
+	}
+	return uMsg, nil
+}
+
+func dynamicTo(dynamicMsg func(underlying proto.Message) (proto.Message, error), value reflect.Value) (interface{}, error) {
+	nMsg, err := dynamicMsg(value.Interface().(proto.Message)) // Safe, already checked
+	if err != nil {
+		return nil, err
+	}
+	return recursivelyCreateTreeFromMessage(nMsg)
+}
+
+type dynamicFieldFactory struct{}
+
+func (dff dynamicFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	dynamicProto, ok := msg.(DynamicFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, dynamicProto.DynamicFields())
+}
+
+func (dff dynamicFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	dynamicProto, _ := msg.(DynamicFieldProto) // Type checked in Handles
+
+	return &plainField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: fieldType,
+			value: fieldValue,
+		},
+		populateFrom: func(v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return dynamicFrom(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicFieldProto(fieldName, underlying)
+			}, v, dT)
+		},
+		populateTo: func(v reflect.Value) (interface{}, error) {
+			return dynamicTo(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicFieldProto(fieldName, underlying)
+			}, v)
+		},
+	}, nil
+}
+
+type dynamicMapFieldFactory struct{}
+
+func (dmff dynamicMapFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	dynamicProto, ok := msg.(DynamicMapFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, dynamicProto.DynamicMapFields())
+}
+
+func (dmff dynamicMapFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	dynamicProto := msg.(DynamicMapFieldProto) // Type checked by Handles
+
+	return &mapField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: fieldType,
+			value: fieldValue,
+		},
+		populateFrom: func(k string, v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return dynamicFrom(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicMapFieldProto(fieldName, k, underlying)
+			}, v, dT)
+		},
+		populateTo: func(k string, v reflect.Value) (interface{}, error) {
+			return dynamicTo(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicMapFieldProto(fieldName, k, underlying)
+			}, v)
+		},
+	}, nil
+}
+
+type dynamicSliceFieldFactory struct{}
+
+func (dmff dynamicSliceFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	dynamicProto, ok := msg.(DynamicSliceFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, dynamicProto.DynamicSliceFields())
+}
+
+func (dmff dynamicSliceFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	dynamicProto := msg.(DynamicSliceFieldProto) // Type checked by Handles
+
+	return &sliceField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: fieldType,
+			value: fieldValue,
+		},
+		populateFrom: func(i int, v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return dynamicFrom(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicSliceFieldProto(fieldName, i, underlying)
+			}, v, dT)
+		},
+		populateTo: func(i int, v reflect.Value) (interface{}, error) {
+			return dynamicTo(func(underlying proto.Message) (proto.Message, error) {
+				return dynamicProto.DynamicSliceFieldProto(fieldName, i, underlying)
+			}, v)
+		},
+	}, nil
+}

common/tools/protolator/dynamic_test.go

@@ -0,0 +1,134 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protolator
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/hyperledger/fabric/common/tools/protolator/testprotos"
+	"github.com/hyperledger/fabric/protos/utils"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPlainDynamicMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}}
+
+	pfValue := "foo"
+	startMsg := &testprotos.DynamicMsg{
+		DynamicType: "SimpleMsg",
+		PlainDynamicField: &testprotos.ContextlessMsg{
+			OpaqueField: utils.MarshalOrPanic(&testprotos.SimpleMsg{
+				PlainField: pfValue,
+			}),
+		},
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.DynamicMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.PlainDynamicField.OpaqueField), extractSimpleMsgPlainField(newMsg.PlainDynamicField.OpaqueField))
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}, dynamicFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.PlainDynamicField.OpaqueField), extractSimpleMsgPlainField(newMsg.PlainDynamicField.OpaqueField))
+}
+
+func TestMapDynamicMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}}
+
+	pfValue := "foo"
+	mapKey := "bar"
+	startMsg := &testprotos.DynamicMsg{
+		DynamicType: "SimpleMsg",
+		MapDynamicField: map[string]*testprotos.ContextlessMsg{
+			mapKey: &testprotos.ContextlessMsg{
+				OpaqueField: utils.MarshalOrPanic(&testprotos.SimpleMsg{
+					PlainField: pfValue,
+				}),
+			},
+		},
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.DynamicMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.MapDynamicField[mapKey].OpaqueField), extractSimpleMsgPlainField(newMsg.MapDynamicField[mapKey].OpaqueField))
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}, dynamicMapFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.MapDynamicField[mapKey].OpaqueField), extractSimpleMsgPlainField(newMsg.MapDynamicField[mapKey].OpaqueField))
+}
+
+func TestSliceDynamicMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}}
+
+	pfValue := "foo"
+	startMsg := &testprotos.DynamicMsg{
+		DynamicType: "SimpleMsg",
+		SliceDynamicField: []*testprotos.ContextlessMsg{
+			&testprotos.ContextlessMsg{
+				OpaqueField: utils.MarshalOrPanic(&testprotos.SimpleMsg{
+					PlainField: pfValue,
+				}),
+			},
+		},
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.DynamicMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.SliceDynamicField[0].OpaqueField), extractSimpleMsgPlainField(newMsg.SliceDynamicField[0].OpaqueField))
+
+	fieldFactories = []protoFieldFactory{tppff, variablyOpaqueFieldFactory{}, dynamicSliceFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.SliceDynamicField[0].OpaqueField), extractSimpleMsgPlainField(newMsg.SliceDynamicField[0].OpaqueField))
+}

common/tools/protolator/json.go

@@ -251,6 +251,9 @@ func jsonToMap(marshaled []byte) (map[string]interface{}, error) {
 // Factories listed lower, may depend on factories listed higher being
 // evaluated first.
 var fieldFactories = []protoFieldFactory{
+	dynamicSliceFieldFactory{},
+	dynamicMapFieldFactory{},
+	dynamicFieldFactory{},
 	variablyOpaqueSliceFieldFactory{},
 	variablyOpaqueMapFieldFactory{},
 	variablyOpaqueFieldFactory{},
@@ -326,6 +329,10 @@ func recursivelyCreateTreeFromMessage(msg proto.Message) (tree map[string]interf
 	}()
 
 	uMsg := msg
+	decorated, ok := msg.(DecoratedProto)
+	if ok {
+		uMsg = decorated.Underlying()
+	}
 
 	fields, err := protoFields(msg, uMsg)
 	if err != nil {
@@ -381,6 +388,10 @@ func recursivelyPopulateMessageFromTree(tree map[string]interface{}, msg proto.M
 	}()
 
 	uMsg := msg
+	decorated, ok := msg.(DecoratedProto)
+	if ok {
+		uMsg = decorated.Underlying()
+	}
 
 	fields, err := protoFields(msg, uMsg)
 	if err != nil {