[FAB-6139] Add new channel config encoder package

configtxgen currently depends on the 'provisional' package for encoding
the information contained in configtx.yaml into the fabric native proto
format.  The provisional package was originally developed as part of the
orderer testing framework, and was never designed to work as it does
today.

This CR introduces a new encoder which more directly translates between
the viper config struct and the native proto format.  It relies on much
less indirection and should be easier for users to understand, easier to
hack on, and already has significantly better test coverage.

In addition, the new encoder allows for encoding of a subset of the
configuration, which is very useful for tests and tools.  There are many
places in fabric codebase which use the provisional package to generate
a genesis block, then unpack the the block, unpack the envelope, unpack
the config envelope, and finally access the config inside, and the new
package allows producing the config directly without these intermediate
steps.

This CR simply introduces the new encoder and tests.  It utilizes the
new simple util functions from the channelconfig and policies packages.

In subsequent CRs, the old provisional package will be removed, as will
the assorted old 'Template<...>' methods it depended upon.

Change-Id: I827504af301631171fc5a5998637b31e99c8dc4b
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

common/tools/configtxgen/encoder/encoder.go

@@ -0,0 +1,274 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package encoder
+
+import (
+	"github.com/hyperledger/fabric/common/cauthdsl"
+	"github.com/hyperledger/fabric/common/channelconfig"
+	"github.com/hyperledger/fabric/common/flogging"
+	"github.com/hyperledger/fabric/common/policies"
+	genesisconfig "github.com/hyperledger/fabric/common/tools/configtxgen/localconfig"
+	"github.com/hyperledger/fabric/msp"
+	cb "github.com/hyperledger/fabric/protos/common"
+	pb "github.com/hyperledger/fabric/protos/peer"
+	"github.com/hyperledger/fabric/protos/utils"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	pkgLogID                = "common/tools/configtxgen/encoder"
+	ordererAdminsPolicyName = "/Channel/Orderer/Admins"
+)
+
+var logger = flogging.MustGetLogger(pkgLogID)
+
+func init() {
+	flogging.SetModuleLevel(pkgLogID, "info")
+}
+
+const (
+	// ConsensusTypeSolo identifies the solo consensus implementation.
+	ConsensusTypeSolo = "solo"
+	// ConsensusTypeKafka identifies the Kafka-based consensus implementation.
+	ConsensusTypeKafka = "kafka"
+
+	// BlockValidationPolicyKey TODO
+	BlockValidationPolicyKey = "BlockValidation"
+
+	// OrdererAdminsPolicy is the absolute path to the orderer admins policy
+	OrdererAdminsPolicy = "/Channel/Orderer/Admins"
+)
+
+func addValue(cg *cb.ConfigGroup, value channelconfig.ConfigValue, modPolicy string) {
+	cg.Values[value.Key()] = &cb.ConfigValue{
+		Value:     utils.MarshalOrPanic(value.Value()),
+		ModPolicy: modPolicy,
+	}
+}
+
+func addPolicy(cg *cb.ConfigGroup, policy policies.ConfigPolicy, modPolicy string) {
+	cg.Policies[policy.Key()] = &cb.ConfigPolicy{
+		Policy:    policy.Value(),
+		ModPolicy: modPolicy,
+	}
+}
+
+// addImplicitMetaPolicyDefaults adds the Readers/Writers/Admins policies, with Any/Any/Majority rules respectively.
+func addImplicitMetaPolicyDefaults(cg *cb.ConfigGroup) {
+	addPolicy(cg, policies.ImplicitMetaMajorityPolicy(channelconfig.AdminsPolicyKey), channelconfig.AdminsPolicyKey)
+	addPolicy(cg, policies.ImplicitMetaAnyPolicy(channelconfig.ReadersPolicyKey), channelconfig.AdminsPolicyKey)
+	addPolicy(cg, policies.ImplicitMetaAnyPolicy(channelconfig.WritersPolicyKey), channelconfig.AdminsPolicyKey)
+}
+
+// addSignaturePolicyDefaults adds the Readers/Writers/Admins policies as signature policies requiring one signature from the given mspID.
+// If devMode is set to true, the Admins policy will accept arbitrary user certs for admin functions, otherwise it requires the cert satisfies
+// the admin role principal.
+func addSignaturePolicyDefaults(cg *cb.ConfigGroup, mspID string, devMode bool) {
+	if devMode {
+		addPolicy(cg, policies.SignaturePolicy(channelconfig.AdminsPolicyKey, cauthdsl.SignedByMspMember(mspID)), channelconfig.AdminsPolicyKey)
+	} else {
+		addPolicy(cg, policies.SignaturePolicy(channelconfig.AdminsPolicyKey, cauthdsl.SignedByMspAdmin(mspID)), channelconfig.AdminsPolicyKey)
+	}
+	addPolicy(cg, policies.SignaturePolicy(channelconfig.ReadersPolicyKey, cauthdsl.SignedByMspMember(mspID)), channelconfig.AdminsPolicyKey)
+	addPolicy(cg, policies.SignaturePolicy(channelconfig.WritersPolicyKey, cauthdsl.SignedByMspMember(mspID)), channelconfig.AdminsPolicyKey)
+}
+
+// NewChannelGroup defines the root of the channel configuration.  It defines basic operating principles like the hashing
+// algorithm used for the blocks, as well as the location of the ordering service.  It will recursively call into the
+// NewOrdererGroup, NewConsortiumsGroup, and NewApplicationGroup depending on whether these sub-elements are set in the
+// configuration.  All mod_policy values are set to "Admins" for this group, with the exception of the OrdererAddresses
+// value which is set to "/Channel/Orderer/Admins".
+func NewChannelGroup(conf *genesisconfig.Profile) (*cb.ConfigGroup, error) {
+	if conf.Orderer == nil {
+		return nil, errors.New("missing orderer config section")
+	}
+
+	channelGroup := cb.NewConfigGroup()
+	addImplicitMetaPolicyDefaults(channelGroup)
+	addValue(channelGroup, channelconfig.HashingAlgorithmValue(), channelconfig.AdminsPolicyKey)
+	addValue(channelGroup, channelconfig.BlockDataHashingStructureValue(), channelconfig.AdminsPolicyKey)
+	addValue(channelGroup, channelconfig.OrdererAddressesValue(conf.Orderer.Addresses), ordererAdminsPolicyName)
+
+	if conf.Consortium != "" {
+		addValue(channelGroup, channelconfig.ConsortiumValue(conf.Consortium), channelconfig.AdminsPolicyKey)
+	}
+
+	if len(conf.Capabilities) > 0 {
+		addValue(channelGroup, channelconfig.CapabilitiesValue(conf.Capabilities), channelconfig.AdminsPolicyKey)
+	}
+
+	var err error
+	channelGroup.Groups[channelconfig.OrdererGroupKey], err = NewOrdererGroup(conf.Orderer)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create orderer group")
+	}
+
+	if conf.Application != nil {
+		channelGroup.Groups[channelconfig.ApplicationGroupKey], err = NewApplicationGroup(conf.Application)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create application group")
+		}
+	}
+
+	if conf.Consortiums != nil {
+		channelGroup.Groups[channelconfig.ConsortiumsGroupKey], err = NewConsortiumsGroup(conf.Consortiums)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create consortiums group")
+		}
+	}
+
+	channelGroup.ModPolicy = channelconfig.AdminsPolicyKey
+	return channelGroup, nil
+}
+
+// NewOrdererGroup returns the orderer component of the channel configuration.  It defines parameters of the ordering service
+// about how large blocks should be, how frequently they should be emitted, etc. as well as the organizations of the ordering network.
+// It sets the mod_policy of all elements to "Admins".  This group is always present in any channel configuration.
+func NewOrdererGroup(conf *genesisconfig.Orderer) (*cb.ConfigGroup, error) {
+	ordererGroup := cb.NewConfigGroup()
+	addImplicitMetaPolicyDefaults(ordererGroup)
+	ordererGroup.Policies[BlockValidationPolicyKey] = &cb.ConfigPolicy{
+		Policy:    policies.ImplicitMetaAnyPolicy(channelconfig.WritersPolicyKey).Value(),
+		ModPolicy: channelconfig.AdminsPolicyKey,
+	}
+	addValue(ordererGroup, channelconfig.ConsensusTypeValue(conf.OrdererType), channelconfig.AdminsPolicyKey)
+	addValue(ordererGroup, channelconfig.BatchSizeValue(
+		conf.BatchSize.MaxMessageCount,
+		conf.BatchSize.AbsoluteMaxBytes,
+		conf.BatchSize.PreferredMaxBytes,
+	), channelconfig.AdminsPolicyKey)
+	addValue(ordererGroup, channelconfig.BatchTimeoutValue(conf.BatchTimeout.String()), channelconfig.AdminsPolicyKey)
+	addValue(ordererGroup, channelconfig.ChannelRestrictionsValue(conf.MaxChannels), channelconfig.AdminsPolicyKey)
+
+	if len(conf.Capabilities) > 0 {
+		addValue(ordererGroup, channelconfig.CapabilitiesValue(conf.Capabilities), channelconfig.AdminsPolicyKey)
+	}
+
+	switch conf.OrdererType {
+	case ConsensusTypeSolo:
+	case ConsensusTypeKafka:
+		addValue(ordererGroup, channelconfig.KafkaBrokersValue(conf.Kafka.Brokers), channelconfig.AdminsPolicyKey)
+	default:
+		return nil, errors.Errorf("unknown orderer type: %s", conf.OrdererType)
+	}
+
+	for _, org := range conf.Organizations {
+		var err error
+		ordererGroup.Groups[org.Name], err = NewOrdererOrgGroup(org)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create orderer org")
+		}
+	}
+
+	ordererGroup.ModPolicy = channelconfig.AdminsPolicyKey
+	return ordererGroup, nil
+}
+
+// NewOrdererOrgGroup returns an orderer org component of the channel configuration.  It defines the crypto material for the
+// organization (its MSP).  It sets the mod_policy of all elements to "Admins".
+func NewOrdererOrgGroup(conf *genesisconfig.Organization) (*cb.ConfigGroup, error) {
+	mspConfig, err := msp.GetVerifyingMspConfig(conf.MSPDir, conf.ID)
+	if err != nil {
+		return nil, errors.Wrapf(err, "1 - Error loading MSP configuration for org %s: %s", conf.Name)
+	}
+
+	ordererOrgGroup := cb.NewConfigGroup()
+	addSignaturePolicyDefaults(ordererOrgGroup, conf.ID, conf.AdminPrincipal != genesisconfig.AdminRoleAdminPrincipal)
+	addValue(ordererOrgGroup, channelconfig.MSPValue(mspConfig), channelconfig.AdminsPolicyKey)
+
+	ordererOrgGroup.ModPolicy = channelconfig.AdminsPolicyKey
+	return ordererOrgGroup, nil
+}
+
+// NewApplicationGroup returns the application component of the channel configuration.  It defines the organizations which are involved
+// in application logic like chaincodes, and how these members may interact with the orderer.  It sets the mod_policy of all elements to "Admins".
+func NewApplicationGroup(conf *genesisconfig.Application) (*cb.ConfigGroup, error) {
+	applicationGroup := cb.NewConfigGroup()
+	addImplicitMetaPolicyDefaults(applicationGroup)
+
+	if len(conf.Capabilities) > 0 {
+		addValue(applicationGroup, channelconfig.CapabilitiesValue(conf.Capabilities), channelconfig.AdminsPolicyKey)
+	}
+
+	for _, org := range conf.Organizations {
+		var err error
+		applicationGroup.Groups[org.Name], err = NewApplicationOrgGroup(org)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create application org")
+		}
+	}
+
+	applicationGroup.ModPolicy = channelconfig.AdminsPolicyKey
+	return applicationGroup, nil
+}
+
+// NewApplicationOrgGroup returns an application org component of the channel configuration.  It defines the crypto material for the organization
+// (its MSP) as well as its anchor peers for use by the gossip network.  It sets the mod_policy of all elements to "Admins".
+func NewApplicationOrgGroup(conf *genesisconfig.Organization) (*cb.ConfigGroup, error) {
+	mspConfig, err := msp.GetVerifyingMspConfig(conf.MSPDir, conf.ID)
+	if err != nil {
+		return nil, errors.Wrapf(err, "1 - Error loading MSP configuration for org %s: %s", conf.Name)
+	}
+
+	applicationOrgGroup := cb.NewConfigGroup()
+	addSignaturePolicyDefaults(applicationOrgGroup, conf.ID, conf.AdminPrincipal != genesisconfig.AdminRoleAdminPrincipal)
+	addValue(applicationOrgGroup, channelconfig.MSPValue(mspConfig), channelconfig.AdminsPolicyKey)
+
+	var anchorProtos []*pb.AnchorPeer
+	for _, anchorPeer := range conf.AnchorPeers {
+		anchorProtos = append(anchorProtos, &pb.AnchorPeer{
+			Host: anchorPeer.Host,
+			Port: int32(anchorPeer.Port),
+		})
+	}
+	addValue(applicationOrgGroup, channelconfig.AnchorPeersValue(anchorProtos), channelconfig.AdminsPolicyKey)
+
+	applicationOrgGroup.ModPolicy = channelconfig.AdminsPolicyKey
+	return applicationOrgGroup, nil
+}
+
+// NewConsortiumsGroup returns the consortiums component of the channel configuration.  This element is only defined for the ordering system channel.
+// It sets the mod_policy for all elements to "/Channel/Orderer/Admins".
+func NewConsortiumsGroup(conf map[string]*genesisconfig.Consortium) (*cb.ConfigGroup, error) {
+	consortiumsGroup := cb.NewConfigGroup()
+	// This policy is not referenced anywhere, it is only used as part of the implicit meta policy rule at the channel level, so this setting
+	// effectively degrades control of the ordering system channel to the ordering admins
+	addPolicy(consortiumsGroup, policies.SignaturePolicy(channelconfig.AdminsPolicyKey, cauthdsl.AcceptAllPolicy), ordererAdminsPolicyName)
+
+	for consortiumName, consortium := range conf {
+		var err error
+		consortiumsGroup.Groups[consortiumName], err = NewConsortiumGroup(consortium)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create consortium %s", consortiumName)
+		}
+	}
+
+	consortiumsGroup.ModPolicy = ordererAdminsPolicyName
+	return consortiumsGroup, nil
+}
+
+// NewConsortiums returns a consortiums component of the channel configuration.  Each consortium defines the organizations which may be involved in channel
+// creation, as well as the channel creation policy the orderer checks at channel creation time to authorize the action.  It sets the mod_policy of all
+// elements to "/Channel/Orderer/Admins".
+func NewConsortiumGroup(conf *genesisconfig.Consortium) (*cb.ConfigGroup, error) {
+	consortiumGroup := cb.NewConfigGroup()
+
+	for _, org := range conf.Organizations {
+		var err error
+		// Note, NewOrdererOrgGroup is correct here, as the structure is identical
+		consortiumGroup.Groups[org.Name], err = NewOrdererOrgGroup(org)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create consortium org")
+		}
+	}
+
+	addValue(consortiumGroup, channelconfig.ChannelCreationPolicyValue(policies.ImplicitMetaAnyPolicy(channelconfig.AdminsPolicyKey).Value()), ordererAdminsPolicyName)
+
+	consortiumGroup.ModPolicy = ordererAdminsPolicyName
+	return consortiumGroup, nil
+}

common/tools/configtxgen/encoder/encoder_test.go

@@ -0,0 +1,79 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package encoder
+
+import (
+	"testing"
+
+	"github.com/hyperledger/fabric/common/channelconfig"
+	"github.com/hyperledger/fabric/common/flogging"
+	genesisconfig "github.com/hyperledger/fabric/common/tools/configtxgen/localconfig"
+	"github.com/hyperledger/fabric/common/tools/configtxgen/provisional"
+	cb "github.com/hyperledger/fabric/protos/common"
+	"github.com/hyperledger/fabric/protos/utils"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+	flogging.SetModuleLevel(pkgLogID, "DEBUG")
+}
+
+func hasModPolicySet(t *testing.T, groupName string, cg *cb.ConfigGroup) {
+	assert.NotEmpty(t, cg.ModPolicy, "group %s has empty mod_policy", groupName)
+
+	for valueName, value := range cg.Values {
+		assert.NotEmpty(t, value.ModPolicy, "group %s has value %s with empty mod_policy", groupName, valueName)
+	}
+
+	for policyName, policy := range cg.Policies {
+		assert.NotEmpty(t, policy.ModPolicy, "group %s has policy %s with empty mod_policy", groupName, policyName)
+	}
+
+	for groupName, group := range cg.Groups {
+		hasModPolicySet(t, groupName, group)
+	}
+}
+
+func TestConfigParsing(t *testing.T) {
+	config := genesisconfig.Load(genesisconfig.SampleDevModeSoloProfile)
+	group, err := NewChannelGroup(config)
+	assert.NoError(t, err)
+	assert.NotNil(t, group)
+
+	_, err = channelconfig.NewBundle("test", &cb.Config{
+		ChannelGroup: group,
+	})
+	assert.NoError(t, err)
+
+	hasModPolicySet(t, "Channel", group)
+}
+
+// This test will be removed with the legacy provisional package, but demonstrates
+// that the old and new encoders produce identical output
+func TestEquivalentParsing(t *testing.T) {
+	for _, profile := range []string{
+		genesisconfig.SampleInsecureSoloProfile,
+		genesisconfig.SampleSingleMSPSoloProfile,
+		genesisconfig.SampleDevModeSoloProfile,
+		genesisconfig.SampleInsecureKafkaProfile,
+		genesisconfig.SampleSingleMSPKafkaProfile,
+		genesisconfig.SampleDevModeKafkaProfile,
+	} {
+		config := genesisconfig.Load(profile)
+		group, err := NewChannelGroup(config)
+		assert.NoError(t, err)
+		assert.NotNil(t, group)
+
+		gb := provisional.New(config).GenesisBlockForChannel("foo")
+		env := utils.ExtractEnvelopeOrPanic(gb, 0)
+		configEnv := &cb.ConfigEnvelope{}
+		utils.UnmarshalEnvelopeOfType(env, cb.HeaderType_CONFIG, configEnv)
+		assert.True(t, proto.Equal(configEnv.Config.ChannelGroup, group))
+	}
+}

sampleconfig/configtx.yaml

@@ -39,13 +39,20 @@ Profiles:
 
     # SampleDevModeSolo defines a configuration which uses the Solo orderer,
     # contains the sample MSP as both orderer and consortium member, and
-    # requires only basic membership for admin privileges.
+    # requires only basic membership for admin privileges. It also defines
+    # an Application on the ordering system channel, which should usually
+    # be avoided.
     SampleDevModeSolo:
         Orderer:
             <<: *OrdererDefaults
             Organizations:
                 - <<: *SampleOrg
                   AdminPrincipal: Role.MEMBER
+        Application:
+            <<: *ApplicationDefaults
+            Organizations:
+                - <<: *SampleOrg
+                  AdminPrincipal: Role.MEMBER
         Consortiums:
             SampleConsortium:
                 Organizations: