[FAB-7715] Handle invalid couchdb index defs

Invalid CouchDB index definitions may be packaged into chaincode
installation, causing runtime problems when the chaincode is
installed/instantiated.

This change
- Adds metadata validation for golang and node.js chaincode packaging
- Adds index JSON validation for files in META-INF/statedb/couchdb/indexes.
- Ensures no metadata files are packaged from other META-INF directories.

Change-Id: I6dc5f7cbb0ae48edfc29824813bf9f5ccaf17b1c
Signed-off-by: David Enyeart <enyeart@us.ibm.com>

Author: denyeart

core/chaincode/platforms/golang/platform.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/hyperledger/fabric/common/metadata"
 	"github.com/hyperledger/fabric/core/chaincode/platforms/util"
+	ccmetadata "github.com/hyperledger/fabric/core/common/ccprovider/metadata"
 	cutil "github.com/hyperledger/fabric/core/container/util"
 	pb "github.com/hyperledger/fabric/protos/peer"
 	"github.com/spf13/viper"
@@ -428,8 +429,8 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 	for _, file := range files {
 
 		// If the file is metadata rather than golang code, remove the leading go code path, for example:
-		// file.Name:  src/github.com/hyperledger/fabric/examples/chaincode/go/marbles02/META-INF/statedb/couchdb/indexes/indexOwner.json
-		// tar file:   META-INF/statedb/couchdb/indexes/indexOwner.json
+		// original file.Name:  src/github.com/hyperledger/fabric/examples/chaincode/go/marbles02/META-INF/statedb/couchdb/indexes/indexOwner.json
+		// updated file.Name:   META-INF/statedb/couchdb/indexes/indexOwner.json
 		if file.IsMetadata {
 
 			// Ensure META-INF directory can be found, then grab the META-INF relative path to use for packaging
@@ -440,9 +441,25 @@ func (goPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte
 			if err != nil {
 				return nil, fmt.Errorf("Could not get relative path for META-INF directory %s. Error:%s", file.Name, err)
 			}
+
+			// Split the filename itself from its path
+			_, filename := filepath.Split(file.Name)
+
+			// Hidden files are not supported as metadata, therefore ignore them.
+			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
+			if strings.HasPrefix(filename, ".") {
+				logger.Warningf("Ignoring hidden file in metadata directory: %s", file.Name)
+				continue
+			}
+
+			// Validate metadata file for inclusion in tar
+			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
+			err = ccmetadata.ValidateMetadataFile(file.Path, filepath.Dir(file.Name))
+			if err != nil {
+				return nil, err
+			}
 		}
 
-		logger.Debug("Writing file to chaincode code package tarball:", file.Name)
 		err = cutil.WriteFileToPackage(file.Path, file.Name, tw)
 		if err != nil {
 			return nil, fmt.Errorf("Error writing %s to tar: %s", file.Name, err)

core/chaincode/platforms/golang/platform_test.go

@@ -271,6 +271,9 @@ func TestGetDeploymentPayload(t *testing.T) {
 		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/examples/chaincode/go/map"}}, succ: true},
 		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/examples/bad/go/map"}}, succ: false},
 		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/test/chaincodes/BadImport"}}, succ: false},
+		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/test/chaincodes/BadMetadataInvalidIndex"}}, succ: false},
+		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/test/chaincodes/BadMetadataUnexpectedFolderContent"}}, succ: false},
+		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/test/chaincodes/BadMetadataIgnoreHiddenFile"}}, succ: true},
 		{spec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "Test Chaincode", Path: "github.com/hyperledger/fabric/core/chaincode/platforms/golang/" + emptyDir}}, succ: false},
 	}
 

core/common/ccprovider/metadata/validators.go

@@ -0,0 +1,89 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package metadata
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/hyperledger/fabric/common/flogging"
+)
+
+var logger = flogging.MustGetLogger("metadata")
+
+// fileValidators are used as handlers to validate specific metadata directories
+type fileValidator func(srcPath string) error
+
+// Currently, the only metadata expected and allowed is for META-INF/statedb/couchdb/indexes.
+var fileValidators = map[string]fileValidator{
+	"META-INF/statedb/couchdb/indexes": couchdbIndexFileValidator,
+}
+
+// UnhandledDirectoryError is returned for metadata files in unhandled directories
+type UnhandledDirectoryError struct {
+	err string
+}
+
+func (e *UnhandledDirectoryError) Error() string {
+	return e.err
+}
+
+// InvalidFileError is returned for invalid metadata files
+type InvalidFileError struct {
+	err string
+}
+
+func (e *InvalidFileError) Error() string {
+	return e.err
+}
+
+// ValidateMetadataFile checks that metadata files are valid
+// according to the validation rules of the metadata directory (metadataType)
+func ValidateMetadataFile(srcPath, metadataType string) error {
+	// Get the validator handler for the metadata directory
+	fileValidator, ok := fileValidators[metadataType]
+
+	// If there is no validator handler for metadata directory, return UnhandledDirectoryError
+	if !ok {
+		return &UnhandledDirectoryError{fmt.Sprintf("Metadata not supported in directory: %s", metadataType)}
+	}
+
+	// If the file is not valid for the given metadata directory, return InvalidFileError
+	err := fileValidator(srcPath)
+	if err != nil {
+		return &InvalidFileError{fmt.Sprintf("Metadata file [%s] failed validation: %s", srcPath, err)}
+	}
+
+	// file is valid, return nil error
+	return nil
+}
+
+// couchdbIndexFileValidator implements fileValidator
+func couchdbIndexFileValidator(srcPath string) error {
+	fileBytes, err := ioutil.ReadFile(srcPath)
+	if err != nil {
+		return err
+	}
+
+	// if the content does not validate as JSON, return err to invalidate the file
+	if !isJSON(string(fileBytes)) {
+		return errors.New("File is not valid JSON")
+	}
+
+	// TODO Additional validation to ensure the JSON represents a valid couchdb index definition
+
+	// file is a valid couchdb index definition, return nil error
+	return nil
+}
+
+// isJSON tests a string to determine if it can be parsed as valid JSON
+func isJSON(s string) bool {
+	var js map[string]interface{}
+	return json.Unmarshal([]byte(s), &js) == nil
+}

core/common/ccprovider/metadata/validators_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package metadata
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var packageTestDir = filepath.Join(os.TempDir(), "ccmetadata-validator-test")
+
+func TestGoodIndexJSON(t *testing.T) {
+	testDir := filepath.Join(packageTestDir, "GoodIndexJSON")
+	cleanupDir(testDir)
+	defer cleanupDir(testDir)
+
+	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
+	filebytes := []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
+
+	err := writeToFile(filename, filebytes)
+	assert.NoError(t, err, "Error writing to file")
+
+	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	assert.NoError(t, err, "Error validating a good index")
+}
+
+func TestBadIndexJSON(t *testing.T) {
+	testDir := filepath.Join(packageTestDir, "BadIndexJSON")
+	cleanupDir(testDir)
+	defer cleanupDir(testDir)
+
+	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
+	filebytes := []byte("invalid json")
+
+	err := writeToFile(filename, filebytes)
+	assert.NoError(t, err, "Error writing to file")
+
+	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+
+	assert.Error(t, err, "Should have received an InvalidFileError")
+
+	// Type assertion on InvalidFileError
+	_, ok := err.(*InvalidFileError)
+	assert.True(t, ok, "Should have received an InvalidFileError")
+
+	t.Log("SAMPLE ERROR STRING:", err.Error())
+}
+
+func TestIndexWrongLocation(t *testing.T) {
+	testDir := filepath.Join(packageTestDir, "IndexWrongLocation")
+	cleanupDir(testDir)
+	defer cleanupDir(testDir)
+
+	// place the index one directory too high
+	filename := filepath.Join(testDir, "META-INF/statedb/couchdb", "myIndex.json")
+	filebytes := []byte("invalid json")
+
+	err := writeToFile(filename, filebytes)
+	assert.NoError(t, err, "Error writing to file")
+
+	err = ValidateMetadataFile(filename, "META-INF/statedb/couchdb")
+	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
+
+	// Type assertion on UnhandledDirectoryError
+	_, ok := err.(*UnhandledDirectoryError)
+	assert.True(t, ok, "Should have received an UnhandledDirectoryError")
+
+	t.Log("SAMPLE ERROR STRING:", err.Error())
+}
+
+func TestInvalidMetadataType(t *testing.T) {
+	testDir := filepath.Join(packageTestDir, "InvalidMetadataType")
+	cleanupDir(testDir)
+	defer cleanupDir(testDir)
+
+	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
+	filebytes := []byte("invalid json")
+
+	err := writeToFile(filename, filebytes)
+	assert.NoError(t, err, "Error writing to file")
+
+	err = ValidateMetadataFile(filename, "Invalid metadata type")
+	assert.Error(t, err, "Should have received an UnhandledDirectoryError")
+
+	// Type assertion on UnhandledDirectoryError
+	_, ok := err.(*UnhandledDirectoryError)
+	assert.True(t, ok, "Should have received an UnhandledDirectoryError")
+}
+
+func TestCantReadFile(t *testing.T) {
+	testDir := filepath.Join(packageTestDir, "CantReadFile")
+	cleanupDir(testDir)
+	defer cleanupDir(testDir)
+
+	filename := filepath.Join(testDir, "META-INF/statedb/couchdb/indexes", "myIndex.json")
+
+	// Don't write the file - test for can't read file
+	// err := writeToFile(filename, filebytes)
+	// assert.NoError(t, err, "Error writing to file")
+
+	err := ValidateMetadataFile(filename, "META-INF/statedb/couchdb/indexes")
+	assert.Error(t, err, "Should have received error reading file")
+}
+
+func cleanupDir(dir string) error {
+	// clean up any previous files
+	err := os.RemoveAll(dir)
+	if err != nil {
+		return nil
+	}
+	return os.Mkdir(dir, os.ModePerm)
+}
+
+func writeToFile(filename string, bytes []byte) error {
+	dir := filepath.Dir(filename)
+	err := os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filename, bytes, 0644)
+}

core/container/util/writer.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/hyperledger/fabric/common/flogging"
+	ccmetadata "github.com/hyperledger/fabric/core/common/ccprovider/metadata"
 	"github.com/pkg/errors"
 )
 
@@ -85,11 +86,30 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 		}
 
 		var newPath string
+
 		// if file is metadata, keep the /META-INF directory, e.g: META-INF/statedb/couchdb/indexes/indexOwner.json
 		// otherwise file is source code, put it in /src dir, e.g: src/marbles_chaincode.js
 		if strings.HasPrefix(path, filepath.Join(rootDirectory, "META-INF")) {
 			newPath = path[rootDirLen+1:]
-		} else {
+
+			// Split the filename itself from its path
+			_, filename := filepath.Split(newPath)
+
+			// Hidden files are not supported as metadata, therefore ignore them.
+			// User often doesn't know that hidden files are there, and may not be able to delete them, therefore warn user rather than error out.
+			if strings.HasPrefix(filename, ".") {
+				vmLogger.Warningf("Ignoring hidden file in metadata directory: %s", newPath)
+				return nil
+			}
+
+			// Validate metadata file for inclusion in tar
+			// Validation is based on the passed metadata directory, e.g. META-INF/statedb/couchdb/indexes
+			err = ccmetadata.ValidateMetadataFile(path, filepath.Dir(newPath))
+			if err != nil {
+				return err
+			}
+
+		} else { // file is not metadata, include in src
 			newPath = fmt.Sprintf("src%s", path[rootDirLen:])
 		}
 
@@ -99,8 +119,6 @@ func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string,
 		}
 		fileCount++
 
-		vmLogger.Debugf("Writing file %s to tar", newPath)
-
 		return nil
 	}