[FAB-7863] Add CouchDB index definition validation

CouchDB index definitions should be validated.

Invalid index definitions can cause peer panic.

This change adds validation for the index definition JSON with
error reporting for bad formatting or invalid entries.

Testing for this CR indicated an improvement was required in
testing indexes.

Change-Id: Iea87c1c263133f078d47d831cba730dff1b47e66
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>

Author: Chris Elder

core/common/ccprovider/metadata/validators.go

@@ -11,6 +11,8 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"reflect"
+	"strings"
 
 	"github.com/hyperledger/fabric/common/flogging"
 )
@@ -72,18 +74,166 @@ func couchdbIndexFileValidator(srcPath string) error {
 	}
 
 	// if the content does not validate as JSON, return err to invalidate the file
-	if !isJSON(string(fileBytes)) {
+	boolIsJSON, indexDefinition := isJSON(fileBytes)
+	if !boolIsJSON {
 		return errors.New("File is not valid JSON")
 	}
 
-	// TODO Additional validation to ensure the JSON represents a valid couchdb index definition
+	// validate the index definition
+	return validateIndexJSON(indexDefinition)
 
-	// file is a valid couchdb index definition, return nil error
-	return nil
 }
 
 // isJSON tests a string to determine if it can be parsed as valid JSON
-func isJSON(s string) bool {
+func isJSON(s []byte) (bool, map[string]interface{}) {
 	var js map[string]interface{}
-	return json.Unmarshal([]byte(s), &js) == nil
+	return json.Unmarshal([]byte(s), &js) == nil, js
+}
+
+func validateIndexJSON(indexDefinition map[string]interface{}) error {
+
+	//flag to track if the "index" key is included
+	indexIncluded := false
+
+	//iterate through the JSON index definition
+	for jsonKey, jsonValue := range indexDefinition {
+
+		//create a case for the top level entries
+		switch jsonKey {
+
+		case "index":
+
+			if reflect.TypeOf(jsonValue).Kind() != reflect.Map {
+				return fmt.Errorf("Invalid entry, \"index\" must be a JSON")
+			}
+
+			err := processIndexMap(jsonValue.(map[string]interface{}))
+			if err != nil {
+				return err
+			}
+
+			indexIncluded = true
+
+		case "ddoc":
+
+			//Verify the design doc is a string
+			if reflect.TypeOf(jsonValue).Kind() != reflect.String {
+				return fmt.Errorf("Invalid entry, \"ddoc\" must be a string")
+			}
+
+			logger.Debugf("Found index object: \"%s\":\"%s\"", jsonKey, jsonValue)
+
+		case "name":
+
+			//Verify the name is a string
+			if reflect.TypeOf(jsonValue).Kind() != reflect.String {
+				return fmt.Errorf("Invalid entry, \"name\" must be a string")
+			}
+
+			logger.Debugf("Found index object: \"%s\":\"%s\"", jsonKey, jsonValue)
+
+		case "type":
+
+			if jsonValue != "json" {
+				return fmt.Errorf("Index type must be json")
+			}
+
+			logger.Debugf("Found index object: \"%s\":\"%s\"", jsonKey, jsonValue)
+
+		default:
+
+			return fmt.Errorf("Invalid Entry.  Entry %s", jsonKey)
+
+		}
+
+	}
+
+	if !indexIncluded {
+		return fmt.Errorf("Index definition must include a \"fields\" definition")
+	}
+
+	return nil
+
+}
+
+//processIndexMap processes an interface map and wraps field names or traverses
+//the next level of the json query
+func processIndexMap(jsonFragment map[string]interface{}) error {
+
+	//iterate the item in the map
+	for jsonKey, jsonValue := range jsonFragment {
+
+		switch jsonKey {
+
+		case "fields":
+
+			switch jsonValueType := jsonValue.(type) {
+
+			case []interface{}:
+
+				//iterate the index field objects
+				for _, itemValue := range jsonValueType {
+
+					switch reflect.TypeOf(itemValue).Kind() {
+
+					case reflect.String:
+						//String is a valid field descriptor  ex: "color", "size"
+						logger.Debugf("Found index field name: \"%s\"", itemValue)
+
+					case reflect.Map:
+						//Handle the case where a sort is included  ex: {"size":"asc"}, {"color":"desc"}
+						err := validateFieldMap(itemValue.(map[string]interface{}))
+						if err != nil {
+							return err
+						}
+
+					}
+				}
+
+			default:
+				return fmt.Errorf("Expecting a JSON array of fields")
+			}
+
+		case "partial_filter_selector":
+
+			//TODO - add support for partial filter selector, for now return nil
+			//Take no other action, will be considered valid for now
+
+		default:
+
+			//if anything other than "fields" or "partial_filter_selector" was found,
+			//return an error
+			return fmt.Errorf("Invalid Entry.  Entry %s", jsonKey)
+
+		}
+
+	}
+
+	return nil
+
+}
+
+//validateFieldMap validates the list of field objects
+func validateFieldMap(jsonFragment map[string]interface{}) error {
+
+	//iterate the fields to validate the sort criteria
+	for jsonKey, jsonValue := range jsonFragment {
+
+		switch jsonValue.(type) {
+
+		case string:
+			//Ensure the sort is either "asc" or "desc"
+			if !(strings.ToLower(jsonValue.(string)) == "asc" || strings.ToLower(jsonValue.(string)) == "desc") {
+				return fmt.Errorf("Sort must be either \"asc\" or \"desc\".  \"%s\" was found.", jsonValue)
+			}
+			logger.Debugf("Found index field name: \"%s\":\"%s\"", jsonKey, jsonValue)
+
+		default:
+			return fmt.Errorf("Invalid field definition, fields must be in the form \"fieldname\":\"sort\"")
+
+		}
+	}
+
+	return nil
+
 }

core/common/ccprovider/metadata/validators_test.go

@@ -110,6 +110,131 @@ func TestCantReadFile(t *testing.T) {
 	assert.Error(t, err, "Should have received error reading file")
 }
 
+func TestIndexValidation(t *testing.T) {
+
+	// Test valid index with field sorts
+	indexDef := []byte(`{"index":{"fields":[{"size":"desc"}, {"color":"desc"}]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition := isJSON(indexDef)
+	err := validateIndexJSON(indexDefinition)
+	assert.NoError(t, err)
+
+	// Test valid index without field sorts
+	indexDef = []byte(`{"index":{"fields":["size","color"]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.NoError(t, err)
+
+	// Test valid index without design doc, name and type
+	indexDef = []byte(`{"index":{"fields":["size","color"]}}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.NoError(t, err)
+
+	// Test valid index with partial filter selector (only tests that it will not return error if included)
+	indexDef = []byte(`{
+		  "index": {
+		    "partial_filter_selector": {
+		      "status": {
+		        "$ne": "archived"
+		      }
+		    },
+		    "fields": ["type"]
+		  },
+		  "ddoc" : "type-not-archived",
+		  "type" : "json"
+		}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.NoError(t, err)
+
+}
+
+func TestIndexValidationInvalidParameters(t *testing.T) {
+
+	// Test numeric values passed in for parameters
+	indexDef := []byte(`{"index":{"fields":[{"size":"desc"}, {"color":"desc"}]},"ddoc":1, "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition := isJSON(indexDef)
+	err := validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for numeric design doc")
+
+	// Test invalid design doc parameter
+	indexDef = []byte(`{"index":{"fields":["size","color"]},"ddoc1":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid design doc parameter")
+
+	// Test invalid name parameter
+	indexDef = []byte(`{"index":{"fields":["size","color"]},"ddoc":"indexSizeSortName", "name1":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid name parameter")
+
+	// Test invalid type parameter, numeric
+	indexDef = []byte(`{"index":{"fields":["size","color"]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":1}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for numeric type parameter")
+
+	// Test invalid type parameter
+	indexDef = []byte(`{"index":{"fields":["size","color"]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"text"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid type parameter")
+
+	// Test invalid index parameter
+	indexDef = []byte(`{"index1":{"fields":["size","color"]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid index parameter")
+
+	// Test missing index parameter
+	indexDef = []byte(`{"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for missing index parameter")
+
+}
+
+func TestIndexValidationInvalidFields(t *testing.T) {
+
+	// Test invalid fields parameter
+	indexDef := []byte(`{"index":{"fields1":[{"size":"desc"}, {"color":"desc"}]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition := isJSON(indexDef)
+	err := validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid fields parameter")
+
+	// Test invalid field name (numeric)
+	indexDef = []byte(`{"index":{"fields":["size", 1]},"ddoc1":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for field name defined as numeric")
+
+	// Test invalid field sort
+	indexDef = []byte(`{"index":{"fields":[{"size":"desc1"}, {"color":"desc"}]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid field sort")
+
+	// Test numeric in sort
+	indexDef = []byte(`{"index":{"fields":[{"size":1}, {"color":"desc"}]},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for a numeric in field sort")
+
+	// Test invalid json for fields
+	indexDef = []byte(`{"index":{"fields":"size"},"ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for invalid field json")
+
+	// Test missing JSON for fields
+	indexDef = []byte(`{"index":"fields","ddoc":"indexSizeSortName", "name":"indexSizeSortName","type":"json"}`)
+	_, indexDefinition = isJSON(indexDef)
+	err = validateIndexJSON(indexDefinition)
+	assert.Error(t, err, "Error should have been thrown for missing JSON for fields")
+
+}
+
 func cleanupDir(dir string) error {
 	// clean up any previous files
 	err := os.RemoveAll(dir)

core/ledger/util/couchdb/couchdb_test.go

@@ -1022,6 +1022,9 @@ func TestIndexOperations(t *testing.T) {
 	err = db.CreateIndex(indexDefSize)
 	testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))
 
+	//Delay for 100ms since CouchDB index list is updated async after index create/drop
+	time.Sleep(100 * time.Millisecond)
+
 	//Execute a query with an index,  this should succeed
 	_, err = db.QueryDocuments(queryString)
 	testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while querying with an index"))