Fix dredd test

.circleci/config.yml

@@ -369,31 +369,33 @@ jobs:
       - store_artifacts:
           path: /go/src/github.com/tendermint/tendermint/tendermint-*.tar.gz
 
-  # # Test RPC implementation against the swagger documented specs
-  # contract_tests:
-  #   working_directory: /home/circleci/.go_workspace/src/github.com/tendermint/tendermint
-  #   machine:
-  #     image: circleci/classic:latest
-  #   environment:
-  #     GOBIN: /home/circleci/.go_workspace/bin
-  #     GOPATH: /home/circleci/.go_workspace/
-  #     GOOS: linux
-  #     GOARCH: amd64
-  #   parallelism: 1
-  #   steps:
-  #     - checkout_with_submodules
-  #     - run:
-  #         name: Test RPC endpoints against swagger documentation
-  #         command: |
-  #           set -x
-  #           export PATH=~/.local/bin:$PATH
-  #           # install node and dredd
-  #           ./scripts/get_nodejs.sh
-  #           # build the binaries with a proper version of Go
-  #           docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang make build-linux build-contract-tests-hooks
-  #           # This docker image works with go 1.7, we can install here the hook handler that contract-tests is going to use
-  #           go get github.com/snikch/goodman/cmd/goodman
-  #           make contract-tests
+  contract_tests:
+    working_directory: /home/circleci/.go_workspace/src/github.com/tendermint/tendermint
+    machine:
+      image: circleci/classic:latest
+    environment:
+      GOBIN: /home/circleci/.go_workspace/bin
+      GOPATH: /home/circleci/.go_workspace/
+      GOOS: linux
+      GOARCH: amd64
+    parallelism: 1
+    steps:
+      - checkout_with_submodules
+      - run:
+          name: Test RPC endpoints against swagger documentation
+          command: |
+            set -x
+            export PATH=~/.local/bin:$PATH
+            # install node and dredd
+            ./scripts/get_nodejs.sh
+            # build the binaries with a proper version of Go
+            # Build Tendermint
+            docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang:1.14.1-alpine /bin/sh -c "apk add --update make git && make build-linux"
+            # Build contract-tests
+            docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint ubuntu:20.10 ./scripts/prepare_dredd_test.sh
+            # This docker image works with go 1.7, we can install here the hook handler that contract-tests is going to use
+            go get github.com/snikch/goodman/cmd/goodman
+            make contract-tests
 
 workflows:
   version: 2
@@ -435,7 +437,6 @@ workflows:
       - localnet:
           requires:
             - setup_dependencies
-#      - test_p2p
       - test_p2p:
             name: test_p2p_ipv6
             ipv: 6
@@ -445,9 +446,9 @@ workflows:
               only:
                 - master
                 - /v[0-9]+\.[0-9]+/
-      # - contract_tests:
-      #     requires:
-      #       - setup_dependencies
+      - contract_tests:
+          requires:
+            - setup_dependencies
 
 #  release:
 #    jobs:

CHANGELOG_PENDING.md

@@ -17,3 +17,4 @@
 ### IMPROVEMENTS:
 
 ### BUG FIXES:
+- [circleCI] [\#76](https://github.com/line/tendermint/pull/76) Fix contract test job of circleCI

cmd/contract_tests/main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/tendermint/tendermint/cmd/contract_tests/unmarshaler"
 	"strings"
 
 	"github.com/snikch/goodman/hooks"
@@ -16,19 +18,60 @@ func main() {
 		fmt.Println(t[0].Name)
 	})
 	h.BeforeEach(func(t *transaction.Transaction) {
-		if strings.HasPrefix(t.Name, "Tx") ||
+		if t.Expected.StatusCode != "200" {
+			t.Skip = true
+		} else if strings.HasPrefix(t.Name, "Tx") ||
 			// We need a proper example of evidence to broadcast
-			strings.HasPrefix(t.Name, "Info > /broadcast_evidence") ||
+			strings.HasPrefix(t.Name, "/broadcast_evidence >") ||
 			// We need a proper example of path and data
-			strings.HasPrefix(t.Name, "ABCI > /abci_query") ||
+			strings.HasPrefix(t.Name, "/abci_query >") ||
 			// We need to find a way to make a transaction before starting the tests,
 			// that hash should replace the dummy one in hte swagger file
-			strings.HasPrefix(t.Name, "Info > /tx") {
+			strings.HasPrefix(t.Name, "/tx >") {
 			t.Skip = true
-			fmt.Printf("%s Has been skipped\n", t.Name)
 		}
 	})
+
+	// TODO This check need to remove if dredd is updated to check optional
+	// dredd can not validate optional items
+	h.Before("/genesis > Get Genesis > 200 > application/json", func(t *transaction.Transaction) {
+		removeOptionalFieldsOfExpected(t, []string{"result.genesis.app_state"})
+	})
+	h.Before("/broadcast_tx_async > Returns right away, with no response. "+
+		"Does not wait for CheckTx nor DeliverTx results. > 200 > application/json", func(t *transaction.Transaction) {
+		removeOptionalFieldsOfExpected(t, []string{"error"})
+	})
+	h.Before("/broadcast_tx_sync > Returns with the response from CheckTx. "+
+		"Does not wait for DeliverTx result. > 200 > application/json", func(t *transaction.Transaction) {
+		removeOptionalFieldsOfExpected(t, []string{"error"})
+	})
+	h.Before("/broadcast_tx_commit > Returns with the responses from CheckTx and DeliverTx. "+
+		"> 200 > application/json", func(t *transaction.Transaction) {
+		removeOptionalFieldsOfExpected(t, []string{"error"})
+	})
+	h.Before("/block_results > Get block results at a specified height > 200 > application/json",
+		func(t *transaction.Transaction) {
+			removeOptionalFieldsOfExpected(t, []string{
+				"result.txs_results",
+				"result.begin_block_events",
+				"result.end_block",
+				"result.end_block_events",
+				"result.validator_updates",
+				"result.consensus_param_updates"})
+		})
+
 	server.Serve()
 	defer server.Listener.Close()
-	fmt.Print("FINE")
+}
+
+func removeOptionalFieldsOfExpected(t *transaction.Transaction, paths []string) {
+	expected := unmarshaler.UnmarshalJSON(&t.Expected.Body)
+	for _, path := range paths {
+		expected.DeleteProperty(strings.Split(path, ".")...)
+	}
+	newBody, err := json.Marshal(expected.Body)
+	if err != nil {
+		panic(fmt.Sprintf("fail to marshal expected body with %s", err))
+	}
+	t.Expected.Body = string(newBody)
 }

cmd/contract_tests/unmarshaler/unmarshal.go

@@ -0,0 +1,57 @@
+package unmarshaler
+
+import (
+	"encoding/json"
+	"gopkg.in/yaml.v3"
+)
+
+type UnmarshalledArbitraryObject struct {
+	Body interface{}
+}
+
+func (obj *UnmarshalledArbitraryObject) GetProperty(keys ...string) interface{} {
+	body := obj.Body
+	for _, key := range keys {
+		body = body.(map[string]interface{})[key]
+	}
+	return body
+}
+
+func (obj *UnmarshalledArbitraryObject) SetProperty(keys []string, value interface{}) {
+	prevKeys := keys[:len(keys)-1]
+	lastKey := keys[len(keys)-1]
+
+	body := obj.Body
+	for _, key := range prevKeys {
+		body = body.(map[string]interface{})[key]
+	}
+	body.(map[string]interface{})[lastKey] = value
+}
+
+func (obj *UnmarshalledArbitraryObject) DeleteProperty(keys ...string) {
+	prevKeys := keys[:len(keys)-1]
+	lastKey := keys[len(keys)-1]
+
+	body := obj.Body
+	for _, key := range prevKeys {
+		body = body.(map[string]interface{})[key]
+	}
+	delete(body.(map[string]interface{}), lastKey)
+}
+
+func UnmarshalJSON(str *string) UnmarshalledArbitraryObject {
+	return UnmarshalledArbitraryObject{unmarshalArbitraryFormat(json.Unmarshal, str)}
+}
+
+func UnmarshalYAML(str *string) UnmarshalledArbitraryObject {
+	return UnmarshalledArbitraryObject{unmarshalArbitraryFormat(yaml.Unmarshal, str)}
+}
+
+func unmarshalArbitraryFormat(unmarshal func([]byte, interface{}) error, str *string) interface{} {
+	var body interface{}
+	err := unmarshal([]byte(*str), &body)
+	if err != nil {
+		panic(err)
+	}
+	return body
+}

cmd/contract_tests/unmarshaler/unmarshal_test.go

@@ -0,0 +1,172 @@
+package unmarshaler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalElementJSON(t *testing.T) {
+	var unmarshalJSONTests = []struct {
+		json     string
+		expected interface{}
+	}{
+		{
+			`"30"`,
+			"30",
+		},
+		{
+			"30",
+			float64(30),
+		},
+		{
+			"null",
+			nil,
+		},
+		{
+			"[]",
+			[]interface{}{},
+		},
+		{
+			`["a", "b", "c"]`,
+			[]interface{}{"a", "b", "c"},
+		},
+		{
+			"{}",
+			map[string]interface{}{},
+		},
+		{
+			`{"key1":"value1", "key2":"value2"}`,
+			map[string]interface{}{"key1": "value1", "key2": "value2"},
+		},
+		{
+			testJSON1,
+			map[string]interface{}{
+				"key1": float64(119),
+				"sub1": map[string]interface{}{"key2": "value2", "sub2": map[string]interface{}{"key3": "value3"}},
+				"sub3": map[string]interface{}{"key4": "value4", "key5": "value5"}},
+		},
+		{
+			testJSON2,
+			map[string]interface{}{
+				"key1": float64(119),
+				"sub1": []interface{}{map[string]interface{}{"key2": "value2", "sub2": map[string]interface{}{"key3": []interface{}{"value2"}}}},
+				"sub3": map[string]interface{}{"key4": "value2", "key5": "value2"}},
+		},
+	}
+
+	for _, tt := range unmarshalJSONTests {
+		t.Logf("unmarshal json test %s", tt.json)
+		{
+			unmarshaledJSON := UnmarshalJSON(&tt.json)
+			require.Equal(t, tt.expected, unmarshaledJSON.Body)
+		}
+	}
+}
+
+func TestUnmarshalElementYAML(t *testing.T) {
+	var unmarshalJSONTests = []struct {
+		yaml     string
+		expected interface{}
+	}{
+		{
+			`"30"`,
+			"30",
+		},
+		{
+			"30",
+			30,
+		},
+		{
+			"null",
+			nil,
+		},
+		{
+			"[]",
+			[]interface{}{},
+		},
+		{
+			`["a", "b", "c"]`,
+			[]interface{}{"a", "b", "c"},
+		},
+		{
+			"{}",
+			map[string]interface{}{},
+		},
+		{
+			"key1: value1\nkey2: value2",
+			map[string]interface{}{"key1": "value1", "key2": "value2"},
+		},
+		{
+			testYAML1,
+			map[string]interface{}{
+				"key1": 119,
+				"sub1": map[string]interface{}{"key2": "value2", "sub2": map[string]interface{}{"key3": "value3"}},
+				"sub3": map[string]interface{}{"key4": "value4", "key5": "value5"}},
+		},
+		{
+			testYAML2,
+			map[string]interface{}{
+				"key1": 119,
+				"sub1": []interface{}{map[string]interface{}{"key2": "value2", "sub2": map[string]interface{}{"key3": []interface{}{"value3"}}}},
+				"sub3": map[string]interface{}{"key4": "value4", "key5": "value5"}},
+		},
+	}
+
+	for _, tt := range unmarshalJSONTests {
+		t.Logf("unmarshal yaml test %s", tt.yaml)
+		{
+			unmarshaledYAML := UnmarshalYAML(&tt.yaml)
+			require.Equal(t, tt.expected, unmarshaledYAML.Body)
+		}
+	}
+}
+
+func TestGetAndSetProperty(t *testing.T) {
+	testJSON := testJSON1
+	unmarshaledJSON := UnmarshalJSON(&testJSON)
+	require.Equal(t, float64(119), unmarshaledJSON.GetProperty("key1"))
+	require.Equal(t, "value2", unmarshaledJSON.GetProperty("sub1", "key2"))
+	require.Equal(t, "value3", unmarshaledJSON.GetProperty("sub1", "sub2", "key3"))
+	require.Equal(t, "value4", unmarshaledJSON.GetProperty("sub3", "key4"))
+	require.Equal(t, "value5", unmarshaledJSON.GetProperty("sub3", "key5"))
+
+	unmarshaledJSON.SetProperty([]string{"key1"}, "newValue1")
+	unmarshaledJSON.SetProperty([]string{"sub1", "sub2", "key3"}, "newValue2")
+
+	require.Equal(t, "newValue1", unmarshaledJSON.GetProperty("key1"))
+	require.Equal(t, "newValue2", unmarshaledJSON.GetProperty("sub1", "sub2", "key3"))
+}
+
+func TestDeleteProposer(t *testing.T) {
+	testJSON := testJSON1
+	unmarshaledJSON := UnmarshalJSON(&testJSON)
+
+	unmarshaledJSON.DeleteProperty("sub3", "key5")
+	require.Nil(t, unmarshaledJSON.GetProperty("sub3", "key5"))
+}
+
+const (
+	testJSON1 = `{"key1":119, "sub1":{"key2":"value2", "sub2":{"key3":"value3"}}, "sub3":{"key4":"value4", "key5":"value5"}}`
+	testJSON2 = `{"key1":119, "sub1":[{"key2":"value2", "sub2":{"key3":["value2"]}}], "sub3":{"key4":"value2", "key5":"value2"}}`
+	testYAML1 = `
+key1: 119
+sub1:
+  key2: value2
+  sub2:
+    key3: value3
+sub3:
+  key4: value4
+  key5: value5
+`
+	testYAML2 = `
+key1: 119
+sub1:
+  - key2: value2
+    sub2:
+      key3: [value3]
+sub3:
+  key4: value4
+  key5: value5
+`
+)