Add namedBuilders for special input encoding of types (istio#6044)

* add istio Value message to test message * Update go:generate command line * Add namedTypes and builders * remove e2e mixer from master tests
249043822 · Jun 6, 2018 · 6432841 · 6432841
1 parent 45977a2
commit 6432841
Show file tree

Hide file tree

Showing 12 changed files with 961 additions and 237 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -179,46 +179,6 @@ jobs:
       - store_test_results:
           path: /go/out/tests
 
-  e2e-mixer:
-    <<: *integrationDefaults
-    environment:
-            - KUBECONFIG: /go/out/minikube.conf
-            - TEST_ENV: minikube-none
-            - GOPATH: /go
-    steps:
-      - <<: *initWorkingDir
-      - checkout
-      - attach_workspace:
-          at:  /go
-      - <<: *markJobStartsOnGCS
-      - run: make sync
-      - run: bin/testEnvRootMinikube.sh start
-      - run:
-          command: |
-            if [ ! -f /go/out/linux_amd64/release/pilot-discovery ]; then
-              # Should only happen when re-running a job, and the workspace is gone
-              time make build test-bins
-            fi
-            make docker.all generate_yaml
-      - run: bin/testEnvRootMinikube.sh wait
-      - run: docker images
-      - run:
-          no_output_timeout: 20m
-          # Run the test even if previous failed
-          when: always
-          name: make e2e_mixer
-          command: |
-            make test/local/e2e_mixer
-      - <<: *recordZeroExitCodeIfTestPassed
-      - <<: *recordNonzeroExitCodeIfTestFailed
-      - <<: *markJobFinishesOnGCS
-      - store_artifacts:
-          path: /home/circleci/logs
-      - store_artifacts:
-          path: /tmp
-      - store_test_results:
-          path: /go/out/tests
-
   e2e-galley:
     <<: *integrationDefaults
     environment:
@@ -609,9 +569,6 @@ workflows:
       - e2e-mixer-noauth-v1alpha3-v2:
           requires:
             - build
-      - e2e-mixer:
-          requires:
-            - build
       - e2e-galley:
           requires:
             - build
@@ -637,9 +594,6 @@ workflows:
       - e2e-mixer-noauth-v1alpha3-v2:
           requires:
             - build
-      - e2e-mixer:
-          requires:
-            - build
       - e2e-galley:
           requires:
             - build

diff --git a/mixer/pkg/protobuf/yaml/dynamic/builder.go b/mixer/pkg/protobuf/yaml/dynamic/builder.go
@@ -39,15 +39,24 @@ type (
 		resolver    yaml.Resolver
 		compiler    Compiler
 		skipUnknown bool
+		namedTypes  map[string]NamedEncoderBuilderFunc
 	}
+
+	// NamedEncoderBuilderFunc funcs have a special way to process input for encoding specific types
+	// for example istio...Value field accepts user input in a specific way
+	NamedEncoderBuilderFunc func(m *descriptor.DescriptorProto, fd *descriptor.FieldDescriptorProto, v interface{}, compiler Compiler) (Encoder, error)
 )
 
 // NewEncoderBuilder creates an EncoderBuilder.
 func NewEncoderBuilder(resolver yaml.Resolver, compiler Compiler, skipUnknown bool) *Builder {
 	return &Builder{
 		resolver:    resolver,
 		compiler:    compiler,
-		skipUnknown: skipUnknown}
+		skipUnknown: skipUnknown,
+		namedTypes: map[string]NamedEncoderBuilderFunc{
+			valueTypeName: valueTypeEncoderBuilder,
+		},
+	}
 }
 
 // Build builds an Encoder
@@ -170,7 +179,7 @@ func (c Builder) buildMessage(md *descriptor.DescriptorProto, data map[string]in
 	}
 
 	// sorting fields is recommended
-	sort.Slice(me.fields, func(i, j int) bool {
+	sort.SliceStable(me.fields, func(i, j int) bool {
 		return me.fields[i].number < me.fields[j].number
 	})
 
@@ -262,24 +271,32 @@ func (c Builder) buildPrimitiveField(v interface{}, fd *descriptor.FieldDescript
 func (c Builder) buildMessageField(ma []interface{}, m *descriptor.DescriptorProto,
 	fd *descriptor.FieldDescriptorProto, me *messageEncoder) error {
 
+	namedBuilder := c.namedTypes[fd.GetTypeName()]
+
 	for _, vv := range ma {
-		var ok bool
+		var de Encoder
+		var err error
 
-		var vq map[string]interface{}
-		if vq, ok = vv.(map[string]interface{}); !ok {
-			return fmt.Errorf("unable to process: %v, got %T, want: map[string]interface{}", fd, vv)
+		if namedBuilder != nil {
+			de, err = namedBuilder(m, fd, vv, c.compiler)
+		} else {
+			var ok bool
+			var vq map[string]interface{}
+			if vq, ok = vv.(map[string]interface{}); !ok {
+				return fmt.Errorf("unable to process: %v, got %T, want: map[string]interface{}", fd, vv)
+			}
+
+			de, err = c.buildMessage(m, vq, false)
 		}
 
-		var de Encoder
-		var err error
-		if de, err = c.buildMessage(m, vq, false); err != nil {
+		if err != nil {
 			return fmt.Errorf("unable to build message field: %v, %v", fd.GetName(), err)
 		}
+
 		fld := makeField(fd)
 		fld.encoder = []Encoder{de}
 		me.fields = append(me.fields, fld)
 	}
-
 	return nil
 }
 

diff --git a/mixer/pkg/protobuf/yaml/dynamic/encoder.go b/mixer/pkg/protobuf/yaml/dynamic/encoder.go
@@ -78,6 +78,7 @@ func (m messageEncoder) encodeWithoutLength(bag attribute.Bag, ba []byte) ([]byt
 
 // expected length of the varint encoded word
 // 2 byte words represent 2 ** 14 = 16K bytes
+// 1 byte words represent 2 ** 7 = 128 bytes
 // If message length is more, it involves an array copy
 const defaultMsgLengthSize = 2
 

diff --git a/mixer/pkg/protobuf/yaml/dynamic/encoder_test.go b/mixer/pkg/protobuf/yaml/dynamic/encoder_test.go
@@ -258,7 +258,7 @@ oth:
     dbl: 99.99
 r_oth:
   - str: "mystring2"
-    i64: 33333
+    i64: 66666
     dbl: 333.333
     b: true
     inenum: INNERTHREE
@@ -267,7 +267,7 @@ r_oth:
       i64: 99
       dbl: 99.99
   - str: "mystring3"
-    i64: 123
+    i64: 77777
     dbl: 333.333
     b: true
     inenum: INNERTHREE
@@ -276,7 +276,7 @@ r_oth:
       i64: 99
       dbl: 99.99
   - str: "mystring3"
-    i64: 123
+    i64: 88888
     dbl: 333.333
     b: true
     inenum: INNERTHREE
@@ -449,7 +449,7 @@ oth:
     dbl: 99.99
 r_oth:
   - str: "'mystring2'"
-    i64: 33333
+    i64: 66666
     dbl: 333.333
     b: true
     inenum: "'INNERTHREE'"
@@ -458,7 +458,7 @@ r_oth:
       i64: 99
       dbl: 99.99
   - str: "'mystring3'"
-    i64: 123
+    i64: 77777
     dbl: 333.333
     b: true
     inenum: request.path
@@ -467,7 +467,7 @@ r_oth:
       i64: 99
       dbl: 99.99
   - str: "'mystring3'"
-    i64: 123
+    i64: 88888
     dbl: 333.333
     b: true
     inenum: "'INNERTHREE'"
@@ -542,6 +542,26 @@ map_str_sint64:
     key1: 123
 `
 
+const valueStrIn = `
+istio_value: test.i64
+map_str_istio_value:
+    test.i64: test.i64
+    float: 5.5
+    str: request.path
+`
+
+const valueStr = `
+istio_value:
+    int64_value: -123
+map_str_istio_value:
+    test.i64: 
+        int64_value: -123
+    float:
+         double_value: 5.5
+    str:
+         string_value: "INNERTHREE"
+`
+
 type testdata struct {
 	desc        string
 	input       string
@@ -556,7 +576,7 @@ func TestDynamicEncoder(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	compiler := compiled.NewBuilder(statdardVocabulary())
+	compiler := compiled.NewBuilder(StatdardVocabulary())
 	res := protoyaml.NewResolver(fds)
 	for _, td := range []testdata{
 		{
@@ -569,8 +589,8 @@ func TestDynamicEncoder(t *testing.T) {
 		{
 			desc:     "metrics",
 			msg:      ".foo.Simple",
-			input:    everythingIn,
-			output:   everything,
+			input:    everythingIn + valueStrIn,
+			output:   everything + valueStr,
 			compiler: compiler,
 		},
 	} {
@@ -630,14 +650,10 @@ func TestStaticEncoder(t *testing.T) {
 
 func testMsg(t *testing.T, input string, output string, res protoyaml.Resolver,
 	compiler Compiler, msgName string, skipUnknown bool) {
-	//data := map[interface{}]interface{}{}
 	data := map[string]interface{}{}
 	var err error
 	var ba []byte
 
-	//if err = yaml2.Unmarshal([]byte(input), data); err != nil {
-	//	t.Fatalf("unable to unmarshal: %v\n%s", err, input)
-	//}
 	if ba, err = yaml.YAMLToJSON([]byte(input)); err != nil {
 		t.Fatalf("failed to marshal: %v", err)
 	}
@@ -702,17 +718,17 @@ func testMsg(t *testing.T, input string, output string, res protoyaml.Resolver,
 	if err != nil {
 		t.Fatalf("unable to decode: %v", err)
 	}
+	t.Logf("ff2 = %v", ff2)
 
 	// confirm that codegen'd code direct unmarshal and unmarhal thru bytes yields the same result.
 
 	if !reflect.DeepEqual(ff2, ff1) {
 		s, _ := diff.PrettyDiff(ff2, ff1)
 		t.Logf("difference: %s", s)
 		t.Fatalf("\n got: %v\nwant: %v", ff2, ff1)
+	} else {
+		t.Logf("\n got: %v\nwant: %v", ff2, ff1)
 	}
-
-	t.Logf("ff2 = %v", ff2)
-
 }
 
 func Test_transFormQuotedString(t *testing.T) {
@@ -741,7 +757,8 @@ func Test_transFormQuotedString(t *testing.T) {
 	}
 }
 
-func statdardVocabulary() ast.AttributeDescriptorFinder {
+// StatdardVocabulary returns Istio standard vocabulary
+func StatdardVocabulary() ast.AttributeDescriptorFinder {
 	attrs := map[string]*v1beta1.AttributeManifest_AttributeInfo{
 		"api.operation":                   {ValueType: v1beta1.STRING},
 		"api.protocol":                    {ValueType: v1beta1.STRING},

diff --git a/mixer/pkg/protobuf/yaml/dynamic/errors_test.go b/mixer/pkg/protobuf/yaml/dynamic/errors_test.go
@@ -284,7 +284,7 @@ func TestBuilderErrors(t *testing.T) {
 		t.Fatal(err)
 	}
 	res := yaml.NewResolver(fds)
-	compiler := compiled.NewBuilder(statdardVocabulary())
+	compiler := compiled.NewBuilder(StatdardVocabulary())
 
 	for _, td := range []struct {
 		desc        string