Merge pull request kubernetes#65298 from nikhita/cherrypick-jsoniter-bump-1.9

Automatic merge from submit-queue.

Manual cherrypick of kubernetes#65034 to 1.9: make json serializer case sensitive

fixes partially kubernetes#64612

This PR imports the latest jsoniterator library so that case sensitivity during unmarshalling is optional. The PR also sets Kubernetes json serializer to be case sensitive.

**Release note**:

```release-note
ACTION REQUIRED: Kubernetes JSON deserializer is now case-sensitive to restore compatibility with pre-1.8 servers. If your config files contains fields with wrong case, the config files will be now invalid.
```

/sig api-machinery
/kind bug
/assign caesarxuchao liggitt thockin sttts mbohlool

Author: Kubernetes Submit Queue

Godeps/Godeps.json

@@ -99,6 +99,7 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/protobuf:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",

Godeps/LICENSES

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	k8s_json "k8s.io/apimachinery/pkg/runtime/serializer/json"
 	"k8s.io/apimachinery/pkg/runtime/serializer/streaming"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -568,3 +569,31 @@ func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
 	}
 	b.StopTimer()
 }
+
+// BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary provides a
+// baseline for JSON decode performance with
+// jsoniter.ConfigCompatibleWithStandardLibrary, but with case sensitivity set
+// to true
+func BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary(b *testing.B) {
+	kcodec := testapi.Default.Codec()
+	items := benchmarkItems(b)
+	width := len(items)
+	encoded := make([][]byte, width)
+	for i := range items {
+		data, err := runtime.Encode(kcodec, &items[i])
+		if err != nil {
+			b.Fatal(err)
+		}
+		encoded[i] = data
+	}
+
+	b.ResetTimer()
+	iter := k8s_json.CaseSensitiveJsonIterator()
+	for i := 0; i < b.N; i++ {
+		obj := v1.Pod{}
+		if err := iter.Unmarshal(encoded[i%width], &obj); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}

pkg/api/testing/BUILD

@@ -22,9 +22,9 @@ go_test(
     library = ":go_default_library",
     deps = [
         "//vendor/github.com/ghodss/yaml:go_default_library",
-        "//vendor/github.com/json-iterator/go:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
     ],
 )
 

pkg/api/testing/serialization_test.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"testing"
 
-	jsoniter "github.com/json-iterator/go"
+	k8s_json "k8s.io/apimachinery/pkg/runtime/serializer/json"
 )
 
 type GroupVersionHolder struct {
@@ -47,7 +47,8 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
 			t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
 		}
 		// test the json-iterator codec
-		if err := jsoniter.ConfigFastest.Unmarshal(c.input, &result); err != nil {
+		iter := k8s_json.CaseSensitiveJsonIterator()
+		if err := iter.Unmarshal(c.input, &result); err != nil {
 			t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
 		}
 		if !reflect.DeepEqual(result.GV, c.expect) {

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"testing"
 
-	jsoniter "github.com/json-iterator/go"
+	k8s_json "k8s.io/apimachinery/pkg/runtime/serializer/json"
 )
 
 func TestVerbsUgorjiMarshalJSON(t *testing.T) {
@@ -45,7 +45,7 @@ func TestVerbsUgorjiMarshalJSON(t *testing.T) {
 	}
 }
 
-func TestVerbsUgorjiUnmarshalJSON(t *testing.T) {
+func TestVerbsUJsonIterUnmarshalJSON(t *testing.T) {
 	cases := []struct {
 		input  string
 		result APIResource
@@ -56,9 +56,10 @@ func TestVerbsUgorjiUnmarshalJSON(t *testing.T) {
 		{`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}},
 	}
 
+	iter := k8s_json.CaseSensitiveJsonIterator()
 	for i, c := range cases {
 		var result APIResource
-		if err := jsoniter.ConfigFastest.Unmarshal([]byte(c.input), &result); err != nil {
+		if err := iter.Unmarshal([]byte(c.input), &result); err != nil {
 			t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
 		}
 		if !reflect.DeepEqual(result, c.result) {

staging/src/k8s.io/apimachinery/Godeps/Godeps.json

@@ -98,6 +98,20 @@ func init() {
 	jsoniter.RegisterTypeDecoderFunc("interface {}", decodeNumberAsInt64IfPossible)
 }
 
+// CaseSensitiveJsonIterator returns a jsoniterator API that's configured to be
+// case-sensitive when unmarshalling, and otherwise compatible with
+// the encoding/json standard library.
+func CaseSensitiveJsonIterator() jsoniter.API {
+	return jsoniter.Config{
+		EscapeHTML:             true,
+		SortMapKeys:            true,
+		ValidateJsonRawMessage: true,
+		CaseSensitive:          true,
+	}.Froze()
+}
+
+var caseSensitiveJsonIterator = CaseSensitiveJsonIterator()
+
 // Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then
 // load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, the raw data will be
 // extracted and no decoding will be performed. If into is not registered with the typer, then the object will be straight decoded using
@@ -154,7 +168,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 		types, _, err := s.typer.ObjectKinds(into)
 		switch {
 		case runtime.IsNotRegisteredError(err), isUnstructured:
-			if err := jsoniter.ConfigFastest.Unmarshal(data, into); err != nil {
+			if err := caseSensitiveJsonIterator.Unmarshal(data, into); err != nil {
 				return nil, actual, err
 			}
 			return into, actual, nil
@@ -188,7 +202,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 		return nil, actual, err
 	}
 
-	if err := jsoniter.ConfigFastest.Unmarshal(data, obj); err != nil {
+	if err := caseSensitiveJsonIterator.Unmarshal(data, obj); err != nil {
 		return nil, actual, err
 	}
 	return obj, actual, nil
@@ -197,7 +211,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 // Encode serializes the provided object to the given writer.
 func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
 	if s.yaml {
-		json, err := jsoniter.ConfigFastest.Marshal(obj)
+		json, err := caseSensitiveJsonIterator.Marshal(obj)
 		if err != nil {
 			return err
 		}
@@ -210,7 +224,7 @@ func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
 	}
 
 	if s.pretty {
-		data, err := jsoniter.ConfigFastest.MarshalIndent(obj, "", "  ")
+		data, err := caseSensitiveJsonIterator.MarshalIndent(obj, "", "  ")
 		if err != nil {
 			return err
 		}

staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/BUILD

@@ -30,10 +30,31 @@ import (
 
 type testDecodable struct {
 	Other string
-	Value int `json:"value"`
+	Value int           `json:"value"`
+	Spec  DecodableSpec `json:"spec"`
 	gvk   schema.GroupVersionKind
 }
 
+// DecodableSpec has 15 fields. json-iterator treats struct with more than 10
+// fields differently from struct that has less than 10 fields.
+type DecodableSpec struct {
+	A int `json:"A"`
+	B int `json:"B"`
+	C int `json:"C"`
+	D int `json:"D"`
+	E int `json:"E"`
+	F int `json:"F"`
+	G int `json:"G"`
+	H int `json:"h"`
+	I int `json:"i"`
+	J int `json:"j"`
+	K int `json:"k"`
+	L int `json:"l"`
+	M int `json:"m"`
+	N int `json:"n"`
+	O int `json:"o"`
+}
+
 func (d *testDecodable) GetObjectKind() schema.ObjectKind                { return d }
 func (d *testDecodable) SetGroupVersionKind(gvk schema.GroupVersionKind) { d.gvk = gvk }
 func (d *testDecodable) GroupVersionKind() schema.GroupVersionKind       { return d.gvk }
@@ -195,6 +216,28 @@ func TestDecode(t *testing.T) {
 				},
 			},
 		},
+		// Unmarshalling is case-sensitive
+		{
+			// "VaLue" should have been "value"
+			data:        []byte(`{"kind":"Test","apiVersion":"other/blah","VaLue":1,"Other":"test"}`),
+			into:        &testDecodable{},
+			typer:       &mockTyper{err: runtime.NewNotRegisteredErrForKind(schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"})},
+			expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
+			expectedObject: &testDecodable{
+				Other: "test",
+			},
+		},
+		// Unmarshalling is case-sensitive for big struct.
+		{
+			// "b" should have been "B", "I" should have been "i"
+			data:        []byte(`{"kind":"Test","apiVersion":"other/blah","spec": {"A": 1, "b": 2, "h": 3, "I": 4}}`),
+			into:        &testDecodable{},
+			typer:       &mockTyper{err: runtime.NewNotRegisteredErrForKind(schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"})},
+			expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
+			expectedObject: &testDecodable{
+				Spec: DecodableSpec{A: 1, H: 3},
+			},
+		},
 	}
 
 	for i, test := range testCases {