coprocess: handle non-UTF8 request bodies

coprocess.go

@@ -5,6 +5,7 @@ package main
 import (
 	"encoding/json"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/Sirupsen/logrus"
 
@@ -67,15 +68,6 @@ type CoProcessor struct {
 
 // ObjectFromRequest constructs a CoProcessObject from a given http.Request.
 func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object {
-	var body string
-	if r.Body == nil {
-		body = ""
-	} else {
-		defer r.Body.Close()
-		originalBody, _ := ioutil.ReadAll(r.Body)
-		body = string(originalBody)
-	}
-
 	headers := ProtoMap(r.Header)
 
 	host := r.Host
@@ -90,7 +82,6 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object {
 		Headers:        headers,
 		SetHeaders:     map[string]string{},
 		DeleteHeaders:  []string{},
-		Body:           body,
 		Url:            r.URL.Path,
 		Params:         ProtoMap(r.URL.Query()),
 		AddParams:      map[string]string{},
@@ -104,6 +95,16 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object {
 		Scheme:     r.URL.Scheme,
 	}
 
+	if r.Body != nil {
+		defer r.Body.Close()
+		rawBody, _ := ioutil.ReadAll(r.Body)
+		if utf8.Valid(rawBody) {
+			miniRequestObject.Body = string(rawBody)
+		} else {
+			miniRequestObject.RawBody = rawBody
+		}
+	}
+
 	object := &coprocess.Object{
 		Request:  miniRequestObject,
 		HookName: c.Middleware.HookName,

coprocess/proto/coprocess_mini_request_object.proto

@@ -19,4 +19,5 @@ message MiniRequestObject {
   string method = 11;
   string request_uri = 12;
   string scheme = 13;
+  bytes raw_body = 14;
 }

coprocess_python_test.go

@@ -4,6 +4,10 @@
 package main
 
 import (
+	"bytes"
+	"encoding/binary"
+	"math/rand"
+	"mime/multipart"
 	"testing"
 	"time"
 
@@ -30,7 +34,6 @@ from gateway import TykGateway as tyk
 
 @Hook
 def MyAuthHook(request, session, metadata, spec):
-    print("MyAuthHook is called")
     auth_header = request.get_header('Authorization')
     if auth_header == 'valid_token':
         session.rate = 1000.0
@@ -41,18 +44,59 @@ def MyAuthHook(request, session, metadata, spec):
 	`,
 }
 
+var pythonBundleWithPreHook = map[string]string{
+	"manifest.json": `
+		{
+		    "file_list": [
+		        "middleware.py"
+		    ],
+		    "custom_middleware": {
+		        "driver": "python",
+		        "pre": [{
+		            "name": "MyPreHook"
+		        }]
+		    }
+		}
+	`,
+	"middleware.py": `
+from tyk.decorators import *
+from gateway import TykGateway as tyk
+
+@Hook
+def MyPreHook(request, session, metadata, spec):
+    content_type = request.get_header("Content-Type")
+    if "json" in content_type:
+      if len(request.object.raw_body) != 0:
+        request.object.return_overrides.response_code = 400
+        request.object.return_overrides.response_error = "Raw body field isn't empty"
+        return request, session, metadata
+      if "{}" not in request.object.body:
+        request.object.return_overrides.response_code = 400
+        request.object.return_overrides.response_error = "Body field doesn't match"
+        return request, session, metadata
+    if "multipart" in content_type:
+      if len(request.object.body) != 0:
+        request.object.return_overrides.response_code = 400
+        request.object.return_overrides.response_error = "Body field isn't empty"
+      if len(request.object.raw_body) <= 0:
+        request.object.return_overrides.response_code = 400
+        request.object.return_overrides.response_error = "Raw body field is empty"
+    return request, session, metadata
+
+`,
+}
+
 func TestPythonBundles(t *testing.T) {
 	ts := newTykTestServer()
 	defer ts.Close()
-
-	bundleID := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck)
+	authCheckBundle := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck)
 
 	t.Run("Single-file bundle with authentication hook", func(t *testing.T) {
 		buildAndLoadAPI(func(spec *APISpec) {
 			spec.Proxy.ListenPath = "/test-api/"
 			spec.UseKeylessAccess = false
 			spec.EnableCoProcessAuth = true
-			spec.CustomMiddlewareBundle = bundleID
+			spec.CustomMiddlewareBundle = authCheckBundle
 			spec.VersionData.NotVersioned = true
 		})
 
@@ -66,4 +110,60 @@ func TestPythonBundles(t *testing.T) {
 			{Path: "/test-api/", Code: 403, Headers: invalidAuth},
 		}...)
 	})
+
+	preHookBundle := registerBundle("python_with_pre_hook", pythonBundleWithPreHook)
+	t.Run("Single-file bundle with pre hook and UTF-8/non-UTF-8 request data", func(t *testing.T) {
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.ListenPath = "/test-api-2/"
+			spec.UseKeylessAccess = true
+			spec.EnableCoProcessAuth = false
+			spec.CustomMiddlewareBundle = preHookBundle
+			spec.VersionData.NotVersioned = true
+		})
+
+		time.Sleep(1 * time.Second)
+
+		fileData := generateBinaryData()
+		var buf bytes.Buffer
+		multipartWriter := multipart.NewWriter(&buf)
+		file, err := multipartWriter.CreateFormFile("file", "test.bin")
+		if err != nil {
+			t.Fatalf("Couldn't use multipart writer: %s", err.Error())
+		}
+		_, err = fileData.WriteTo(file)
+		if err != nil {
+			t.Fatalf("Couldn't write to multipart file: %s", err.Error())
+		}
+		field, err := multipartWriter.CreateFormField("testfield")
+		if err != nil {
+			t.Fatalf("Couldn't use multipart writer: %s", err.Error())
+		}
+		_, err = field.Write([]byte("testvalue"))
+		if err != nil {
+			t.Fatalf("Couldn't write to form field: %s", err.Error())
+		}
+		err = multipartWriter.Close()
+		if err != nil {
+			t.Fatalf("Couldn't close multipart writer: %s", err.Error())
+		}
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/test-api-2/", Code: 200, Data: &buf, Headers: map[string]string{"Content-Type": multipartWriter.FormDataContentType()}},
+			{Path: "/test-api-2/", Code: 200, Data: "{}", Headers: map[string]string{"Content-Type": "application/json"}},
+		}...)
+	})
+}
+
+func generateBinaryData() (buf *bytes.Buffer) {
+	buf = new(bytes.Buffer)
+	type testData struct {
+		a float32
+		b float64
+		c uint32
+	}
+	for i := 0; i < 10; i++ {
+		s := &testData{rand.Float32(), rand.Float64(), rand.Uint32()}
+		binary.Write(buf, binary.BigEndian, s)
+	}
+	return buf
 }