test(storage): add scenario 8 to retry conformance tests (#6556)

* test(storage): add scenarios to retry conf test

* update NewRangeReader to handle downloads interrupted midstream with status 200

* update tests to check data integrity

* add more instructions

* revert changes to NewRangeReader after testbench fix

* fix license header per checker requirements

* run protoc again

* run goimports

* upload large test files only for s8

* fix nit

storage/internal/test/conformance/retry_tests.json

@@ -239,6 +239,43 @@
       ],
       "preconditionProvided": true,
       "expectSuccess": false
+    },
+    {
+      "id": 7,
+      "description": "resumable_uploads_handle_complex_retries",
+      "cases": [
+        {
+          "instructions": ["return-reset-connection", "return-503"]
+        },
+        {
+          "instructions": ["return-503-after-256K"]
+        },
+        {
+          "instructions": ["return-503-after-8192K"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.objects.insert", "group": "storage.resumable.upload", "resources": ["BUCKET"]}
+      ],
+      "preconditionProvided": true,
+      "expectSuccess": true
+    },
+    {
+      "id": 8,
+      "description": "downloads_handle_complex_retries",
+      "cases": [
+        {
+          "instructions": ["return-broken-stream"]
+        },
+        {
+          "instructions": ["return-broken-stream-after-256K"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.objects.get", "group": "storage.objects.download", "resources": ["BUCKET", "OBJECT"]}
+      ],
+      "preconditionProvided": false,
+      "expectSuccess": true
     }
   ]
 }

storage/internal/test/conformance/test.proto

@@ -1,4 +1,4 @@
-// Copyright 2019, Google LLC
+// Copyright 2019 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -177,6 +177,7 @@ enum Resource {
 message Method {
   string name = 1;  // e.g. storage.objects.get
   repeated Resource resources = 2;
+  string group = 3; // e.g. storage.resumable.upload
 }
 
 // Schema for a retry test, corresponding to a single scenario from the design

storage/retry_conformance_test.go

@@ -17,6 +17,7 @@ package storage
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -43,6 +44,8 @@ var (
 	projectID           = "my-project-id"
 	serviceAccountEmail = "my-sevice-account@my-project-id.iam.gserviceaccount.com"
 	randomBytesToWrite  = []byte("abcdef")
+	randomBytes9MB      = generateRandomBytes(size9MB)
+	size9MB             = 9437184 // 9 MiB
 )
 
 type retryFunc func(ctx context.Context, c *Client, fs *resources, preconditions bool) error
@@ -199,10 +202,39 @@ var methods = map[string][]retryFunc{
 			if err != nil {
 				return err
 			}
-			_, err = io.Copy(ioutil.Discard, r)
+			wr, err := io.Copy(ioutil.Discard, r)
+			if got, want := wr, len(randomBytesToWrite); got != int64(want) {
+				return fmt.Errorf("body length mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
+			}
 			return err
 		},
 	},
+	"storage.objects.download": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			// Before running the test method, populate a large test object of 9 MiB.
+			objName := objectIDs.New()
+			if err := uploadTestObject(fs.bucket.Name, objName, randomBytes9MB); err != nil {
+				return fmt.Errorf("failed to create 9 MiB large object pre test, err: %v", err)
+			}
+			// Download the large test object for the S8 download method group.
+			r, err := c.Bucket(fs.bucket.Name).Object(objName).NewReader(ctx)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+			data, err := ioutil.ReadAll(r)
+			if err != nil {
+				return fmt.Errorf("failed to ReadAll, err: %v", err)
+			}
+			if got, want := len(data), size9MB; got != want {
+				return fmt.Errorf("body length mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
+			}
+			if got, want := data, randomBytes9MB; !bytes.Equal(got, want) {
+				return fmt.Errorf("body mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
+			}
+			return nil
+		},
+	},
 	"storage.objects.list": {
 		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
 			it := c.Bucket(fs.bucket.Name).Objects(ctx, nil)
@@ -404,11 +436,15 @@ func TestRetryConformance(t *testing.T) {
 		for _, retryTest := range testFile.RetryTests {
 			for _, instructions := range retryTest.Cases {
 				for _, method := range retryTest.Methods {
-					if len(methods[method.Name]) == 0 {
-						t.Logf("No tests for operation %v", method.Name)
+					methodName := method.Name
+					if method.Group != "" {
+						methodName = method.Group
+					}
+					if len(methods[methodName]) == 0 {
+						t.Logf("No tests for operation %v", methodName)
 					}
-					for i, fn := range methods[method.Name] {
-						testName := fmt.Sprintf("%v-%v-%v-%v", retryTest.Id, instructions.Instructions, method.Name, i)
+					for i, fn := range methods[methodName] {
+						testName := fmt.Sprintf("%v-%v-%v-%v", retryTest.Id, instructions.Instructions, methodName, i)
 						t.Run(testName, func(t *testing.T) {
 
 							// Create the retry subtest
@@ -511,6 +547,32 @@ func (et *emulatorTest) populateResources(ctx context.Context, c *Client, resour
 	}
 }
 
+// Generates size random bytes.
+func generateRandomBytes(n int) []byte {
+	b := make([]byte, n)
+	_, _ = rand.Read(b)
+	return b
+}
+
+// Upload test object with given bytes.
+func uploadTestObject(bucketName, objName string, n []byte) error {
+	// Create non-wrapped client to create test object.
+	ctx := context.Background()
+	c, err := NewClient(ctx)
+	if err != nil {
+		return fmt.Errorf("storage.NewClient: %v", err)
+	}
+	obj := c.Bucket(bucketName).Object(objName)
+	w := obj.NewWriter(ctx)
+	if _, err := w.Write(n); err != nil {
+		return fmt.Errorf("writing test object: %v", err)
+	}
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("closing object: %v", err)
+	}
+	return nil
+}
+
 // Creates a retry test resource in the emulator
 func (et *emulatorTest) create(instructions map[string][]string) {
 	c := http.DefaultClient