test(analyzer): Update endtoend tests for new analyzer

Add a `contexts` key to exec.json to opt certain tests into or out of
database-backed analysis.

Fix many incorrect test cases that didn't run against an actual
database.

Author: kyleconroy

examples/authors/sqlc.yaml

@@ -6,6 +6,7 @@ sql:
   queries: postgresql/query.sql
   engine: postgresql
   database:
+    analyzer: false
     managed: true
   rules:
   - sqlc/db-prepare

examples/booktest/sqlc.json

@@ -11,7 +11,8 @@
       "queries": "postgresql/query.sql",
       "engine": "postgresql",
       "database": {
-        "managed": true
+        "managed": true,
+        "analyzer": false
       },
       "rules": [
         "sqlc/db-prepare"

examples/jets/sqlc.json

@@ -11,6 +11,7 @@
       "queries": "postgresql/query-building.sql",
       "engine": "postgresql",
       "database": {
+        "analyzer": false,
         "managed": true
       },
       "rules": [

examples/ondeck/sqlc.json

@@ -11,7 +11,8 @@
       "queries": "postgresql/query",
       "engine": "postgresql",
       "database": {
-        "managed": true
+        "managed": true,
+        "analyzer": false
       },
       "rules": [
         "sqlc/db-prepare"

internal/endtoend/endtoend_test.go

@@ -12,13 +12,15 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"golang.org/x/exp/slices"
 
 	"github.com/sqlc-dev/sqlc/internal/cmd"
+	"github.com/sqlc-dev/sqlc/internal/config"
 	"github.com/sqlc-dev/sqlc/internal/opts"
 )
 
 func TestExamples(t *testing.T) {
-	t.Parallel()
+	// t.Parallel()
 	ctx := context.Background()
 
 	examples, err := filepath.Abs(filepath.Join("..", "..", "examples"))
@@ -37,10 +39,14 @@ func TestExamples(t *testing.T) {
 		}
 		tc := replay.Name()
 		t.Run(tc, func(t *testing.T) {
-			t.Parallel()
+			// t.Parallel()
 			path := filepath.Join(examples, tc)
 			var stderr bytes.Buffer
-			output, err := cmd.Generate(ctx, cmd.Env{}, path, "", &stderr)
+			opts := &cmd.Options{
+				Env:    cmd.Env{},
+				Stderr: &stderr,
+			}
+			output, err := cmd.Generate(ctx, path, "", opts)
 			if err != nil {
 				t.Fatalf("sqlc generate failed: %s", stderr.String())
 			}
@@ -68,18 +74,27 @@ func BenchmarkExamples(b *testing.B) {
 			path := filepath.Join(examples, tc)
 			for i := 0; i < b.N; i++ {
 				var stderr bytes.Buffer
-				cmd.Generate(ctx, cmd.Env{}, path, "", &stderr)
+				opts := &cmd.Options{
+					Env:    cmd.Env{},
+					Stderr: &stderr,
+				}
+				cmd.Generate(ctx, path, "", opts)
 			}
 		})
 	}
 }
 
+type textContext struct {
+	Mutate  func(*config.Config)
+	Enabled func() bool
+}
+
 func TestReplay(t *testing.T) {
 	// Ensure that this environment variable is always set to true when running
 	// end-to-end tests
 	os.Setenv("SQLC_DUMMY_VALUE", "true")
 
-	t.Parallel()
+	// t.Parallel()
 	ctx := context.Background()
 	var dirs []string
 	err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
@@ -95,52 +110,103 @@ func TestReplay(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	contexts := map[string]textContext{
+		"base": {
+			Mutate:  func(c *config.Config) {},
+			Enabled: func() bool { return true },
+		},
+		"managed-db": {
+			Mutate: func(c *config.Config) {
+				c.Cloud.Project = "01HAQMMECEYQYKFJN8MP16QC41" // TODO: Read from environment
+				for i := range c.SQL {
+					c.SQL[i].Database = &config.Database{
+						Managed: true,
+					}
+				}
+			},
+			Enabled: func() bool {
+				if len(os.Getenv("CI")) > 0 {
+					return false
+				}
+				return len(os.Getenv("SQLC_AUTH_TOKEN")) > 0
+			},
+		},
+	}
+
 	for _, replay := range dirs {
 		tc := replay
-		t.Run(tc, func(t *testing.T) {
-			t.Parallel()
+		for name, testctx := range contexts {
+			name := name
+			testctx := testctx
 
-			var stderr bytes.Buffer
-			var output map[string]string
-			var err error
+			if !testctx.Enabled() {
+				continue
+			}
 
-			path, _ := filepath.Abs(tc)
-			args := parseExec(t, path)
-			expected := expectedStderr(t, path)
+			t.Run(filepath.Join(name, tc), func(t *testing.T) {
+				t.Parallel()
+				var stderr bytes.Buffer
+				var output map[string]string
+				var err error
 
-			if args.Process != "" {
-				_, err := osexec.LookPath(args.Process)
-				if err != nil {
-					t.Skipf("executable not found: %s %s", args.Process, err)
+				path, _ := filepath.Abs(tc)
+				args := parseExec(t, path)
+
+				if args.Process != "" {
+					_, err := osexec.LookPath(args.Process)
+					if err != nil {
+						t.Skipf("executable not found: %s %s", args.Process, err)
+					}
 				}
-			}
 
-			env := cmd.Env{
-				Debug:    opts.DebugFromString(args.Env["SQLCDEBUG"]),
-				NoRemote: true,
-			}
-			switch args.Command {
-			case "diff":
-				err = cmd.Diff(ctx, env, path, "", &stderr)
-			case "generate":
-				output, err = cmd.Generate(ctx, env, path, "", &stderr)
-				if err == nil {
-					cmpDirectory(t, path, output)
+				if len(args.Contexts) > 0 {
+					if !slices.Contains(args.Contexts, name) {
+						t.Skipf("unsupported context: %s", name)
+					}
 				}
-			case "vet":
-				err = cmd.Vet(ctx, env, path, "", &stderr)
-			default:
-				t.Fatalf("unknown command")
-			}
 
-			if len(expected) == 0 && err != nil {
-				t.Fatalf("sqlc %s failed: %s", args.Command, stderr.String())
-			}
+				expected := expectedStderr(t, path)
+				opts := cmd.Options{
+					Env: cmd.Env{
+						Debug:    opts.DebugFromString(args.Env["SQLCDEBUG"]),
+						NoRemote: true,
+					},
+					Stderr:       &stderr,
+					MutateConfig: testctx.Mutate,
+				}
 
-			if diff := cmp.Diff(expected, stderr.String()); diff != "" {
-				t.Errorf("stderr differed (-want +got):\n%s", diff)
-			}
-		})
+				switch args.Command {
+				case "diff":
+					err = cmd.Diff(ctx, path, "", &opts)
+				case "generate":
+					output, err = cmd.Generate(ctx, path, "", &opts)
+					if err == nil {
+						cmpDirectory(t, path, output)
+					}
+				case "vet":
+					err = cmd.Vet(ctx, path, "", &opts)
+				default:
+					t.Fatalf("unknown command")
+				}
+
+				if len(expected) == 0 && err != nil {
+					t.Fatalf("sqlc %s failed: %s", args.Command, stderr.String())
+				}
+
+				var diff string
+				for _, expectedErr := range strings.Split(expected, "---\n") {
+					diff = cmp.Diff(strings.TrimSpace(expectedErr), strings.TrimSpace(stderr.String()))
+					if diff == "" {
+						break
+					}
+				}
+				if diff != "" {
+					t.Log(stderr.String()) // TODO: Remove
+					t.Fatalf("stderr differed (-want +got):\n%s", diff)
+				}
+			})
+		}
 	}
 }
 
@@ -192,16 +258,13 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) {
 		t.Errorf("%s contents differ", dir)
 		for name, contents := range expected {
 			name := name
-			tn := strings.Replace(name, dir+"/", "", -1)
-			t.Run(tn, func(t *testing.T) {
-				if actual[name] == "" {
-					t.Errorf("%s is empty", name)
-					return
-				}
-				if diff := cmp.Diff(contents, actual[name]); diff != "" {
-					t.Errorf("%s differed (-want +got):\n%s", name, diff)
-				}
-			})
+			if actual[name] == "" {
+				t.Errorf("%s is empty", name)
+				return
+			}
+			if diff := cmp.Diff(contents, actual[name]); diff != "" {
+				t.Errorf("%s differed (-want +got):\n%s", name, diff)
+			}
 		}
 	}
 }
@@ -220,9 +283,10 @@ func expectedStderr(t *testing.T, dir string) string {
 }
 
 type exec struct {
-	Command string            `json:"command"`
-	Process string            `json:"process"`
-	Env     map[string]string `json:"env"`
+	Command  string            `json:"command"`
+	Process  string            `json:"process"`
+	Contexts []string          `json:"contexts"`
+	Env      map[string]string `json:"env"`
 }
 
 func parseExec(t *testing.T, dir string) exec {
@@ -266,7 +330,11 @@ func BenchmarkReplay(b *testing.B) {
 			path, _ := filepath.Abs(tc)
 			for i := 0; i < b.N; i++ {
 				var stderr bytes.Buffer
-				cmd.Generate(ctx, cmd.Env{}, path, "", &stderr)
+				opts := &cmd.Options{
+					Env:    cmd.Env{},
+					Stderr: &stderr,
+				}
+				cmd.Generate(ctx, path, "", opts)
 			}
 		})
 	}

internal/endtoend/testdata/alias/postgresql/stdlib/schema.sql

@@ -1,2 +1 @@
-CREATE TABLE bar (id serial not null);
-
+CREATE TABLE bar (id serial not null);

internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/exec.json

@@ -0,0 +1,3 @@
+{
+  "contexts": ["managed-db"]
+}

internal/endtoend/testdata/func_return/postgresql/pgx/v5/go/db.go renamed to internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/go/db.go

@@ -0,0 +1,4 @@
+-- name: SumBaz :many
+SELECT bar, coalesce(sum(baz), 0) as quantity
+FROM foo
+GROUP BY 1;

internal/endtoend/testdata/func_return/postgresql/pgx/v5/go/models.go renamed to internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/go/models.go

@@ -0,0 +1,5 @@
+CREATE TABLE foo (
+    bar text,
+    baz bigint
+);
+

internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/go/query.sql.go

@@ -0,0 +1,19 @@
+{
+  "version": "1",
+  "cloud": {
+    "project": "01HAQMMECEYQYKFJN8MP16QC41"
+  },
+  "packages": [
+    {
+      "path": "go",
+      "engine": "postgresql",
+      "sql_package": "pgx/v5",
+      "database": {
+        "managed": true
+      },
+      "name": "querytest",
+      "schema": "schema.sql",
+      "queries": "query.sql"
+    }
+  ]
+}

internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/query.sql

@@ -0,0 +1,3 @@
+{
+  "contexts": ["base"]
+}

internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/schema.sql

@@ -0,0 +1,3 @@
+{
+  "contexts": ["base"]
+}

internal/endtoend/testdata/coalesce_as/postgresql/pganalyze/sqlc.json

@@ -0,0 +1,3 @@
+{
+  "contexts": ["base"]
+}