feat(sql/ast): Render AST to SQL (#2815)

Add a new Format method to AST nodes to support writing an AST tree back to SQL. The rendered SQL is checked to be equivalent to the input SQL, not exact. The tests current run against all query.sql files in pgx/v5 directories. As support is improved, we'll expand to all PostgreSQL queries.

The hope is to eventually use this to power a sqlc fmt subcommand, but that is a ways off from happening.

Author: kyleconroy

internal/endtoend/case_test.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+type Testcase struct {
+	Name       string
+	Path       string
+	ConfigName string
+	Stderr     []byte
+	Exec       *Exec
+}
+
+type Exec struct {
+	Command  string            `json:"command"`
+	Contexts []string          `json:"contexts"`
+	Process  string            `json:"process"`
+	Env      map[string]string `json:"env"`
+}
+
+func parseStderr(t *testing.T, dir, testctx string) []byte {
+	t.Helper()
+	paths := []string{
+		filepath.Join(dir, "stderr", fmt.Sprintf("%s.txt", testctx)),
+		filepath.Join(dir, "stderr.txt"),
+	}
+	for _, path := range paths {
+		if _, err := os.Stat(path); !os.IsNotExist(err) {
+			blob, err := os.ReadFile(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return blob
+		}
+	}
+	return nil
+}
+
+func parseExec(t *testing.T, dir string) *Exec {
+	t.Helper()
+	path := filepath.Join(dir, "exec.json")
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil
+	}
+	var e Exec
+	blob, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.Unmarshal(blob, &e); err != nil {
+		t.Fatal(err)
+	}
+	if e.Command == "" {
+		e.Command = "generate"
+	}
+	return &e
+}
+
+func FindTests(t *testing.T, root, testctx string) []*Testcase {
+	var tcs []*Testcase
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Name() == "sqlc.json" || info.Name() == "sqlc.yaml" || info.Name() == "sqlc.yml" {
+			dir := filepath.Dir(path)
+			tcs = append(tcs, &Testcase{
+				Path:       dir,
+				Name:       strings.TrimPrefix(dir, root+string(filepath.Separator)),
+				ConfigName: info.Name(),
+				Stderr:     parseStderr(t, dir, testctx),
+				Exec:       parseExec(t, dir),
+			})
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tcs
+}

internal/endtoend/endtoend_test.go

@@ -3,8 +3,6 @@ package main
 import (
 	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
 	"os"
 	osexec "os/exec"
 	"path/filepath"
@@ -97,20 +95,6 @@ func TestReplay(t *testing.T) {
 
 	// t.Parallel()
 	ctx := context.Background()
-	var dirs []string
-	err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.Name() == "sqlc.json" || info.Name() == "sqlc.yaml" || info.Name() == "sqlc.yml" {
-			dirs = append(dirs, filepath.Dir(path))
-			return filepath.SkipDir
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	contexts := map[string]textContext{
 		"base": {
@@ -135,24 +119,29 @@ func TestReplay(t *testing.T) {
 		},
 	}
 
-	for _, replay := range dirs {
-		tc := replay
-		for name, testctx := range contexts {
-			name := name
-			testctx := testctx
+	for name, testctx := range contexts {
+		name := name
+		testctx := testctx
 
-			if !testctx.Enabled() {
-				continue
-			}
+		if !testctx.Enabled() {
+			continue
+		}
 
-			t.Run(filepath.Join(name, tc), func(t *testing.T) {
+		for _, replay := range FindTests(t, "testdata", name) {
+			tc := replay
+			t.Run(filepath.Join(name, tc.Name), func(t *testing.T) {
 				t.Parallel()
+
 				var stderr bytes.Buffer
 				var output map[string]string
 				var err error
 
-				path, _ := filepath.Abs(tc)
-				args := parseExec(t, path)
+				path, _ := filepath.Abs(tc.Path)
+				args := tc.Exec
+				if args == nil {
+					args = &Exec{Command: "generate"}
+				}
+				expected := string(tc.Stderr)
 
 				if args.Process != "" {
 					_, err := osexec.LookPath(args.Process)
@@ -167,7 +156,6 @@ func TestReplay(t *testing.T) {
 					}
 				}
 
-				expected := expectedStderr(t, path, name)
 				opts := cmd.Options{
 					Env: cmd.Env{
 						Debug:    opts.DebugFromString(args.Env["SQLCDEBUG"]),
@@ -263,50 +251,6 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) {
 	}
 }
 
-func expectedStderr(t *testing.T, dir, testctx string) string {
-	t.Helper()
-	paths := []string{
-		filepath.Join(dir, "stderr", fmt.Sprintf("%s.txt", testctx)),
-		filepath.Join(dir, "stderr.txt"),
-	}
-	for _, path := range paths {
-		if _, err := os.Stat(path); !os.IsNotExist(err) {
-			blob, err := os.ReadFile(path)
-			if err != nil {
-				t.Fatal(err)
-			}
-			return string(blob)
-		}
-	}
-	return ""
-}
-
-type exec struct {
-	Command  string            `json:"command"`
-	Process  string            `json:"process"`
-	Contexts []string          `json:"contexts"`
-	Env      map[string]string `json:"env"`
-}
-
-func parseExec(t *testing.T, dir string) exec {
-	t.Helper()
-	var e exec
-	path := filepath.Join(dir, "exec.json")
-	if _, err := os.Stat(path); !os.IsNotExist(err) {
-		blob, err := os.ReadFile(path)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := json.Unmarshal(blob, &e); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if e.Command == "" {
-		e.Command = "generate"
-	}
-	return e
-}
-
 func BenchmarkReplay(b *testing.B) {
 	ctx := context.Background()
 	var dirs []string

internal/endtoend/fmt_test.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	pg_query "github.com/pganalyze/pg_query_go/v4"
+	"github.com/sqlc-dev/sqlc/internal/debug"
+	"github.com/sqlc-dev/sqlc/internal/engine/postgresql"
+	"github.com/sqlc-dev/sqlc/internal/sql/ast"
+)
+
+func TestFormat(t *testing.T) {
+	t.Parallel()
+	parse := postgresql.NewParser()
+	for _, tc := range FindTests(t, "testdata", "base") {
+		tc := tc
+
+		if !strings.Contains(tc.Path, filepath.Join("pgx/v5")) {
+			continue
+		}
+
+		q := filepath.Join(tc.Path, "query.sql")
+		if _, err := os.Stat(q); os.IsNotExist(err) {
+			continue
+		}
+
+		t.Run(tc.Name, func(t *testing.T) {
+			contents, err := os.ReadFile(q)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for i, query := range bytes.Split(bytes.TrimSpace(contents), []byte(";")) {
+				if len(query) <= 1 {
+					continue
+				}
+				query := query
+				t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+					expected, err := pg_query.Fingerprint(string(query))
+					if err != nil {
+						t.Fatal(err)
+					}
+					stmts, err := parse.Parse(bytes.NewReader(query))
+					if err != nil {
+						t.Fatal(err)
+					}
+					if len(stmts) != 1 {
+						t.Fatal("expected one statement")
+					}
+					if false {
+						r, err := pg_query.Parse(string(query))
+						debug.Dump(r, err)
+					}
+
+					out := ast.Format(stmts[0].Raw)
+					actual, err := pg_query.Fingerprint(out)
+					if err != nil {
+						t.Error(err)
+					}
+					if expected != actual {
+						debug.Dump(stmts[0].Raw)
+						t.Errorf("- %s", expected)
+						t.Errorf("- %s", string(query))
+						t.Errorf("+ %s", actual)
+						t.Errorf("+ %s", out)
+					}
+				})
+			}
+		})
+	}
+}

internal/endtoend/testdata/join_where_clause/postgresql/pgx/v5/query.sql

@@ -14,4 +14,4 @@ WHERE owner = $1;
 SELECT foo.*
 FROM foo
 CROSS JOIN bar
-WHERE bar.id = $2 AND owner = $1;
+WHERE bar.id = $2 AND owner = $1;

internal/endtoend/testdata/materialized_views/postgresql/pgx/v4/go/query.sql.go

@@ -1,11 +1,2 @@
-CREATE TABLE authors (
-    id   BIGSERIAL PRIMARY KEY,
-    name TEXT NOT NULL,
-    bio  TEXT
-);
-
-ALTER TABLE authors ADD COLUMN gender INTEGER NULL;
-
-CREATE MATERIALIZED VIEW authors_names as SELECT name from authors;
-
+-- name: ListAuthors :many
 SELECT * FROM authors;