encoding/jsonschema: commit external test stats

This makes the JSON Schema external test statistics a little more
obvious in code review.

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: Id508a24d0636e9af6a1a44d5fed98bf89947d986
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200529
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>

encoding/jsonschema/external_test.go

@@ -40,6 +40,7 @@ import (
 // The commit below references the JSON schema test main branch as of Sun May 19 19:01:03 2024 +0300
 
 //go:generate go run vendor_external.go -- 9fc880bfb6d8ccd093bc82431f17d13681ffae8e
+//go:generate go run teststats.go -o external_teststats.txt
 
 const testDir = "testdata/external"
 

encoding/jsonschema/external_teststats.txt

@@ -0,0 +1,10 @@
+# Generated by teststats. DO NOT EDIT
+v2:
+	schema extract (pass / total): 975 / 1637 = 59.6%
+	tests (pass / total): 3140 / 7175 = 43.8%
+	tests on extracted schemas (pass / total): 3140 / 3546 = 88.6%
+
+v3:
+	schema extract (pass / total): 967 / 1637 = 59.1%
+	tests (pass / total): 3074 / 7175 = 42.8%
+	tests on extracted schemas (pass / total): 3074 / 3538 = 86.9%

encoding/jsonschema/teststats.go

@@ -20,36 +20,50 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"os"
 	"path"
 	"sort"
 
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/jsonschema/internal/externaltest"
 )
 
-var list = flag.String("list", "", "list all failed tests for a given evaluator version")
+var (
+	list = flag.String("list", "", "list all failed tests for a given evaluator version")
+	out  = flag.String("o", "", "write output to a file")
+)
 
 const testDir = "testdata/external"
 
 func main() {
+	flag.Parse()
 	tests, err := externaltest.ReadTestDir(testDir)
 	if err != nil {
 		log.Fatal(err)
 	}
-	flag.Parse()
+	outw := os.Stdout
+	if *out != "" {
+		var err error
+		outw, err = os.Create(*out)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Fprintf(outw, "# Generated by teststats. DO NOT EDIT\n")
+	}
 	if *list != "" {
-		listFailures(*list, tests)
+		listFailures(outw, *list, tests)
 	} else {
-		fmt.Printf("v2:\n")
-		showStats("v2", tests)
-		fmt.Println()
-		fmt.Printf("v3:\n")
-		showStats("v3", tests)
+		fmt.Fprintf(outw, "v2:\n")
+		showStats(outw, "v2", tests)
+		fmt.Fprintf(outw, "\n")
+		fmt.Fprintf(outw, "v3:\n")
+		showStats(outw, "v3", tests)
 	}
 }
 
-func showStats(version string, tests map[string][]*externaltest.Schema) {
+func showStats(outw io.Writer, version string, tests map[string][]*externaltest.Schema) {
 	schemaOK := 0
 	schemaTot := 0
 	testOK := 0
@@ -76,17 +90,17 @@ func showStats(version string, tests map[string][]*externaltest.Schema) {
 			}
 		}
 	}
-	fmt.Printf("\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot))
-	fmt.Printf("\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot))
-	fmt.Printf("\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot))
+	fmt.Fprintf(outw, "\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot))
+	fmt.Fprintf(outw, "\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot))
+	fmt.Fprintf(outw, "\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot))
 }
 
-func listFailures(version string, tests map[string][]*externaltest.Schema) {
+func listFailures(outw io.Writer, version string, tests map[string][]*externaltest.Schema) {
 	for _, filename := range sortedKeys(tests) {
 		schemas := tests[filename]
 		for _, schema := range schemas {
 			if schema.Skip[version] != "" {
-				fmt.Printf("%s: schema fail (%s)\n", testdataPos(schema), schema.Description)
+				fmt.Fprintf(outw, "%s: schema fail (%s)\n", testdataPos(schema), schema.Description)
 				continue
 			}
 			for _, test := range schema.Tests {
@@ -95,7 +109,7 @@ func listFailures(version string, tests map[string][]*externaltest.Schema) {
 					if !test.Valid {
 						reason = "unexpected success"
 					}
-					fmt.Printf("%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description)
+					fmt.Fprintf(outw, "%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description)
 				}
 			}
 		}