minor fixes/improvements

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>

go/test/endtoend/vtgate/queries/random/query_gen.go

@@ -30,10 +30,13 @@ import (
 
 // this file contains the structs and functions to generate random queries
 
+// to test only a particular type of query, delete the corresponding testFailingQueries clause
+// there should be a comment indicating the type of query being disabled
 // if true then known failing query types are still generated by randomQuery()
 const testFailingQueries = false
 
 type (
+	// selectGenerator generates select statements
 	selectGenerator struct {
 		r            *rand.Rand
 		genConfig    sqlparser.ExprGeneratorConfig
@@ -43,16 +46,20 @@ type (
 		schemaTables []tableT
 		sel          *sqlparser.Select
 	}
+
+	// queryGenerator generates queries, which can either be unions or select statements
 	queryGenerator struct {
 		stmt   sqlparser.SelectStatement
 		selGen *selectGenerator
 	}
+
 	column struct {
 		name string
 		// TODO: perhaps remove tableName and always pass columns through a tableT
 		tableName string
 		typ       string
 	}
+
 	tableT struct {
 		// the tableT struct can be used to represent the schema of a table or a derived table
 		// in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
@@ -172,6 +179,7 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 }
 
 // Generate generates a subquery based on sg
+// TODO: currently unused; generate random expressions with union
 func (sg *selectGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	var schemaTablesCopy []tableT
 	for _, tbl := range sg.schemaTables {
@@ -184,8 +192,6 @@ func (sg *selectGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenera
 	return &sqlparser.Subquery{Select: newSG.selGen.sel}
 }
 
-func (sg *selectGenerator) IsQueryGenerator() {}
-
 // Generate generates a subquery based on qg
 func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	var schemaTablesCopy []tableT
@@ -199,7 +205,8 @@ func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenerat
 	return &sqlparser.Subquery{Select: newQG.stmt}
 }
 
-func (qg *queryGenerator) IsQueryGenerator() {}
+func (sg *selectGenerator) IsQueryGenerator() {}
+func (qg *queryGenerator) IsQueryGenerator()  {}
 
 func (qg *queryGenerator) randomQuery() {
 	if qg.selGen.r.Intn(10) < 1 && testFailingQueries {
@@ -210,6 +217,7 @@ func (qg *queryGenerator) randomQuery() {
 	}
 }
 
+// createUnion creates a simple UNION or UNION ALL; no LIMIT or ORDER BY
 func (qg *queryGenerator) createUnion() {
 	union := &sqlparser.Union{}
 
@@ -263,12 +271,12 @@ func (sg *selectGenerator) randomSelect() {
 
 		// having
 		isHaving := sg.r.Intn(2) < 1
+		// TODO: having creates a lot of results mismatched
 		if isHaving && testFailingQueries {
 			sg.createHavingPredicates(grouping)
 		}
 
 		// alias the grouping columns
-		// TODO: alias the grouping columns after having because vitess doesn't recognize aliases in the HAVING
 		grouping = sg.aliasGroupingColumns(grouping)
 
 		// aggregation columns
@@ -282,23 +290,19 @@ func (sg *selectGenerator) randomSelect() {
 	// where
 	sg.createWherePredicates(tables)
 
-	var f func() sqlparser.Expr
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
 	isRandomExpr := sg.r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
+	var exprGenerators []sqlparser.ExprGenerator
 	if canAggregate && testFailingQueries {
-		exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
+		exprGenerators = slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 		// add scalar subqueries
-		if sg.r.Intn(10) < 1 && testFailingQueries {
+		if sg.r.Intn(10) < 1 {
 			exprGenerators = append(exprGenerators, sg)
 		}
-
-		f = func() sqlparser.Expr { return sg.getRandomExpr(exprGenerators...) }
-	} else {
-		f = func() sqlparser.Expr { return sg.getRandomExpr() }
 	}
 
 	// make sure we have at least one select expression
@@ -309,7 +313,7 @@ func (sg *selectGenerator) randomSelect() {
 		// TODO: but we cannot do this for int literals,
 		// TODO: so we loop until we get a non-int-literal random expression
 		// TODO: this is necessary because grouping by the alias (crandom0) currently fails on vitess
-		randomExpr := f()
+		randomExpr := sg.getRandomExpr(exprGenerators...)
 		literal, ok := randomExpr.(*sqlparser.Literal)
 		isIntLiteral := ok && literal.Type == sqlparser.IntVal
 		if isIntLiteral && canAggregate {
@@ -345,22 +349,22 @@ func (sg *selectGenerator) randomSelect() {
 		sg.createLimit()
 	}
 
+	// this makes sure the query generated has the correct number of columns (sg.selGen.genConfig.numCols)
 	newTable = sg.matchNumCols(tables, newTable, canAggregate)
 
 	// add new table to schemaTables
 	newTable.tableExpr = sqlparser.NewDerivedTable(false, sg.sel)
 	sg.schemaTables = append(sg.schemaTables, newTable)
 
 	// derived tables (partially unsupported)
-	// TODO: derived tables fails a lot
 	if sg.r.Intn(10) < 1 {
 		sg.randomSelect()
 	}
 }
 
 func (sg *selectGenerator) createTablesAndJoin() ([]tableT, bool) {
 	var tables []tableT
-	// add at least one of original emp/dept tables for now because derived tables have nil columns
+	// add at least one of original emp/dept tables
 	tables = append(tables, sg.schemaTables[sg.r.Intn(2)])
 
 	tables[0].setAlias("tbl0")
@@ -412,14 +416,15 @@ func (sg *selectGenerator) createJoinPredicates(tables []tableT) sqlparser.Exprs
 
 	exprGenerators := []sqlparser.ExprGenerator{&tables[len(tables)-2], &tables[len(tables)-1]}
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
 
 	return sg.createRandomExprs(1, 3, exprGenerators...)
 }
 
-// returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
+// returns the grouping columns as []column
 func (sg *selectGenerator) createGroupBy(tables []tableT) (grouping []column) {
 	if sg.maxGBs <= 0 {
 		return
@@ -467,6 +472,7 @@ func (sg *selectGenerator) aliasGroupingColumns(grouping []column) []column {
 func (sg *selectGenerator) createAggregations(tables []tableT) (aggregates []column) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -507,6 +513,7 @@ func (sg *selectGenerator) createOrderBy() {
 func (sg *selectGenerator) createWherePredicates(tables []tableT) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -519,6 +526,7 @@ func (sg *selectGenerator) createWherePredicates(tables []tableT) {
 func (sg *selectGenerator) createHavingPredicates(grouping []column) {
 	exprGenerators := slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -547,7 +555,13 @@ func (sg *selectGenerator) createRandomExprs(minExprs, maxExprs int, generators
 
 // getRandomExpr returns a random expression
 func (sg *selectGenerator) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
-	g := sqlparser.NewGenerator(sg.r, 2, generators...)
+	var g *sqlparser.Generator
+	if generators == nil {
+		g = sqlparser.NewGenerator(sg.r, 2)
+	} else {
+		g = sqlparser.NewGenerator(sg.r, 2, generators...)
+	}
+
 	return g.Expression(sg.genConfig.SingleRowConfig().SetNumCols(1))
 }
 
@@ -567,7 +581,7 @@ func (sg *selectGenerator) createLimit() {
 	}
 }
 
-// randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs and returns the column created
+// randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs, and returns the column created
 func (sg *selectGenerator) randomlyAlias(expr sqlparser.Expr, alias string) column {
 	var col column
 	if sg.r.Intn(2) < 1 {

go/test/endtoend/vtgate/queries/random/random_test.go

@@ -79,7 +79,7 @@ func helperTest(t *testing.T, query string) {
 }
 
 func TestMustFix(t *testing.T) {
-	//t.Skip("Skip CI")
+	t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
@@ -154,14 +154,17 @@ func TestMustFix(t *testing.T) {
 }
 
 func TestKnownFailures(t *testing.T) {
-	//t.Skip("Skip CI")
+	t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// column 'tbl1.`not exists (select 1 from dual)`' not found
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.`not exists (select 1 from dual)`, count(*) from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ not exists (select 1 from dual) from dept as tbl0 where tbl0.dname) as tbl1 group by tbl0.deptno, tbl1.`not exists (select 1 from dual)`")
+
 	// VT13001: [BUG] failed to find the corresponding column
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
@@ -259,6 +262,8 @@ func TestKnownFailures(t *testing.T) {
 }
 
 func TestRandom(t *testing.T) {
+	// t.Skip("Skip CI; random expressions generate too many failures to properly limit")
+
 	mcmp, closer := start(t)
 	defer closer()
 

go/test/endtoend/vtgate/queries/random/simplifier_test.go

@@ -33,7 +33,8 @@ import (
 )
 
 func TestSimplifyResultsMismatchedQuery(t *testing.T) {
-	// t.Skip("Skip CI")
+	t.Skip("Skip CI")
+
 	var queries []string
 	queries = append(queries, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1",
 		"select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'",
@@ -69,7 +70,6 @@ func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 	}
 }
 
-// TODO: suppress output from comparing intermediate simplified results
 // given a query that errors with results mismatched, simplifyResultsMismatchedQuery returns a simpler version with the same error
 func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
 	t.Helper()