[CALCITE-5615] Run SQL Logic Test suite using Calcite's HSQLDB JDBC a…

…dapter

Signed-off-by: Mihai Budiu <mbudiu@vmware.com>

Close apache#3145

bom/build.gradle.kts

@@ -90,6 +90,7 @@ dependencies {
         apiv("net.hydromatic:foodmart-queries")
         apiv("net.hydromatic:quidem")
         apiv("net.hydromatic:scott-data-hsqldb")
+        apiv("net.hydromatic:sql-logic-test")
         apiv("net.hydromatic:tpcds", "hydromatic.tpcds")
         apiv("net.java.dev.jna:jna")
         apiv("net.sf.opencsv:opencsv")

gradle.properties

@@ -159,6 +159,7 @@ sketches-core.version=0.9.0
 slf4j.version=1.7.25
 spark.version=2.2.2
 sqlline.version=1.12.0
+sql-logic-test.version=0.3
 teradata.tpcds.version=1.2
 testcontainers.version=1.17.3
 tpch.version=1.0

plus/build.gradle.kts

@@ -29,6 +29,7 @@ dependencies {
     implementation("org.apache.calcite.avatica:avatica-server")
     implementation("org.hsqldb:hsqldb")
 
+    testImplementation("net.hydromatic:sql-logic-test")
     testImplementation(project(":testkit"))
     testImplementation("org.incava:java-diff")
     testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j-impl")

plus/src/test/java/org/apache/calcite/slt/SqlLogicTests.java

@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.slt;
+
+import org.apache.calcite.slt.executors.CalciteExecutor;
+import org.apache.calcite.util.trace.CalciteTrace;
+
+import com.google.common.collect.ImmutableSet;
+
+import net.hydromatic.sqllogictest.Main;
+import net.hydromatic.sqllogictest.OptionsParser;
+import net.hydromatic.sqllogictest.TestStatistics;
+
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestFactory;
+import org.slf4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+/**
+ * Tests using sql-logic-test suite.
+ *
+ * <p>For each test file the number of failed tests is saved in a "golden" file.
+ * These results are checked in as part of the sltttestfailures.txt resource file.
+ * Currently, there are quite a few errors, so this tool does not track of the actual
+ * errors that were encountered; we expect that, as bugs are fixed in Calcite,
+ * the number of errors will shrink, and a more precise accounting method will be used.
+ *
+ * <p>The tests will fail if any test script generates
+ * more errors than the number from the golden file.
+ */
+public class SqlLogicTests {
+  private static final Logger LOGGER =
+      CalciteTrace.getTestTracer(SqlLogicTests.class);
+
+  /**
+   * Short summary of the results of a test execution.
+   */
+  public static class TestSummary {
+    /**
+     * File containing tests.
+     */
+    final String file;
+    /**
+     * Number of tests that have failed.
+     */
+    final int failed;
+
+    TestSummary(String file, int failed) {
+      this.file = file;
+      this.failed = failed;
+    }
+
+    /**
+     * Parses a TestSummary from a string.
+     * The inverse of 'toString'.
+     *
+     * @return The parsed TestSummary or null on failure.
+     */
+    public static TestSummary parse(String line) {
+      String[] parts = line.split(":");
+      if (parts.length != 2) {
+        return null;
+      }
+      try {
+        int failed = Integer.parseInt(parts[1]);
+        return new TestSummary(parts[0], failed);
+      } catch (NumberFormatException ex) {
+        return null;
+      }
+    }
+
+    @Override public String toString() {
+      return this.file + ":" + this.failed;
+    }
+
+    /**
+     * Check if the 'other' TestSummaries indicate a regressions
+     * when compared to 'this'.
+     *
+     * @param other TestSummary to compare against.
+     * @return 'true' if 'other' is a regression from 'this'.
+     */
+    public boolean isRegression(TestSummary other) {
+      return other.failed > this.failed;
+    }
+  }
+
+  /**
+   * Summary for all tests executed.
+   */
+  public static class AllTestSummaries {
+    /**
+     * Map test summary name to test summary.
+     */
+    final Map<String, TestSummary> testResults;
+
+    AllTestSummaries() {
+      this.testResults = new HashMap<>();
+    }
+
+    void add(TestSummary summary) {
+      this.testResults.put(summary.file, summary);
+    }
+
+    AllTestSummaries read(InputStream stream) {
+      try (BufferedReader reader =
+               new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+        reader.lines().forEach(line -> {
+          TestSummary summary = TestSummary.parse(line);
+          if (summary != null) {
+            this.add(summary);
+          } else {
+            LOGGER.warn("Could not parse line " + line);
+          }
+        });
+        return this;
+      } catch (IOException ex) {
+        // Wrapping the IOException makes it easier to use this method in the
+        // initializer of a static variable.
+        throw new RuntimeException(ex);
+      }
+    }
+
+    boolean regression(TestSummary summary) {
+      TestSummary original = this.testResults.get(summary.file);
+      if (original == null) {
+        LOGGER.warn("No historical data for test " + summary.file);
+        return false;
+      }
+      if (original.isRegression(summary)) {
+        LOGGER.error("Regression: " + original.file
+            + " had " + original.failed + " failures, now has " + summary.failed);
+        return true;
+      }
+      return false;
+    }
+
+    @Override public String toString() {
+      List<TestSummary> results = new ArrayList<>(this.testResults.values());
+      results.sort(Comparator.comparing(left -> left.file));
+      StringBuilder result = new StringBuilder();
+      for (TestSummary summary: results) {
+        result.append(summary.toString());
+        result.append(System.lineSeparator());
+      }
+      return result.toString();
+    }
+
+    /**
+     * Write the test results to the specified file.
+     */
+    public void writeToFile(File file) throws IOException {
+      try (BufferedWriter writer =
+               new BufferedWriter(
+                   new OutputStreamWriter(
+                       Files.newOutputStream(file.toPath()), StandardCharsets.UTF_8))) {
+        writer.write(this.toString());
+      }
+    }
+  }
+
+  /**
+   * Summaries produced for the current run.
+   * Must be static since it is written by the `findRegressions`
+   * static method.
+   */
+  private static final AllTestSummaries SUMMARIES = new AllTestSummaries();
+
+  private static final String GOLDEN_FILE = "/slttestfailures.txt";
+  /**
+   * Summaries checked-in as resources that we compare against.
+   */
+  private static final AllTestSummaries GOLDEN_SUMMARIES =
+      new AllTestSummaries()
+          .read(SqlLogicTests.class.getResourceAsStream(GOLDEN_FILE));
+  /**
+   * The following tests currently timeout during execution.
+   * Technically these are Calcite bugs.
+   */
+  private static final Set<String> TIMEOUT =
+      ImmutableSet.of("test/select5.test",
+          "test/random/groupby/slt_good_10.test");
+
+  /**
+   * The following tests contain SQL statements that are not supported by HSQLDB.
+   */
+  private static final Set<String> UNSUPPORTED =
+      ImmutableSet.of("test/evidence/slt_lang_replace.test",
+          "test/evidence/slt_lang_createtrigger.test",
+          "test/evidence/slt_lang_droptrigger.test",
+          "test/evidence/slt_lang_update.test",
+          "test/evidence/slt_lang_reindex.test");
+
+  private static void runTestFile(String testFile) {
+    Assumptions.assumeFalse(TIMEOUT.contains(testFile), testFile + " currently timeouts");
+    Assumptions.assumeFalse(UNSUPPORTED.contains(testFile),
+        testFile + " contains unsupported statements");
+    OptionsParser options = new OptionsParser(false, nullStream(), nullStream());
+    CalciteExecutor.register(options);
+    TestStatistics res;
+    try {
+      res = Main.execute(options, "-e", "calcite", testFile);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    checkStatsForSingleRun(res);
+    TestSummary summary = new TestSummary(testFile, res.getFailedTestCount());
+    boolean regression = GOLDEN_SUMMARIES.regression(summary);
+    assertFalse(regression, "Regression in " + summary.file);
+    // The following is only useful if a new golden file need to be created
+    SUMMARIES.add(summary);
+  }
+
+  private static PrintStream nullStream() {
+    try {
+      return new PrintStream(new OutputStream() {
+        @Override public void write(final int b) {
+          // Do nothing
+        }
+      }, false, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static void checkStatsForSingleRun(TestStatistics stats) {
+    Objects.requireNonNull(stats, "stats");
+    if (stats.getParseFailureCount() > 0) {
+      throw new IllegalStateException("Failed to parse file");
+    } else if (stats.getIgnoredTestCount() > 0) {
+      throw new IllegalStateException("File was ignored");
+    } else if (stats.getTestFileCount() > 1) {
+      throw new IllegalStateException("Running multiple files not supported");
+    }
+  }
+
+  @TestFactory @Tag("slow")
+  List<DynamicTest> testSlow() {
+    return generateTests(ImmutableSet.of("select1.test"));
+  }
+
+  @TestFactory @Disabled("This takes very long, should be run manually")
+  List<DynamicTest> testAll() {
+    // Run in parallel each test file.  There are 622 of these, each taking
+    // a few minutes.
+    return generateTests(Main.getTestList());
+  }
+
+  /**
+   * Generate a list of all the tests that can be executed.
+   *
+   * @param testFiles Names of files containing tests.
+   */
+  private List<DynamicTest> generateTests(Set<String> testFiles) {
+    List<DynamicTest> result = new ArrayList<>();
+    for (String test: testFiles) {
+      DynamicTest dynamicTest = DynamicTest.dynamicTest(test, () -> runTestFile(test));
+      result.add(dynamicTest);
+    }
+    return result;
+  }
+
+  /**
+   * Create the golden reference file with test results.
+   */
+  public static void createGoldenFile() throws IOException {
+    // Currently this method is not invoked.
+    // It can be used to create a new version of the GOLDEN_FILE
+    // when bugs in Calcite are fixed.  It should be called after
+    // all tests have executed and passed.
+    File file = new File(GOLDEN_FILE);
+    if (!file.exists()) {
+      SUMMARIES.writeToFile(file);
+    }
+  }
+}