Merge branch 'master' into phoenix-test-run

.github/workflows/ci.yml

@@ -878,7 +878,6 @@ jobs:
             - suite-7-non-generic
             - suite-hive-transactional
             - suite-azure
-            - suite-delta-lake-databricks91
             - suite-delta-lake-databricks104
             - suite-delta-lake-databricks113
             - suite-delta-lake-databricks122
@@ -919,9 +918,6 @@ jobs:
               ignore exclusion if: >-
                 ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }}
 
-            - suite: suite-delta-lake-databricks91
-              ignore exclusion if: >-
-                ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
             - suite: suite-delta-lake-databricks104
               ignore exclusion if: >-
                 ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
@@ -989,7 +985,6 @@ jobs:
           AWS_REGION: ""
           TRINO_AWS_ACCESS_KEY_ID: ""
           TRINO_AWS_SECRET_ACCESS_KEY: ""
-          DATABRICKS_91_JDBC_URL: ""
           DATABRICKS_104_JDBC_URL: ""
           DATABRICKS_113_JDBC_URL: ""
           DATABRICKS_122_JDBC_URL: ""
@@ -1067,7 +1062,6 @@ jobs:
           AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
           TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
           TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
-          DATABRICKS_91_JDBC_URL: ${{ vars.DATABRICKS_91_JDBC_URL }}
           DATABRICKS_104_JDBC_URL: ${{ vars.DATABRICKS_104_JDBC_URL }}
           DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }}
           DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }}

client/trino-client/src/main/java/io/trino/client/StatementStats.java

@@ -36,14 +36,18 @@ public class StatementStats
     private final int queuedSplits;
     private final int runningSplits;
     private final int completedSplits;
+    private final long planningTimeMillis;
+    private final long analysisTimeMillis;
     private final long cpuTimeMillis;
     private final long wallTimeMillis;
     private final long queuedTimeMillis;
     private final long elapsedTimeMillis;
+    private final long finishingTimeMillis;
     private final long processedRows;
     private final long processedBytes;
     private final long physicalInputBytes;
     private final long physicalWrittenBytes;
+    private final long internalNetworkInputBytes;
     private final long peakMemoryBytes;
     private final long spilledBytes;
     private final StageStats rootStage;
@@ -60,14 +64,18 @@ public StatementStats(
             @JsonProperty("queuedSplits") int queuedSplits,
             @JsonProperty("runningSplits") int runningSplits,
             @JsonProperty("completedSplits") int completedSplits,
+            @JsonProperty("planningTimeMillis") long planningTimeMillis,
+            @JsonProperty("analysisTimeMillis") long analysisTimeMillis,
             @JsonProperty("cpuTimeMillis") long cpuTimeMillis,
             @JsonProperty("wallTimeMillis") long wallTimeMillis,
             @JsonProperty("queuedTimeMillis") long queuedTimeMillis,
             @JsonProperty("elapsedTimeMillis") long elapsedTimeMillis,
+            @JsonProperty("finishingTimeMillis") long finishingTimeMillis,
             @JsonProperty("processedRows") long processedRows,
             @JsonProperty("processedBytes") long processedBytes,
             @JsonProperty("physicalInputBytes") long physicalInputBytes,
             @JsonProperty("physicalWrittenBytes") long physicalWrittenBytes,
+            @JsonProperty("internalNetworkInputBytes") long internalNetworkInputBytes,
             @JsonProperty("peakMemoryBytes") long peakMemoryBytes,
             @JsonProperty("spilledBytes") long spilledBytes,
             @JsonProperty("rootStage") StageStats rootStage)
@@ -82,14 +90,18 @@ public StatementStats(
         this.queuedSplits = queuedSplits;
         this.runningSplits = runningSplits;
         this.completedSplits = completedSplits;
+        this.planningTimeMillis = planningTimeMillis;
+        this.analysisTimeMillis = analysisTimeMillis;
         this.cpuTimeMillis = cpuTimeMillis;
         this.wallTimeMillis = wallTimeMillis;
         this.queuedTimeMillis = queuedTimeMillis;
         this.elapsedTimeMillis = elapsedTimeMillis;
+        this.finishingTimeMillis = finishingTimeMillis;
         this.processedRows = processedRows;
         this.processedBytes = processedBytes;
         this.physicalInputBytes = physicalInputBytes;
         this.physicalWrittenBytes = physicalWrittenBytes;
+        this.internalNetworkInputBytes = internalNetworkInputBytes;
         this.peakMemoryBytes = peakMemoryBytes;
         this.spilledBytes = spilledBytes;
         this.rootStage = rootStage;
@@ -155,6 +167,18 @@ public int getCompletedSplits()
         return completedSplits;
     }
 
+    @JsonProperty
+    public long getPlanningTimeMillis()
+    {
+        return planningTimeMillis;
+    }
+
+    @JsonProperty
+    public long getAnalysisTimeMillis()
+    {
+        return analysisTimeMillis;
+    }
+
     @JsonProperty
     public long getCpuTimeMillis()
     {
@@ -179,6 +203,12 @@ public long getElapsedTimeMillis()
         return elapsedTimeMillis;
     }
 
+    @JsonProperty
+    public long getFinishingTimeMillis()
+    {
+        return finishingTimeMillis;
+    }
+
     @JsonProperty
     public long getProcessedRows()
     {
@@ -203,6 +233,12 @@ public long getPhysicalWrittenBytes()
         return physicalWrittenBytes;
     }
 
+    @JsonProperty
+    public long getInternalNetworkInputBytes()
+    {
+        return internalNetworkInputBytes;
+    }
+
     @JsonProperty
     public long getPeakMemoryBytes()
     {
@@ -236,14 +272,18 @@ public String toString()
                 .add("queuedSplits", queuedSplits)
                 .add("runningSplits", runningSplits)
                 .add("completedSplits", completedSplits)
+                .add("planningTimeMillis", planningTimeMillis)
+                .add("analysisTimeMillis", analysisTimeMillis)
                 .add("cpuTimeMillis", cpuTimeMillis)
                 .add("wallTimeMillis", wallTimeMillis)
                 .add("queuedTimeMillis", queuedTimeMillis)
                 .add("elapsedTimeMillis", elapsedTimeMillis)
+                .add("finishingTimeMillis", finishingTimeMillis)
                 .add("processedRows", processedRows)
                 .add("processedBytes", processedBytes)
                 .add("physicalInputBytes", physicalInputBytes)
                 .add("physicalWrittenBytes", physicalWrittenBytes)
+                .add("internalNetworkInputBytes", internalNetworkInputBytes)
                 .add("peakMemoryBytes", peakMemoryBytes)
                 .add("spilledBytes", spilledBytes)
                 .add("rootStage", rootStage)
@@ -267,14 +307,18 @@ public static class Builder
         private int queuedSplits;
         private int runningSplits;
         private int completedSplits;
+        private long planningTimeMillis;
+        private long analysisTimeMillis;
         private long cpuTimeMillis;
         private long wallTimeMillis;
         private long queuedTimeMillis;
         private long elapsedTimeMillis;
+        private long finishingTimeMillis;
         private long processedRows;
         private long processedBytes;
         private long physicalInputBytes;
         private long physicalWrittenBytes;
+        private long internalNetworkInputBytes;
         private long peakMemoryBytes;
         private long spilledBytes;
         private StageStats rootStage;
@@ -341,6 +385,18 @@ public Builder setCompletedSplits(int completedSplits)
             return this;
         }
 
+        public Builder setPlanningTimeMillis(long planningTimeMillis)
+        {
+            this.planningTimeMillis = planningTimeMillis;
+            return this;
+        }
+
+        public Builder setAnalysisTimeMillis(long analysisTimeMillis)
+        {
+            this.analysisTimeMillis = analysisTimeMillis;
+            return this;
+        }
+
         public Builder setCpuTimeMillis(long cpuTimeMillis)
         {
             this.cpuTimeMillis = cpuTimeMillis;
@@ -365,6 +421,12 @@ public Builder setElapsedTimeMillis(long elapsedTimeMillis)
             return this;
         }
 
+        public Builder setFinishingTimeMillis(long finishingTimeMillis)
+        {
+            this.finishingTimeMillis = finishingTimeMillis;
+            return this;
+        }
+
         public Builder setProcessedRows(long processedRows)
         {
             this.processedRows = processedRows;
@@ -389,6 +451,12 @@ public Builder setPhysicalWrittenBytes(long physicalWrittenBytes)
             return this;
         }
 
+        public Builder setInternalNetworkInputBytes(long internalNetworkInputBytes)
+        {
+            this.internalNetworkInputBytes = internalNetworkInputBytes;
+            return this;
+        }
+
         public Builder setPeakMemoryBytes(long peakMemoryBytes)
         {
             this.peakMemoryBytes = peakMemoryBytes;
@@ -420,14 +488,18 @@ public StatementStats build()
                     queuedSplits,
                     runningSplits,
                     completedSplits,
+                    planningTimeMillis,
+                    analysisTimeMillis,
                     cpuTimeMillis,
                     wallTimeMillis,
                     queuedTimeMillis,
                     elapsedTimeMillis,
+                    finishingTimeMillis,
                     processedRows,
                     processedBytes,
                     physicalInputBytes,
                     physicalWrittenBytes,
+                    internalNetworkInputBytes,
                     peakMemoryBytes,
                     spilledBytes,
                     rootStage);

client/trino-client/src/test/java/io/trino/client/TestRetry.java

@@ -143,7 +143,7 @@ private String newQueryResults(String state)
                 TypedQueryData.of(IntStream.range(0, numRecords)
                         .mapToObj(index -> Stream.of((Object) index, "a").collect(toList()))
                         .collect(toList())),
-                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
+                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
                 null,
                 ImmutableList.of(),
                 null,

client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java

@@ -32,13 +32,19 @@ public final class QueryStats
     private final int queuedSplits;
     private final int runningSplits;
     private final int completedSplits;
+    private final long planningTimeMillis;
+    private final long analysisTimeMillis;
     private final long cpuTimeMillis;
     private final long wallTimeMillis;
     private final long queuedTimeMillis;
     private final long elapsedTimeMillis;
+    private final long finishingTimeMillis;
     private final long processedRows;
     private final long processedBytes;
     private final long peakMemoryBytes;
+    private final long physicalInputBytes;
+    private final long physicalWrittenBytes;
+    private final long internalNetworkInputBytes;
     private final Optional<StageStats> rootStage;
 
     public QueryStats(
@@ -52,13 +58,19 @@ public QueryStats(
             int queuedSplits,
             int runningSplits,
             int completedSplits,
+            long planningTimeMillis,
+            long analysisTimeMillis,
             long cpuTimeMillis,
             long wallTimeMillis,
             long queuedTimeMillis,
             long elapsedTimeMillis,
+            long finishingTimeMillis,
             long processedRows,
             long processedBytes,
             long peakMemoryBytes,
+            long physicalInputBytes,
+            long physicalWrittenBytes,
+            long internalNetworkInputBytes,
             Optional<StageStats> rootStage)
     {
         this.queryId = requireNonNull(queryId, "queryId is null");
@@ -71,13 +83,19 @@ public QueryStats(
         this.queuedSplits = queuedSplits;
         this.runningSplits = runningSplits;
         this.completedSplits = completedSplits;
+        this.planningTimeMillis = planningTimeMillis;
+        this.analysisTimeMillis = analysisTimeMillis;
         this.cpuTimeMillis = cpuTimeMillis;
         this.wallTimeMillis = wallTimeMillis;
         this.queuedTimeMillis = queuedTimeMillis;
         this.elapsedTimeMillis = elapsedTimeMillis;
+        this.finishingTimeMillis = finishingTimeMillis;
         this.processedRows = processedRows;
         this.processedBytes = processedBytes;
         this.peakMemoryBytes = peakMemoryBytes;
+        this.physicalInputBytes = physicalInputBytes;
+        this.physicalWrittenBytes = physicalWrittenBytes;
+        this.internalNetworkInputBytes = internalNetworkInputBytes;
         this.rootStage = requireNonNull(rootStage, "rootStage is null");
     }
 
@@ -94,13 +112,19 @@ static QueryStats create(String queryId, StatementStats stats)
                 stats.getQueuedSplits(),
                 stats.getRunningSplits(),
                 stats.getCompletedSplits(),
+                stats.getPlanningTimeMillis(),
+                stats.getAnalysisTimeMillis(),
                 stats.getCpuTimeMillis(),
                 stats.getWallTimeMillis(),
                 stats.getQueuedTimeMillis(),
                 stats.getElapsedTimeMillis(),
+                stats.getFinishingTimeMillis(),
                 stats.getProcessedRows(),
                 stats.getProcessedBytes(),
                 stats.getPeakMemoryBytes(),
+                stats.getPhysicalInputBytes(),
+                stats.getPhysicalWrittenBytes(),
+                stats.getInternalNetworkInputBytes(),
                 Optional.ofNullable(stats.getRootStage()).map(StageStats::create));
     }
 
@@ -154,6 +178,16 @@ public int getCompletedSplits()
         return completedSplits;
     }
 
+    public long getPlanningTimeMillis()
+    {
+        return planningTimeMillis;
+    }
+
+    public long getAnalysisTimeMillis()
+    {
+        return analysisTimeMillis;
+    }
+
     public long getCpuTimeMillis()
     {
         return cpuTimeMillis;
@@ -174,6 +208,11 @@ public long getElapsedTimeMillis()
         return elapsedTimeMillis;
     }
 
+    public long getFinishingTimeMillis()
+    {
+        return finishingTimeMillis;
+    }
+
     public long getProcessedRows()
     {
         return processedRows;
@@ -189,6 +228,21 @@ public long getPeakMemoryBytes()
         return peakMemoryBytes;
     }
 
+    public long getPhysicalInputBytes()
+    {
+        return physicalInputBytes;
+    }
+
+    public long getPhysicalWrittenBytes()
+    {
+        return physicalWrittenBytes;
+    }
+
+    public long getInternalNetworkInputBytes()
+    {
+        return internalNetworkInputBytes;
+    }
+
     public Optional<StageStats> getRootStage()
     {
         return rootStage;

client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java

@@ -333,6 +333,10 @@ public StatementStats getStats()
                         100,
                         100,
                         100,
+                        100,
+                        100,
+                        100,
+                        100,
                         StageStats.builder()
                                 .setStageId("id")
                                 .setDone(false)

client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java

@@ -99,7 +99,7 @@ private String newQueryResults(Integer partialCancelId, Integer nextUriId, List<
                 nextUriId == null ? null : server.url(format("/v1/statement/%s/%s", queryId, nextUriId)).uri(),
                 responseColumns,
                 TypedQueryData.of(data),
-                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
+                new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
                 null,
                 ImmutableList.of(),
                 null,