Add processedStrides and processedSplits metrics (#264)

Author: rui-mo

velox/connectors/hive/HiveConnector.cpp

@@ -547,6 +547,7 @@ void HiveDataSource::addSplit(std::shared_ptr<ConnectorSplit> split) {
     runtimeStats_.skippedSplitBytes += split_->length;
     return;
   }
+  ++runtimeStats_.processedSplits;
 
   auto fileType = reader_->rowType();
 

velox/dwio/common/Statistics.h

@@ -517,18 +517,26 @@ struct RuntimeStatistics {
   // Number of splits skipped based on statistics.
   int64_t skippedSplits{0};
 
+  // Number of splits processed based on statistics.
+  int64_t processedSplits{0};
+
   // Total bytes in splits skipped based on statistics.
   int64_t skippedSplitBytes{0};
 
   // Number of strides (row groups) skipped based on statistics.
   int64_t skippedStrides{0};
 
+  // Number of strides (row groups) processed based on statistics.
+  int64_t processedStrides{0};
+
   std::unordered_map<std::string, RuntimeCounter> toMap() {
     return {
         {"skippedSplits", RuntimeCounter(skippedSplits)},
+        {"processedSplits", RuntimeCounter(processedSplits)},
         {"skippedSplitBytes",
          RuntimeCounter(skippedSplitBytes, RuntimeCounter::Unit::kBytes)},
-        {"skippedStrides", RuntimeCounter(skippedStrides)}};
+        {"skippedStrides", RuntimeCounter(skippedStrides)},
+        {"processedStrides", RuntimeCounter(processedStrides)}};
   }
 };
 

velox/dwio/parquet/reader/ParquetReader.cpp

@@ -612,6 +612,7 @@ bool ParquetRowReader::advanceToNextRowGroup() {
 void ParquetRowReader::updateRuntimeStats(
     dwio::common::RuntimeStatistics& stats) const {
   stats.skippedStrides += skippedRowGroups_;
+  stats.processedStrides += rowGroupIds_.size();
 }
 
 void ParquetRowReader::resetFilterCaches() {

velox/dwio/parquet/reader/TimestampColumnReader.h

@@ -29,9 +29,6 @@ class TimestampColumnReader : public IntegerColumnReader {
       common::ScanSpec& scanSpec)
       : IntegerColumnReader(nodeType, nodeType, params, scanSpec) {}
 
-  static constexpr int64_t JULIAN_TO_UNIX_EPOCH_DAYS = 2440588LL;
-  static constexpr int64_t SECONDS_PER_DAY = 86400LL;
-
   void read(
       vector_size_t offset,
       RowSet rows,

velox/exec/tests/PrintPlanWithStatsTest.cpp

@@ -185,6 +185,8 @@ TEST_F(PrintPlanWithStatsTest, innerJoinWithTableScan) {
        {"          prefetchBytes       [ ]* sum: .+, count: 1, min: .+, max: .+"},
        {"          preloadedSplits[ ]+sum: .+, count: .+, min: .+, max: .+",
         true},
+       {"          processedSplits     [ ]* sum: 1, count: 1, min: 1, max: 1"},
+       {"          processedStrides    [ ]* sum: 0, count: 1, min: 0, max: 0"},
        {"          queryThreadIoLatency[ ]* sum: .+, count: .+ min: .+, max: .+"},
        {"          ramReadBytes        [ ]* sum: .+, count: 1, min: .+, max: .+"},
        {"          readyPreloadedSplits[ ]+sum: .+, count: .+, min: .+, max: .+",
@@ -257,6 +259,8 @@ TEST_F(PrintPlanWithStatsTest, partialAggregateWithTableScan) {
          {"        numRamRead       [ ]* sum: 6, count: 1, min: 6, max: 6"},
          {"        numStorageRead   [ ]* sum: .+, count: 1, min: .+, max: .+"},
          {"        prefetchBytes    [ ]* sum: .+, count: 1, min: .+, max: .+"},
+         {"        processedSplits  [ ]* sum: 1, count: 1, min: 1, max: 1"},
+         {"        processedStrides [ ]* sum: 0, count: 1, min: 0, max: 0"},
          {"        preloadedSplits[ ]+sum: .+, count: .+, min: .+, max: .+",
           true},
          {"        queryThreadIoLatency[ ]* sum: .+, count: .+ min: .+, max: .+"},