#2104

mabel-dev · Dec 22, 2024 · a4322cc · a4322cc
1 parent 1cc86ec
commit a4322cc
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 0 deletions.
diff --git a/opteryx/operators/async_read_node.py b/opteryx/operators/async_read_node.py
@@ -168,6 +168,7 @@ def execute(self, morsel, **kwargs) -> Generator:
                     # due to a read-after-free type error
                     start = time.monotonic_ns()
                     blob_bytes = self.pool.read_and_release(reference, zero_copy=False)
+                    self.statistics.bytes_read += len(blob_bytes)
                     decoded = decoder(
                         blob_bytes, projection=self.columns, selection=self.predicates
                     )
@@ -193,6 +194,7 @@ def execute(self, morsel, **kwargs) -> Generator:
 
                 self.statistics.blobs_read += 1
                 self.statistics.rows_read += morsel.num_rows
+                self.statistics.bytes_processed += morsel.nbytes
 
                 yield morsel
             except Exception as err:

diff --git a/opteryx/operators/function_dataset_node.py b/opteryx/operators/function_dataset_node.py
@@ -143,5 +143,7 @@ def execute(self, morsel, **kwargs) -> Generator:
             table = data
 
         self.statistics.columns_read += len(table.column_names)
+        self.statistics.rows_read += table.num_rows
+        self.statistics.bytes_processed += table.nbytes
 
         yield table
diff --git a/opteryx/operators/read_node.py b/opteryx/operators/read_node.py
@@ -222,6 +222,7 @@ def execute(self, morsel, **kwargs) -> Generator:
             self.statistics.time_reading_blobs += time.monotonic_ns() - start_clock
             self.statistics.blobs_read += 1
             self.statistics.rows_read += morsel.num_rows
+            self.statistics.bytes_processed += morsel.nbytes
             yield morsel
             start_clock = time.monotonic_ns()
         if morsel: