Merge pull request #2141 from mabel-dev/#2104

#2104

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 909
+__build__ = 910
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/operators/async_read_node.py

@@ -168,6 +168,7 @@ def execute(self, morsel, **kwargs) -> Generator:
                     # due to a read-after-free type error
                     start = time.monotonic_ns()
                     blob_bytes = self.pool.read_and_release(reference, zero_copy=False)
+                    self.statistics.bytes_read += len(blob_bytes)
                     decoded = decoder(
                         blob_bytes, projection=self.columns, selection=self.predicates
                     )
@@ -193,6 +194,7 @@ def execute(self, morsel, **kwargs) -> Generator:
 
                 self.statistics.blobs_read += 1
                 self.statistics.rows_read += morsel.num_rows
+                self.statistics.bytes_processed += morsel.nbytes
 
                 yield morsel
             except Exception as err:

opteryx/operators/function_dataset_node.py

@@ -143,5 +143,7 @@ def execute(self, morsel, **kwargs) -> Generator:
             table = data
 
         self.statistics.columns_read += len(table.column_names)
+        self.statistics.rows_read += table.num_rows
+        self.statistics.bytes_processed += table.nbytes
 
         yield table

opteryx/operators/read_node.py

@@ -222,6 +222,7 @@ def execute(self, morsel, **kwargs) -> Generator:
             self.statistics.time_reading_blobs += time.monotonic_ns() - start_clock
             self.statistics.blobs_read += 1
             self.statistics.rows_read += morsel.num_rows
+            self.statistics.bytes_processed += morsel.nbytes
             yield morsel
             start_clock = time.monotonic_ns()
         if morsel: