For nested features with N nested levels (N > 1), the statistics counting the number of values in CommonStatistics and WeightedCommonStatistics will rely on the innermost level.

tfx-copybara · tfx-copybara · commit 9cfefc250c7b · 2024-05-06T19:57:44.000-07:00
PiperOrigin-RevId: 631265288
diff --git a/RELEASE.md b/RELEASE.md
@@ -54,10 +54,13 @@
     time, any M1 support for TFDV is experimental and untested.
 *   Bumps the pybind11 version to 2.11.1.
 *   Depends on `tensorflow~=2.15.0`.
-*   Depends on `apache-beam[gcp]>=2.53.0,<3` for Python 3.11 and on 
+*   Depends on `apache-beam[gcp]>=2.53.0,<3` for Python 3.11 and on
     `apache-beam[gcp]>=2.47.0,<3` for 3.9 and 3.10.
 *   Depends on `protobuf>=4.25.2,<5` for Python 3.11 and on `protobuf>3.20.3,<5`
     for 3.9 and 3.10.
+*   For nested features with N nested levels (N > 1), the statistics counting
+    the number of values in `CommonStatistics` and `WeightedCommonStatistics`
+    will rely on the innermost level.
 
 ## Known Issues
 
diff --git a/tensorflow_data_validation/anomalies/schema_anomalies_test.cc b/tensorflow_data_validation/anomalies/schema_anomalies_test.cc
@@ -214,7 +214,7 @@ TEST(SchemaAnomalies, SimpleBadSchemaConfigurations) {
               num_non_missing: 4
               min_num_values: 1
               max_num_values: 3
-              avg_num_values: 1.5
+              avg_num_values: 2
               presence_and_valency_stats {
                 num_missing: 10
                 num_non_missing: 4
diff --git a/tensorflow_data_validation/anomalies/schema_test.cc b/tensorflow_data_validation/anomalies/schema_test.cc
@@ -1933,8 +1933,8 @@ TEST(SchemaTest, GetSchemaWithValueCounts) {
               num_missing: 10
               min_num_values: 1
               max_num_values: 1
-              avg_num_values: 1.0
-              tot_num_values: 10
+              avg_num_values: 1.5
+              tot_num_values: 15
               presence_and_valency_stats {
                 num_non_missing: 10
                 num_missing: 10
@@ -2383,7 +2383,7 @@ TEST(SchemaTest, UpdateBadStartingSchema) {
               num_non_missing: 4
               min_num_values: 1
               max_num_values: 3
-              avg_num_values: 1.5
+              avg_num_values: 2
               presence_and_valency_stats {
                 num_missing: 10
                 num_non_missing: 4
diff --git a/tensorflow_data_validation/anomalies/statistics_view_test.cc b/tensorflow_data_validation/anomalies/statistics_view_test.cc
@@ -331,7 +331,7 @@ TEST(FeatureStatsView, GetNumMissingNestedMultipleNestednessLevels) {
         type: FLOAT
         num_stats: {
           common_stats {
-            num_missing: 3
+            num_missing: 2
             weighted_common_stats { num_missing: 2 }
             presence_and_valency_stats { num_missing: 2 }
             presence_and_valency_stats { num_missing: 0 }
diff --git a/tensorflow_data_validation/integration_tests/sequence_example_e2e_test.py b/tensorflow_data_validation/integration_tests/sequence_example_e2e_test.py
@@ -416,28 +416,28 @@
       common_stats {
         num_non_missing: 10
         num_missing: 10
-        min_num_values: 1
-        max_num_values: 1
-        avg_num_values: 1.0
+        min_num_values: 2
+        max_num_values: 2
+        avg_num_values: 2.0
         num_values_histogram {
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           type: QUANTILES
         }
-        tot_num_values: 10
+        tot_num_values: 20
         presence_and_valency_stats {
           num_non_missing: 10
           num_missing: 10
@@ -511,17 +511,17 @@
     num_stats {
       common_stats {
         num_non_missing: 20
-        min_num_values: 1
+        min_num_values: 2
         max_num_values: 3
-        avg_num_values: 2.0
+        avg_num_values: 2.5
         num_values_histogram {
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 10.0
           }
           buckets {
-            low_value: 1.0
+            low_value: 2.0
             high_value: 3.0
             sample_count: 5.0
           }
@@ -532,7 +532,7 @@
           }
           type: QUANTILES
         }
-        tot_num_values: 40
+        tot_num_values: 50
         presence_and_valency_stats {
           num_non_missing: 20
           min_num_values: 1
@@ -1026,34 +1026,34 @@
       common_stats {
         num_non_missing: 10
         num_missing: 10
-        min_num_values: 1
-        max_num_values: 1
-        avg_num_values: 1.0
+        min_num_values: 2
+        max_num_values: 2
+        avg_num_values: 2.0
         num_values_histogram {
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 3.3333333333333335
           }
           type: QUANTILES
         }
         weighted_common_stats {
           num_non_missing: 50.0
           num_missing: 100.0
-          avg_num_values: 1.0
-          tot_num_values: 50.0
+          avg_num_values: 2.0
+          tot_num_values: 100.0
         }
-        tot_num_values: 10
+        tot_num_values: 20
         presence_and_valency_stats {
           num_non_missing: 10
           num_missing: 10
@@ -1157,17 +1157,17 @@
     num_stats {
       common_stats {
         num_non_missing: 20
-        min_num_values: 1
+        min_num_values: 2
         max_num_values: 3
-        avg_num_values: 2.0
+        avg_num_values: 2.5
         num_values_histogram {
           buckets {
-            low_value: 1.0
-            high_value: 1.0
+            low_value: 2.0
+            high_value: 2.0
             sample_count: 10.0
           }
           buckets {
-            low_value: 1.0
+            low_value: 2.0
             high_value: 3.0
             sample_count: 5.0
           }
@@ -1180,10 +1180,10 @@
         }
         weighted_common_stats {
           num_non_missing: 150.0
-          avg_num_values: 1.6666666666666667
-          tot_num_values: 250.0
+          avg_num_values: 2.6666666666666667
+          tot_num_values: 400.0
         }
-        tot_num_values: 40
+        tot_num_values: 50
         presence_and_valency_stats {
           num_non_missing: 20
           min_num_values: 1
diff --git a/tensorflow_data_validation/statistics/generators/basic_stats_generator.py b/tensorflow_data_validation/statistics/generators/basic_stats_generator.py
diff --git a/tensorflow_data_validation/statistics/generators/basic_stats_generator_test.py b/tensorflow_data_validation/statistics/generators/basic_stats_generator_test.py
