Add DRY to baked providers

provider/baked/src/export.rs

@@ -309,7 +309,7 @@ impl BakedExporter {
         marker: DataMarkerInfo,
         stats: Statistics,
         body: TokenStream,
-        dry_body: TokenStream,
+        dry_body: Option<TokenStream>,
         iterable_body: TokenStream,
     ) -> Result<(), DataError> {
         let marker_unqualified = bake_marker(marker).into_iter().last().unwrap().to_string();
@@ -356,6 +356,28 @@ impl BakedExporter {
 
         let maybe_msrv = maybe_msrv();
 
+        let dry = if let Some(dry_body) = dry_body {
+            quote! {
+                ($provider:ty, DRY_IF_RETAIN) => {
+                    #prefixed_macro_ident!($provider);
+                    #dry_body
+                };
+                ($provider:ty, DRY_IF_RETAIN, ITER) => {
+                    #prefixed_macro_ident!($provider);
+                    #dry_body
+                    #iterable_body
+                };
+            }
+        } else {
+            quote! {
+                ($provider:ty, DRY_IF_RETAIN) => {
+                };
+                ($provider:ty, DRY_IF_RETAIN, ITER) => {
+                    #prefixed_macro_ident!($provider, ITER);
+                };
+            }
+        };
+
         self.write_to_file(
             Path::new(&format!("{ident}.rs.data")),
             quote! {
@@ -368,19 +390,11 @@ impl BakedExporter {
                         const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO;
                         #body
                     };
-                    ($provider:ty, DRY) => {
-                        #prefixed_macro_ident!($provider);
-                        #dry_body
-                    };
                     ($provider:ty, ITER) => {
                         #prefixed_macro_ident!($provider);
                         #iterable_body
                     };
-                    ($provider:ty, DRY, ITER) => {
-                        #prefixed_macro_ident!($provider);
-                        #dry_body
-                        #iterable_body
-                    };
+                    #dry
                 }
                 #[doc(inline)]
                 pub use #prefixed_macro_ident as #macro_ident;
@@ -469,7 +483,7 @@ impl DataExporter for BakedExporter {
                 }
             }
         },
-        quote! {
+        Some(quote! {
             #maybe_msrv
             impl icu_provider::DryDataProvider<#marker_bake> for $provider {
                 fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
@@ -480,7 +494,7 @@ impl DataExporter for BakedExporter {
                     }
                 }
             }
-        },
+        }),
         quote! {
             #maybe_msrv
             impl icu_provider::IterableDataProvider<#marker_bake> for $provider {
@@ -491,7 +505,11 @@ impl DataExporter for BakedExporter {
         })
     }
 
-    fn flush(&self, marker: DataMarkerInfo) -> Result<(), DataError> {
+    fn flush(
+        &self,
+        marker: DataMarkerInfo,
+        deduplication_mode: DeduplicationStrategy,
+    ) -> Result<(), DataError> {
         let maybe_msrv = maybe_msrv();
 
         let marker_bake = bake_marker(marker);
@@ -518,14 +536,14 @@ impl DataExporter for BakedExporter {
                         }
                     }
                 },
-                quote! {
+                Some(quote! {
                     #maybe_msrv
                     impl icu_provider::DryDataProvider<#marker_bake> for $provider {
                         fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
                             Err(icu_provider::DataErrorKind::IdentifierNotFound.with_req(<#marker_bake as icu_provider::DataMarker>::INFO, req))
                         }
                     }
-                },
+                }),
                 quote! {
                     #maybe_msrv
                     impl icu_provider::IterableDataProvider<#marker_bake> for $provider {
@@ -633,13 +651,17 @@ impl DataExporter for BakedExporter {
                         }
                     }
                 },
-                quote! {
-                    #maybe_msrv
-                    impl icu_provider::DryDataProvider<#marker_bake> for $provider {
-                        fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
-                            icu_provider::DataProvider::<#marker_bake>::load(self, req).map(|r| r.metadata)
+                if deduplication_mode == DeduplicationStrategy::RetainBaseLanguages {
+                    Some(quote! {
+                        #maybe_msrv
+                        impl icu_provider::DryDataProvider<#marker_bake> for $provider {
+                            fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
+                                icu_provider::DataProvider::<#marker_bake>::load(self, req).map(|r| r.metadata)
+                            }
                         }
-                    }
+                    })
+                } else {
+                    None
                 },
                 quote! {
                     #maybe_msrv

provider/baked/tests/data/hello_world_v1_marker.rs.data

@@ -27,25 +27,25 @@ macro_rules! __impl_hello_world_v1_marker {
             }
         }
     };
-    ($ provider : ty , DRY) => {
+    ($ provider : ty , ITER) => {
         __impl_hello_world_v1_marker!($provider);
         #[clippy::msrv = "1.71.1"]
-        impl icu_provider::DryDataProvider<icu_provider::hello_world::HelloWorldV1Marker> for $provider {
-            fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
-                icu_provider::DataProvider::<icu_provider::hello_world::HelloWorldV1Marker>::load(self, req).map(|r| r.metadata)
+        impl icu_provider::IterableDataProvider<icu_provider::hello_world::HelloWorldV1Marker> for $provider {
+            fn iter_ids(&self) -> Result<std::collections::BTreeSet<icu_provider::DataIdentifierCow<'static>>, icu_provider::DataError> {
+                Ok(icu_provider_baked::DataStore::iter(&Self::DATA_HELLO_WORLD_V1_MARKER).collect())
             }
         }
     };
-    ($ provider : ty , ITER) => {
+    ($ provider : ty , DRY_IF_RETAIN) => {
         __impl_hello_world_v1_marker!($provider);
         #[clippy::msrv = "1.71.1"]
-        impl icu_provider::IterableDataProvider<icu_provider::hello_world::HelloWorldV1Marker> for $provider {
-            fn iter_ids(&self) -> Result<std::collections::BTreeSet<icu_provider::DataIdentifierCow<'static>>, icu_provider::DataError> {
-                Ok(icu_provider_baked::DataStore::iter(&Self::DATA_HELLO_WORLD_V1_MARKER).collect())
+        impl icu_provider::DryDataProvider<icu_provider::hello_world::HelloWorldV1Marker> for $provider {
+            fn dry_load(&self, req: icu_provider::DataRequest) -> Result<icu_provider::DataResponseMetadata, icu_provider::DataError> {
+                icu_provider::DataProvider::<icu_provider::hello_world::HelloWorldV1Marker>::load(self, req).map(|r| r.metadata)
             }
         }
     };
-    ($ provider : ty , DRY , ITER) => {
+    ($ provider : ty , DRY_IF_RETAIN , ITER) => {
         __impl_hello_world_v1_marker!($provider);
         #[clippy::msrv = "1.71.1"]
         impl icu_provider::DryDataProvider<icu_provider::hello_world::HelloWorldV1Marker> for $provider {

provider/baked/tests/test-baked-source.rs

@@ -14,7 +14,7 @@ const _: () = {
     #[path = "data/mod.rs"]
     mod baked_data;
     baked_data::make_provider!(Baked);
-    baked_data::impl_hello_world_v1_marker!(Baked, DRY, ITER);
+    baked_data::impl_hello_world_v1_marker!(Baked, DRY_IF_RETAIN, ITER);
 };
 
 #[test]

provider/blob/benches/auxkey_bench.rs

@@ -267,7 +267,9 @@ where
             )
             .unwrap();
     }
-    exporter.flush(M::INFO).unwrap();
+    exporter
+        .flush(M::INFO, DeduplicationStrategy::None)
+        .unwrap();
 }
 
 fn make_blob_v1() -> Vec<u8> {

provider/blob/src/blob_data_provider.rs

@@ -181,7 +181,9 @@ mod test {
                     BlobExporter::new_v2_with_sink(Box::new(&mut blob))
                 };
 
-                exporter.flush(HelloWorldV1Marker::INFO).unwrap();
+                exporter
+                    .flush(HelloWorldV1Marker::INFO, DeduplicationStrategy::None)
+                    .unwrap();
 
                 exporter.close().unwrap();
             }
@@ -213,7 +215,9 @@ mod test {
                     BlobExporter::new_v2_with_sink(Box::new(&mut blob))
                 };
 
-                exporter.flush(HelloSingletonV1Marker::INFO).unwrap();
+                exporter
+                    .flush(HelloSingletonV1Marker::INFO, DeduplicationStrategy::None)
+                    .unwrap();
 
                 exporter.close().unwrap();
             }

provider/blob/src/export/blob_exporter.rs

@@ -119,7 +119,11 @@ impl DataExporter for BlobExporter<'_> {
         Ok(())
     }
 
-    fn flush(&self, marker: DataMarkerInfo) -> Result<(), DataError> {
+    fn flush(
+        &self,
+        marker: DataMarkerInfo,
+        _deduplication: DeduplicationStrategy,
+    ) -> Result<(), DataError> {
         self.all_markers
             .lock()
             .expect("poison")

provider/blob/tests/test_versions.rs

@@ -34,7 +34,9 @@ where
             )
             .unwrap();
     }
-    exporter.flush(HelloWorldV1Marker::INFO).unwrap();
+    exporter
+        .flush(HelloWorldV1Marker::INFO, DeduplicationStrategy::None)
+        .unwrap();
     exporter.close().unwrap();
 }
 

provider/core/src/export/mod.rs

@@ -17,6 +17,7 @@ use std::collections::HashSet;
 /// An object capable of exporting data payloads in some form.
 pub trait DataExporter: Sync {
     /// Save a `payload` corresponding to the given marker and locale.
+    ///
     /// Takes non-mut self as it can be called concurrently.
     fn put_payload(
         &self,
@@ -26,21 +27,25 @@ pub trait DataExporter: Sync {
     ) -> Result<(), DataError>;
 
     /// Function called for singleton markers.
+    ///
     /// Takes non-mut self as it can be called concurrently.
     fn flush_singleton(
         &self,
         marker: DataMarkerInfo,
         payload: &DataPayload<ExportMarker>,
     ) -> Result<(), DataError> {
         self.put_payload(marker, Default::default(), payload)?;
-        self.flush(marker)
+        self.flush(marker, DeduplicationStrategy::Maximal)
     }
 
     /// Function called after a non-singleton marker has been fully enumerated.
-    /// Does not include built-in fallback.
     ///
     /// Takes non-mut self as it can be called concurrently.
-    fn flush(&self, _marker: DataMarkerInfo) -> Result<(), DataError> {
+    fn flush(
+        &self,
+        _marker: DataMarkerInfo,
+        _deduplication: DeduplicationStrategy,
+    ) -> Result<(), DataError> {
         Ok(())
     }
 
@@ -70,8 +75,12 @@ impl DataExporter for Box<dyn DataExporter> {
         (**self).flush_singleton(marker, payload)
     }
 
-    fn flush(&self, marker: DataMarkerInfo) -> Result<(), DataError> {
-        (**self).flush(marker)
+    fn flush(
+        &self,
+        marker: DataMarkerInfo,
+        deduplication: DeduplicationStrategy,
+    ) -> Result<(), DataError> {
+        (**self).flush(marker, deduplication)
     }
 
     fn close(&mut self) -> Result<(), DataError> {
@@ -176,11 +185,48 @@ impl DataExporter for MultiExporter {
             .try_for_each(|e| e.flush_singleton(marker, payload))
     }
 
-    fn flush(&self, marker: DataMarkerInfo) -> Result<(), DataError> {
-        self.0.iter().try_for_each(|e| e.flush(marker))
+    fn flush(
+        &self,
+        marker: DataMarkerInfo,
+        deduplication: DeduplicationStrategy,
+    ) -> Result<(), DataError> {
+        self.0
+            .iter()
+            .try_for_each(|e| e.flush(marker, deduplication))
     }
 
     fn close(&mut self) -> Result<(), DataError> {
         self.0.iter_mut().try_for_each(|e| e.close())
     }
 }
+
+/// Choices for determining the deduplication of locales for exported data payloads.
+///
+/// Deduplication affects the lookup table from locales to data payloads. If a child locale
+/// points to the same payload as its parent locale, then the child locale can be removed from
+/// the lookup table. Therefore, all deduplication strategies guarantee that data requests for
+/// selected locales will succeed so long as fallback is enabled at runtime (either internally
+/// or externally). They also do not impact which _payloads_ are included: only the lookup table.
+///
+/// Comparison of the deduplication strategies:
+///
+/// | Name | Data file size | Supported locale queries? | Needs runtime fallback? |
+/// |---|---|---|---|
+/// | [`Maximal`] | Smallest | No | Yes |
+/// | [`RetainBaseLanguages`] | Small | Yes | Yes |
+/// | [`None`] | Medium/Small | Yes | No |
+///
+/// [`Maximal`]: DeduplicationStrategy::Maximal
+/// [`RetainBaseLanguages`]: DeduplicationStrategy::RetainBaseLanguages
+/// [`None`]: DeduplicationStrategy::None
+#[non_exhaustive]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum DeduplicationStrategy {
+    /// Removes from the lookup table any locale whose parent maps to the same data.
+    Maximal,
+    /// Removes from the lookup table any locale whose parent maps to the same data, except if
+    /// the parent is `und`.
+    RetainBaseLanguages,
+    /// Keeps all selected locales in the lookup table.
+    None,
+}