Add payload conversion and heap measurement tool

components/properties/src/provider.rs

@@ -217,7 +217,7 @@ impl<T: TrieValue> icu_provider::DataMarker for UnicodePropertyMapV1Marker<T> {
 
 /// A data structure efficiently storing `Script` and `Script_Extensions` property data.
 #[icu_provider::data_struct(ScriptWithExtensionsPropertyV1Marker)]
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Eq, PartialEq, Clone)]
 #[cfg_attr(
     feature = "provider_serde",
     derive(serde::Serialize, serde::Deserialize)

components/properties/src/script.rs

@@ -178,7 +178,7 @@ impl From<ScriptWithExt> for Script {
 /// the data and data structures that are stored in the corresponding ICU data
 /// file for these properties.
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[derive(Debug, Eq, PartialEq, yoke::Yokeable, zerofrom::ZeroFrom)]
+#[derive(Clone, Debug, Eq, PartialEq, yoke::Yokeable, zerofrom::ZeroFrom)]
 pub struct ScriptWithExtensions<'data> {
     /// Note: The `ScriptWithExt` values in this array will assume a 12-bit layout. The 2
     /// higher order bits 11..10 will indicate how to deduce the Script value and

experimental/list_formatter/src/provider.rs

@@ -19,7 +19,7 @@ use writeable::{LengthHint, Writeable};
     OrListV1Marker = "list/or@1",
     UnitListV1Marker = "list/unit@1"
 )]
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 #[cfg_attr(
     feature = "provider_serde",
     derive(serde::Deserialize, serde::Serialize)

provider/cldr/Cargo.toml

@@ -36,7 +36,7 @@ extra_features = [
 all-features = true
 
 [dependencies]
-icu_provider = { version = "0.5", path = "../../provider/core", features = ["serialize"] }
+icu_provider = { version = "0.5", path = "../../provider/core", features = ["serialize", "datagen"] }
 icu_locid = { version = "0.5", path = "../../components/locid" }
 icu_plurals = { version = "0.5", path = "../../components/plurals" }
 icu_datetime = { version = "0.5", path = "../../components/datetime", features = ["provider_transform_internals"] }

provider/cldr/src/lib.rs

@@ -33,8 +33,8 @@ pub use cldr_paths::CldrPathsAllInOne;
 pub use cldr_paths::CldrPathsLocal;
 pub use error::Error as CldrError;
 
+use icu_provider::datagen::OmnibusDatagenProvider;
 use icu_provider::fork::by_key::MultiForkByKeyProvider;
-use icu_provider::iter::IterableDynProvider;
 use icu_provider::prelude::*;
 use std::convert::TryFrom;
 use std::path::PathBuf;
@@ -57,19 +57,19 @@ type ListProvider = PluralsProvider; // we can't cfg-exclude part of the bound,
 pub fn create_exportable_provider<T: DataMarker>(
     cldr_paths: &dyn CldrPaths,
     _uprops_root: PathBuf,
-) -> Result<MultiForkByKeyProvider<Box<dyn IterableDynProvider<T> + Sync>>, CldrError>
+) -> Result<MultiForkByKeyProvider<Box<dyn OmnibusDatagenProvider<T> + Sync>>, CldrError>
 where
-    AliasesProvider: IterableDynProvider<T>,
-    DateSymbolsProvider: IterableDynProvider<T>,
-    DateSkeletonPatternsProvider: IterableDynProvider<T>,
-    DatePatternsProvider: IterableDynProvider<T>,
-    JapaneseErasProvider: IterableDynProvider<T>,
-    LikelySubtagsProvider: IterableDynProvider<T>,
-    NumbersProvider: IterableDynProvider<T>,
-    PluralsProvider: IterableDynProvider<T>,
-    TimeZonesProvider: IterableDynProvider<T>,
-    ListProvider: IterableDynProvider<T>,
-    WeekDataProvider: IterableDynProvider<T>,
+    AliasesProvider: OmnibusDatagenProvider<T>,
+    DateSymbolsProvider: OmnibusDatagenProvider<T>,
+    DateSkeletonPatternsProvider: OmnibusDatagenProvider<T>,
+    DatePatternsProvider: OmnibusDatagenProvider<T>,
+    JapaneseErasProvider: OmnibusDatagenProvider<T>,
+    LikelySubtagsProvider: OmnibusDatagenProvider<T>,
+    NumbersProvider: OmnibusDatagenProvider<T>,
+    PluralsProvider: OmnibusDatagenProvider<T>,
+    TimeZonesProvider: OmnibusDatagenProvider<T>,
+    ListProvider: OmnibusDatagenProvider<T>,
+    WeekDataProvider: OmnibusDatagenProvider<T>,
 {
     #[allow(unused_variables)] // uprops_root is only used if icu_list
     Ok(MultiForkByKeyProvider {

provider/cldr/src/transform/calendar/japanese.rs

@@ -218,7 +218,12 @@ fn era_to_code(original: &str, year: i32) -> Result<TinyStr16, String> {
     Ok(code)
 }
 
-icu_provider::impl_dyn_provider!(JapaneseErasProvider, [JapaneseErasV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    JapaneseErasProvider,
+    [JapaneseErasV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<JapaneseErasV1Marker> for JapaneseErasProvider {
     fn supported_options(&self) -> Result<Box<dyn Iterator<Item = ResourceOptions>>, DataError> {

provider/cldr/src/transform/datetime/patterns.rs

@@ -42,7 +42,12 @@ impl ResourceProvider<DatePatternsV1Marker> for DatePatternsProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(DatePatternsProvider, [DatePatternsV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    DatePatternsProvider,
+    [DatePatternsV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<DatePatternsV1Marker> for DatePatternsProvider {
     fn supported_options(

provider/cldr/src/transform/datetime/skeletons.rs

@@ -46,7 +46,8 @@ impl ResourceProvider<DateSkeletonPatternsV1Marker> for DateSkeletonPatternsProv
 icu_provider::impl_dyn_provider!(
     DateSkeletonPatternsProvider,
     [DateSkeletonPatternsV1Marker,],
-    SERDE_SE
+    SERDE_SE,
+    impl DataConverter
 );
 
 impl IterableResourceProvider<DateSkeletonPatternsV1Marker> for DateSkeletonPatternsProvider {

provider/cldr/src/transform/datetime/symbols.rs

@@ -44,7 +44,12 @@ impl ResourceProvider<DateSymbolsV1Marker> for DateSymbolsProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(DateSymbolsProvider, [DateSymbolsV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    DateSymbolsProvider,
+    [DateSymbolsV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<DateSymbolsV1Marker> for DateSymbolsProvider {
     fn supported_options(

provider/cldr/src/transform/datetime/week_data.rs

@@ -123,7 +123,12 @@ impl ResourceProvider<WeekDataV1Marker> for WeekDataProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(WeekDataProvider, [WeekDataV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    WeekDataProvider,
+    [WeekDataV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 #[test]
 fn basic_cldr_week_data() {

provider/cldr/src/transform/decimal/mod.rs

@@ -133,7 +133,12 @@ impl ResourceProvider<DecimalSymbolsV1Marker> for NumbersProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(NumbersProvider, [DecimalSymbolsV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    NumbersProvider,
+    [DecimalSymbolsV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<DecimalSymbolsV1Marker> for NumbersProvider {
     fn supported_options(

provider/cldr/src/transform/list/mod.rs

@@ -146,7 +146,8 @@ impl<M: ResourceMarker<Yokeable = ListFormatterPatternsV1<'static>>> ResourcePro
 icu_provider::impl_dyn_provider!(
     ListProvider,
     [AndListV1Marker, OrListV1Marker, UnitListV1Marker,],
-    SERDE_SE
+    SERDE_SE,
+    impl DataConverter
 );
 
 impl<M: ResourceMarker<Yokeable = ListFormatterPatternsV1<'static>>> IterableResourceProvider<M>

provider/cldr/src/transform/locale_canonicalizer/aliases.rs

@@ -54,7 +54,12 @@ impl ResourceProvider<AliasesV1Marker> for AliasesProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(AliasesProvider, [AliasesV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    AliasesProvider,
+    [AliasesV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<AliasesV1Marker> for AliasesProvider {
     fn supported_options(&self) -> Result<Box<dyn Iterator<Item = ResourceOptions>>, DataError> {

provider/cldr/src/transform/locale_canonicalizer/likely_subtags.rs

@@ -59,7 +59,12 @@ impl ResourceProvider<LikelySubtagsV1Marker> for LikelySubtagsProvider {
     }
 }
 
-icu_provider::impl_dyn_provider!(LikelySubtagsProvider, [LikelySubtagsV1Marker,], SERDE_SE);
+icu_provider::impl_dyn_provider!(
+    LikelySubtagsProvider,
+    [LikelySubtagsV1Marker,],
+    SERDE_SE,
+    impl DataConverter
+);
 
 impl IterableResourceProvider<LikelySubtagsV1Marker> for LikelySubtagsProvider {
     fn supported_options(&self) -> Result<Box<dyn Iterator<Item = ResourceOptions>>, DataError> {

provider/cldr/src/transform/plurals/mod.rs

@@ -101,7 +101,8 @@ impl<M: ResourceMarker<Yokeable = PluralRulesV1<'static>>> ResourceProvider<M> f
 icu_provider::impl_dyn_provider!(
     PluralsProvider,
     [OrdinalV1Marker, CardinalV1Marker,],
-    SERDE_SE
+    SERDE_SE,
+    impl DataConverter
 );
 
 impl<M: ResourceMarker<Yokeable = PluralRulesV1<'static>>> IterableResourceProvider<M>

provider/cldr/src/transform/time_zones/mod.rs

@@ -88,7 +88,7 @@ macro_rules! impl_data_provider {
             }
         )+
 
-        icu_provider::impl_dyn_provider!(TimeZonesProvider, [$($marker),+,], SERDE_SE);
+        icu_provider::impl_dyn_provider!(TimeZonesProvider, [$($marker),+,], SERDE_SE, impl DataConverter);
     };
 }
 

provider/core/Cargo.toml

@@ -47,6 +47,9 @@ deserialize_json = ["serde", "serde_json"]
 deserialize_bincode_1 = ["serde", "bincode", "std"]
 deserialize_postcard_07 = ["serde", "postcard"]
 
+# Dependencies for running data generation
+datagen = ["deserialize_postcard_07", "postcard/alloc", "dhat", "serialize"]
+
 [dependencies]
 icu_locid = { version = "0.5", path = "../../components/locid" }
 tinystr = { path = "../../utils/tinystr", version = "0.5.0", features = ["alloc"], default-features = false }
@@ -68,6 +71,7 @@ serde = { version = "1.0", default-features = false, features = ["derive", "allo
 serde_json = { version = "1.0", default-features = false, features = ["alloc"], optional = true }
 bincode = { version = "1.3", optional = true }
 postcard = { version = "0.7.0", default-features = false, optional = true }
+dhat = { version = "0.3.0", optional = true }
 
 [dev-dependencies]
 serde_json = "1.0"