Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Less ZeroMap mutation in datagen #3098

Merged
merged 3 commits into from
Feb 9, 2023
Merged

Less ZeroMap mutation in datagen #3098

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
283b1d6
dgmaps
robertbastian Jan 30, 2023
d64accd
-impl
robertbastian Feb 9, 2023
bb9fa2f
clippy
robertbastian Feb 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions provider/datagen/src/transform/cldr/calendar/japanese.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ use std::env;
use std::str::FromStr;
use tinystr::tinystr;
use tinystr::TinyStr16;
use zerovec::ule::AsULE;
use zerovec::ZeroVec;

const JAPANESE_FILE: &str = include_str!("./snapshot-japanese@1.json");

Expand Down Expand Up @@ -53,9 +51,7 @@ impl crate::DatagenProvider {
.abbr;
let era_dates_map = &era_dates.supplemental.calendar_data.japanese.eras;

let mut ret = JapaneseErasV1 {
dates_to_eras: ZeroVec::new(),
};
let mut dates_to_eras = BTreeMap::new();

for (era_id, era_name) in era_name_map.iter() {
// These don't exist but may in the future
Expand All @@ -78,12 +74,13 @@ impl crate::DatagenProvider {
let code = era_to_code(era_name, start_date.year)
.map_err(|e| DataError::custom("Era codes").with_display_context(&e))?;
if start_date.year >= 1868 || japanext {
ret.dates_to_eras
.with_mut(|v| v.push((start_date, code).to_unaligned()));
dates_to_eras.insert(start_date, code);
}
}

ret.dates_to_eras.to_mut_slice().sort_unstable();
let ret = JapaneseErasV1 {
dates_to_eras: dates_to_eras.into_iter().collect(),
};

// Integrity check
//
Expand Down
7 changes: 3 additions & 4 deletions provider/datagen/src/transform/cldr/datetime/symbols.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ use icu_datetime::provider::calendar::*;
use std::borrow::Cow;
use std::collections::BTreeMap;
use tinystr::{tinystr, TinyStr16, TinyStr4};
use zerovec::ZeroMap;

pub fn convert_dates(other: &cldr_serde::ca::Dates, calendar: &str) -> DateSymbolsV1<'static> {
DateSymbolsV1 {
Expand Down Expand Up @@ -222,7 +221,7 @@ impl cldr_serde::ca::months::Symbols {
}
months::SymbolsV1::SolarTwelve(arr)
} else {
let mut map: ZeroMap<MonthCode, str> = ZeroMap::default();
let mut map = BTreeMap::new();
for (k, v) in self.0.iter() {
let index: usize = k
.parse()
Expand All @@ -234,9 +233,9 @@ impl cldr_serde::ca::months::Symbols {
.get(index - 1)
.expect("Found out of bounds month index for calendar");

map.insert(&MonthCode(*code), v);
map.insert(MonthCode(*code), v.as_ref());
}
months::SymbolsV1::Other(map)
months::SymbolsV1::Other(map.into_iter().collect())
}
}
}
Expand Down
18 changes: 9 additions & 9 deletions provider/datagen/src/transform/cldr/displaynames/language.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use core::convert::TryFrom;
use icu_displaynames::provider::*;
use icu_provider::datagen::IterableDataProvider;
use icu_provider::prelude::*;
use std::collections::BTreeMap;
use zerovec::ule::UnvalidatedStr;
use zerovec::ZeroMap;

impl DataProvider<LanguageDisplayNamesV1Marker> for crate::DatagenProvider {
fn load(
Expand Down Expand Up @@ -57,10 +57,10 @@ const ALT_MENU_SUBSTRING: &str = "-alt-menu";

impl From<&cldr_serde::language_displaynames::Resource> for LanguageDisplayNamesV1<'static> {
fn from(other: &cldr_serde::language_displaynames::Resource) -> Self {
let mut names = ZeroMap::new();
let mut short_names = ZeroMap::new();
let mut long_names = ZeroMap::new();
let mut menu_names = ZeroMap::new();
let mut names = BTreeMap::new();
let mut short_names = BTreeMap::new();
let mut long_names = BTreeMap::new();
let mut menu_names = BTreeMap::new();
for lang_data_entry in other.main.0.iter() {
for entry in lang_data_entry.1.localedisplaynames.languages.iter() {
if let Some(region) = entry.0.strip_suffix(ALT_SHORT_SUBSTRING) {
Expand All @@ -79,10 +79,10 @@ impl From<&cldr_serde::language_displaynames::Resource> for LanguageDisplayNames
}
}
Self {
names,
short_names,
long_names,
menu_names,
names: names.into_iter().collect(),
short_names: short_names.into_iter().collect(),
long_names: long_names.into_iter().collect(),
menu_names: menu_names.into_iter().collect(),
}
}
}
Expand Down
41 changes: 14 additions & 27 deletions provider/datagen/src/transform/cldr/displaynames/region.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ use core::convert::TryFrom;
use icu_displaynames::provider::*;
use icu_provider::datagen::IterableDataProvider;
use icu_provider::prelude::*;
use std::collections::BTreeMap;
use tinystr::TinyAsciiStr;
use tinystr::TinyStrError;
use zerovec::ZeroMap;

impl DataProvider<RegionDisplayNamesV1Marker> for crate::DatagenProvider {
fn load(
Expand Down Expand Up @@ -50,39 +50,26 @@ impl IterableDataProvider<RegionDisplayNamesV1Marker> for crate::DatagenProvider
/// Substring used to denote alternative region names data variants for a given region. For example: "BA-alt-short", "TL-alt-variant".
const ALT_SUBSTRING: &str = "-alt-";
/// Substring used to denote short region display names data variants for a given region. For example: "BA-alt-short".
const SHORT_SUBSTRING: &str = "-short";
const SHORT_SUBSTRING: &str = "-alt-short";

impl TryFrom<&cldr_serde::region_displaynames::Resource> for RegionDisplayNamesV1<'static> {
type Error = TinyStrError;
fn try_from(other: &cldr_serde::region_displaynames::Resource) -> Result<Self, Self::Error> {
let mut names = ZeroMap::new();
let mut short_names = ZeroMap::new();
for lang_data_entry in other.main.0.iter() {
for entry in lang_data_entry.1.localedisplaynames.regions.iter() {
let mut region = String::from(entry.0);
if !region.contains(ALT_SUBSTRING) {
match <TinyAsciiStr<3>>::from_str(&region) {
Ok(key) => {
names.insert(&key, entry.1.as_ref());
}
Err(err) => {
return Err(err);
}
}
} else if region.contains(SHORT_SUBSTRING) {
region.truncate(region.find('-').unwrap());
match <TinyAsciiStr<3>>::from_str(&region) {
Ok(key) => {
short_names.insert(&key, entry.1.as_ref());
}
Err(err) => {
return Err(err);
}
}
let mut names = BTreeMap::new();
let mut short_names = BTreeMap::new();
for (_, lang_display_names) in other.main.0.iter() {
for (region, value) in lang_display_names.localedisplaynames.regions.iter() {
if let Some(region) = region.strip_suffix(SHORT_SUBSTRING) {
short_names.insert(TinyAsciiStr::from_str(region)?, value.as_ref());
} else if !region.contains(ALT_SUBSTRING) {
names.insert(TinyAsciiStr::from_str(region)?, value.as_ref());
}
}
}
Ok(Self { names, short_names })
Ok(Self {
names: names.into_iter().collect(),
short_names: short_names.into_iter().collect(),
})
}
}

Expand Down
32 changes: 19 additions & 13 deletions provider/datagen/src/transform/cldr/fallback/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ use icu_locid::{
use icu_provider::datagen::IterableDataProvider;
use icu_provider::prelude::*;
use icu_provider_adapters::fallback::provider::*;

use std::collections::BTreeMap;
use tinystr::TinyAsciiStr;
use writeable::Writeable;
use zerovec::{maps::ZeroMap2d, ule::UnvalidatedStr, ZeroMap};
use zerovec::{maps::ZeroMap2d, ule::UnvalidatedStr};

impl DataProvider<LocaleFallbackLikelySubtagsV1Marker> for crate::DatagenProvider {
fn load(
Expand Down Expand Up @@ -119,9 +120,9 @@ impl IterableDataProvider<CollationFallbackSupplementV1Marker> for crate::Datage

impl From<&cldr_serde::likely_subtags::Resource> for LocaleFallbackLikelySubtagsV1<'static> {
fn from(source_data: &cldr_serde::likely_subtags::Resource) -> Self {
let mut l2s = ZeroMap::new();
let mut l2s = BTreeMap::<TinyAsciiStr<3>, _>::new();
let mut lr2s = ZeroMap2d::new();
let mut l2r = ZeroMap::new();
let mut l2r = BTreeMap::<TinyAsciiStr<3>, _>::new();
let mut ls2r = ZeroMap2d::new();

// First collect the l2s and l2r maps
Expand All @@ -138,10 +139,10 @@ impl From<&cldr_serde::likely_subtags::Resource> for LocaleFallbackLikelySubtags
let script = maximized.script.expect("maximized");
let region = maximized.region.expect("maximized");
if script != DEFAULT_SCRIPT {
l2s.insert(&language.into(), &script);
l2s.insert(language.into(), script);
}
if region != DEFAULT_REGION {
l2r.insert(&language.into(), &region);
l2r.insert(language.into(), region);
}
}

Expand All @@ -160,14 +161,14 @@ impl From<&cldr_serde::likely_subtags::Resource> for LocaleFallbackLikelySubtags
let region = maximized.region.expect("maximized");
if minimized.script.is_some() {
assert!(minimized.region.is_none(), "{minimized:?}");
let region_for_lang = l2r.get_copied(&language.into()).unwrap_or(DEFAULT_REGION);
let region_for_lang = l2r.get(&language.into()).copied().unwrap_or(DEFAULT_REGION);
if region != region_for_lang {
ls2r.insert(&language.into(), &script.into(), &region);
}
continue;
}
if minimized.region.is_some() {
let script_for_lang = l2s.get_copied(&language.into()).unwrap_or(DEFAULT_SCRIPT);
let script_for_lang = l2s.get(&language.into()).copied().unwrap_or(DEFAULT_SCRIPT);
if script != script_for_lang {
lr2s.insert(&language.into(), &region.into(), &script);
}
Expand All @@ -177,17 +178,17 @@ impl From<&cldr_serde::likely_subtags::Resource> for LocaleFallbackLikelySubtags
}

LocaleFallbackLikelySubtagsV1 {
l2s,
l2s: l2s.into_iter().collect(),
lr2s,
l2r,
l2r: l2r.into_iter().collect(),
ls2r,
}
}
}

impl From<&cldr_serde::parent_locales::Resource> for LocaleFallbackParentsV1<'static> {
fn from(source_data: &cldr_serde::parent_locales::Resource) -> Self {
let mut parents = ZeroMap::new();
let mut parents = BTreeMap::<_, (Language, Option<Script>, Option<Region>)>::new();

for (source, target) in source_data.supplemental.parent_locales.parent_locale.iter() {
assert!(!source.language.is_empty());
Expand All @@ -198,10 +199,15 @@ impl From<&cldr_serde::parent_locales::Resource> for LocaleFallbackParentsV1<'st
// We always fall back from language-script to und
continue;
}
parents.insert((&*source.write_to_string()).into(), &target.into());
parents.insert(source.write_to_string(), target.into());
}

LocaleFallbackParentsV1 { parents }
LocaleFallbackParentsV1 {
parents: parents
.iter()
.map(|(k, v)| (<&UnvalidatedStr>::from(k.as_ref()), v))
.collect(),
}
}
}

Expand Down
Loading