Skip to content

Commit

Permalink
Split default icu data into lazily deserialized parts (#3948)
Browse files Browse the repository at this point in the history 
* Split default icu data into lazily deserialized parts

* FIx no_std compilation

* Lazily load more ICU tools

* Fix regressions and use more stable constructors
  • Loading branch information
@jedel1043
jedel1043 authored Aug 15, 2024
1 parent c6c6e4c commit 00f8e00
Show file tree
Hide file tree
Showing 25 changed files with 385 additions and 207 deletions.
11 changes: 10 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 19 additions & 14 deletions core/engine/src/builtins/intl/collator/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::{
OrdinaryObject,
},
context::{
icu::IntlProvider,
icu::{ErasedProvider, IntlProvider},
intrinsics::{Intrinsics, StandardConstructor, StandardConstructors},
},
js_string,
Expand Down Expand Up @@ -279,7 +279,7 @@ impl BuiltInConstructor for Collator {
requested_locales,
&mut intl_options,
context.intl_provider(),
);
)?;

let collator_locale = {
// `collator_locale` needs to be different from the resolved locale because ECMA402 doesn't
Expand Down Expand Up @@ -335,18 +335,23 @@ impl BuiltInConstructor for Collator {
.then_some((AlternateHandling::Shifted, MaxVariable::Punctuation))
.unzip();

let collator =
icu_collator::Collator::try_new_unstable(context.intl_provider(), &collator_locale, {
let mut options = icu_collator::CollatorOptions::new();
options.strength = strength;
options.case_level = case_level;
options.case_first = case_first;
options.numeric = Some(if numeric { Numeric::On } else { Numeric::Off });
options.alternate_handling = alternate_handling;
options.max_variable = max_variable;
options
})
.map_err(|e| JsNativeError::typ().with_message(e.to_string()))?;
let mut options = icu_collator::CollatorOptions::new();
options.strength = strength;
options.case_level = case_level;
options.case_first = case_first;
options.numeric = Some(if numeric { Numeric::On } else { Numeric::Off });
options.alternate_handling = alternate_handling;
options.max_variable = max_variable;

let collator = match context.intl_provider().erased_provider() {
ErasedProvider::Any(a) => {
icu_collator::Collator::try_new_with_any_provider(a, &collator_locale, options)
}
ErasedProvider::Buffer(b) => {
icu_collator::Collator::try_new_with_buffer_provider(b, &collator_locale, options)
}
}
.map_err(|e| JsNativeError::typ().with_message(e.to_string()))?;

let prototype =
get_prototype_from_constructor(new_target, StandardConstructors::collator, context)?;
Expand Down
44 changes: 25 additions & 19 deletions core/engine/src/builtins/intl/list_format/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ use crate::{
options::{get_option, get_options_object},
Array, BuiltInBuilder, BuiltInConstructor, BuiltInObject, IntrinsicObject, OrdinaryObject,
},
context::intrinsics::{Intrinsics, StandardConstructor, StandardConstructors},
context::{
icu::ErasedProvider,
intrinsics::{Intrinsics, StandardConstructor, StandardConstructors},
},
js_string,
object::{internal_methods::get_prototype_from_constructor, JsObject},
property::Attribute,
Expand Down Expand Up @@ -128,7 +131,7 @@ impl BuiltInConstructor for ListFormat {
..Default::default()
},
context.intl_provider(),
);
)?;

// 11. Let type be ? GetOption(options, "type", string, « "conjunction", "disjunction", "unit" », "conjunction").
// 12. Set listFormat.[[Type]] to type.
Expand All @@ -142,23 +145,26 @@ impl BuiltInConstructor for ListFormat {
// 16. Let dataLocaleData be localeData.[[<dataLocale>]].
// 17. Let dataLocaleTypes be dataLocaleData.[[<type>]].
// 18. Set listFormat.[[Templates]] to dataLocaleTypes.[[<style>]].
let data_locale = DataLocale::from(&locale);
let formatter = match typ {
ListFormatType::Conjunction => ListFormatter::try_new_and_with_length_unstable(
context.intl_provider(),
&data_locale,
style,
),
ListFormatType::Disjunction => ListFormatter::try_new_or_with_length_unstable(
context.intl_provider(),
&data_locale,
style,
),
ListFormatType::Unit => ListFormatter::try_new_unit_with_length_unstable(
context.intl_provider(),
&data_locale,
style,
),
let data_locale = &DataLocale::from(&locale);
let formatter = match (typ, context.intl_provider().erased_provider()) {
(ListFormatType::Conjunction, ErasedProvider::Any(a)) => {
ListFormatter::try_new_and_with_length_with_any_provider(a, data_locale, style)
}
(ListFormatType::Disjunction, ErasedProvider::Any(a)) => {
ListFormatter::try_new_or_with_length_with_any_provider(a, data_locale, style)
}
(ListFormatType::Unit, ErasedProvider::Any(a)) => {
ListFormatter::try_new_unit_with_length_with_any_provider(a, data_locale, style)
}
(ListFormatType::Conjunction, ErasedProvider::Buffer(b)) => {
ListFormatter::try_new_and_with_length_with_buffer_provider(b, data_locale, style)
}
(ListFormatType::Disjunction, ErasedProvider::Buffer(b)) => {
ListFormatter::try_new_or_with_length_with_buffer_provider(b, data_locale, style)
}
(ListFormatType::Unit, ErasedProvider::Buffer(b)) => {
ListFormatter::try_new_unit_with_length_with_buffer_provider(b, data_locale, style)
}
}
.map_err(|e| JsNativeError::typ().with_message(e.to_string()))?;

Expand Down
16 changes: 11 additions & 5 deletions core/engine/src/builtins/intl/locale/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ impl BuiltInConstructor for Locale {
// 10. Set tag to ! CanonicalizeUnicodeLocaleId(tag).
context
.intl_provider()
.locale_canonicalizer()
.locale_canonicalizer()?
.canonicalize(&mut tag);

// Skipping some boilerplate since this is easier to do using the `Locale` type, but putting the
Expand Down Expand Up @@ -282,7 +282,7 @@ impl BuiltInConstructor for Locale {
// 17. Return ! CanonicalizeUnicodeLocaleId(tag).
context
.intl_provider()
.locale_canonicalizer()
.locale_canonicalizer()?
.canonicalize(&mut tag);
}

Expand Down Expand Up @@ -368,7 +368,7 @@ impl BuiltInConstructor for Locale {

context
.intl_provider()
.locale_canonicalizer()
.locale_canonicalizer()?
.canonicalize(&mut tag);

// 6. Let locale be ? OrdinaryCreateFromConstructor(NewTarget, "%Locale.prototype%", internalSlotsList).
Expand Down Expand Up @@ -409,7 +409,10 @@ impl Locale {
.clone();

// 3. Let maximal be the result of the Add Likely Subtags algorithm applied to loc.[[Locale]]. If an error is signaled, set maximal to loc.[[Locale]].
context.intl_provider().locale_expander().maximize(&mut loc);
context
.intl_provider()
.locale_expander()?
.maximize(&mut loc);

// 4. Return ! Construct(%Locale%, maximal).
let prototype = context.intrinsics().constructors().locale().prototype();
Expand Down Expand Up @@ -445,7 +448,10 @@ impl Locale {
.clone();

// 3. Let minimal be the result of the Remove Likely Subtags algorithm applied to loc.[[Locale]]. If an error is signaled, set minimal to loc.[[Locale]].
context.intl_provider().locale_expander().minimize(&mut loc);
context
.intl_provider()
.locale_expander()?
.minimize(&mut loc);

// 4. Return ! Construct(%Locale%, minimal).
let prototype = context.intrinsics().constructors().locale().prototype();
Expand Down
11 changes: 6 additions & 5 deletions core/engine/src/builtins/intl/locale/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ impl Service for TestService {

#[test]
fn locale_resolution() {
let provider = IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer()).unwrap();
let mut default = default_locale(provider.locale_canonicalizer());
let provider = IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer());
let mut default = default_locale(provider.locale_canonicalizer().unwrap());
default
.extensions
.unicode
Expand All @@ -88,7 +88,7 @@ fn locale_resolution() {
hc: Some(HourCycle::H11),
},
};
let locale = resolve_locale::<TestService>([], &mut options, &provider);
let locale = resolve_locale::<TestService>([], &mut options, &provider).unwrap();
assert_eq!(locale, default);

// test best fit
Expand All @@ -99,7 +99,7 @@ fn locale_resolution() {
},
};

let locale = resolve_locale::<TestService>([], &mut options, &provider);
let locale = resolve_locale::<TestService>([], &mut options, &provider).unwrap();
assert_eq!(locale, default);

// requested: [es-ES]
Expand All @@ -108,6 +108,7 @@ fn locale_resolution() {
service_options: TestOptions { hc: None },
};

let locale = resolve_locale::<TestService>([locale!("es-AR")], &mut options, &provider);
let locale =
resolve_locale::<TestService>([locale!("es-AR")], &mut options, &provider).unwrap();
assert_eq!(locale, "es-u-hc-h23".parse().unwrap());
}
23 changes: 14 additions & 9 deletions core/engine/src/builtins/intl/locale/utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ pub(crate) fn canonicalize_locale_list(
// vi. Let canonicalizedTag be CanonicalizeUnicodeLocaleId(tag).
context
.intl_provider()
.locale_canonicalizer()
.locale_canonicalizer()?
.canonicalize(&mut tag);

// vii. If canonicalizedTag is not an element of seen, append canonicalizedTag as the last element of seen.
Expand Down Expand Up @@ -316,7 +316,7 @@ pub(in crate::builtins::intl) fn resolve_locale<S>(
requested_locales: impl IntoIterator<Item = Locale>,
options: &mut IntlOptions<S::LocaleOptions>,
provider: &IntlProvider,
) -> Locale
) -> JsResult<Locale>
where
S: Service,
IntlProvider: DataProvider<S::LangMarker>,
Expand All @@ -327,12 +327,17 @@ where
// 3. Else,
// a. Let r be LookupMatchingLocaleByBestFit(availableLocales, requestedLocales).
// 4. If r is undefined, set r to the Record { [[locale]]: DefaultLocale(), [[extension]]: empty }.
let mut found_locale = if options.matcher == LocaleMatcher::Lookup {
let found_locale = if options.matcher == LocaleMatcher::Lookup {
lookup_matching_locale_by_prefix::<S::LangMarker>(requested_locales, provider)
} else {
lookup_matching_locale_by_best_fit::<S::LangMarker>(requested_locales, provider)
}
.unwrap_or_else(|| default_locale(provider.locale_canonicalizer()));
};

let mut found_locale = if let Some(loc) = found_locale {
loc
} else {
default_locale(provider.locale_canonicalizer()?)
};

// From here, the spec differs significantly from the implementation,
// since ICU4X allows us to skip some steps and modularize the
Expand Down Expand Up @@ -388,9 +393,9 @@ where
// 12. Return result.
S::resolve(&mut found_locale, &mut options.service_options, provider);
provider
.locale_canonicalizer()
.locale_canonicalizer()?
.canonicalize(&mut found_locale);
found_locale
Ok(found_locale)
}

/// Abstract operation [`FilterLocales ( availableLocales, requestedLocales, options )`][spec]
Expand Down Expand Up @@ -493,7 +498,7 @@ mod tests {

#[test]
fn best_fit() {
let icu = &IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer()).unwrap();
let icu = &IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer());

assert_eq!(
lookup_matching_locale_by_best_fit::<CardinalV1Marker>([locale!("en")], icu),
Expand All @@ -513,7 +518,7 @@ mod tests {

#[test]
fn lookup_match() {
let icu = &IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer()).unwrap();
let icu = &IntlProvider::try_new_with_buffer_provider(boa_icu_provider::buffer());

// requested: [fr-FR-u-hc-h12]
let requested: Locale = "fr-FR-u-hc-h12".parse().unwrap();
Expand Down
30 changes: 19 additions & 11 deletions core/engine/src/builtins/intl/number_format/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use icu_locid::{
extensions::unicode::{key, Value},
Locale,
};
use icu_provider::DataLocale;
use num_bigint::BigInt;
use num_traits::Num;
pub(crate) use options::*;
Expand All @@ -24,7 +25,10 @@ use crate::{
builder::BuiltInBuilder, options::get_option, string::is_trimmable_whitespace,
BuiltInConstructor, BuiltInObject, IntrinsicObject,
},
context::intrinsics::{Intrinsics, StandardConstructor, StandardConstructors},
context::{
icu::ErasedProvider,
intrinsics::{Intrinsics, StandardConstructor, StandardConstructors},
},
js_string,
object::{
internal_methods::get_prototype_from_constructor, FunctionObjectBuilder, JsFunction,
Expand Down Expand Up @@ -240,7 +244,7 @@ impl BuiltInConstructor for NumberFormat {
requested_locales,
&mut intl_options,
context.intl_provider(),
);
)?;

// 11. Set numberFormat.[[Locale]] to r.[[locale]].
// 12. Set numberFormat.[[DataLocale]] to r.[[dataLocale]].
Expand Down Expand Up @@ -365,15 +369,19 @@ impl BuiltInConstructor for NumberFormat {
let sign_display =
get_option(&options, js_str!("signDisplay"), context)?.unwrap_or(SignDisplay::Auto);

let formatter = FixedDecimalFormatter::try_new_unstable(
context.intl_provider(),
&locale.clone().into(),
{
let mut options = FixedDecimalFormatterOptions::default();
options.grouping_strategy = use_grouping;
options
},
)
let mut options = FixedDecimalFormatterOptions::default();
options.grouping_strategy = use_grouping;

let data_locale = &DataLocale::from(&locale);

let formatter = match context.intl_provider().erased_provider() {
ErasedProvider::Any(a) => {
FixedDecimalFormatter::try_new_with_any_provider(a, data_locale, options)
}
ErasedProvider::Buffer(b) => {
FixedDecimalFormatter::try_new_with_buffer_provider(b, data_locale, options)
}
}
.map_err(|err| JsNativeError::typ().with_message(err.to_string()))?;

let number_format = JsObject::from_proto_and_data_with_shared_shape(
Expand Down
Loading

0 comments on commit 00f8e00

Please sign in to comment.