From 7193a9464018289cc7053bdaf18b1d417dd312c1 Mon Sep 17 00:00:00 2001 From: Robert Bastian <4706271+robertbastian@users.noreply.github.com> Date: Fri, 6 Sep 2024 21:14:09 +0200 Subject: [PATCH] Fix transliterator fallback loading (#5489) --- .../src/transliterate/transliterator/mod.rs | 99 ++++++++----------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/components/experimental/src/transliterate/transliterator/mod.rs b/components/experimental/src/transliterate/transliterator/mod.rs index 300f48a9774..0c96fb98c65 100644 --- a/components/experimental/src/transliterate/transliterator/mod.rs +++ b/components/experimental/src/transliterate/transliterator/mod.rs @@ -205,13 +205,7 @@ type Env = LiteMap; /// let provider = collection.as_provider(); /// let t = Transliterator::try_new_with_override_unstable( /// "und-t-und-x0-custom".parse().unwrap(), -/// |locale| { -/// if locale.normalizing_eq("und-t-und-x0-dep2") { -/// Some(Box::new(AsciiUpperTransliterator)) -/// } else { -/// None -/// } -/// }, +/// |locale| locale.normalizing_eq("und-t-und-x0-dep2").then_some(Ok(Box::new(AsciiUpperTransliterator))), /// &provider, /// ) /// .unwrap(); @@ -254,7 +248,7 @@ impl Transliterator { { Self::internal_try_new_with_override_unstable( locale, - None::<&fn(&Locale) -> Option>>, + None::<&fn(&Locale) -> Option, DataError>>>, provider, provider, ) @@ -307,7 +301,7 @@ impl Transliterator { + DataProvider + DataProvider + ?Sized, - F: Fn(&Locale) -> Option>, + F: Fn(&Locale) -> Option, DataError>>, { Self::internal_try_new_with_override_unstable(locale, Some(&lookup), provider, provider) } @@ -326,7 +320,7 @@ impl Transliterator { + DataProvider + DataProvider + ?Sized, - F: Fn(&Locale) -> Option>, + F: Fn(&Locale) -> Option, DataError>>, { let payload = Transliterator::load_rbt( #[allow(clippy::unwrap_used)] // infallible @@ -369,19 +363,18 @@ impl Transliterator { + DataProvider + DataProvider + ?Sized, - F: Fn(&Locale) -> Option>, + F: Fn(&Locale) -> Option, DataError>>, { for dep in rbt.deps() { if !env.contains_key(&*dep) { // 1. Insert a placeholder to avoid infinite recursion. env.insert(dep.to_string(), InternalTransliterator::Null); // 2. Load the transliterator, by checking - let internal_t = dep + let internal_t = // a) hardcoded specials - .strip_prefix("x-") - .map(|special| Transliterator::load_special(special, normalizer_provider)) + dep.strip_prefix("x-").and_then(|special| Transliterator::load_special(special, normalizer_provider)) // b) the user-provided override - .or_else(|| Transliterator::load_with_override(&dep, lookup?).transpose()) + .or_else(|| Some(lookup?(&dep.parse().ok()?)?.map(InternalTransliterator::Dyn))) // c) the data .unwrap_or_else(|| { let rbt = Transliterator::load_rbt( @@ -411,7 +404,7 @@ impl Transliterator { fn load_special

( special: &str, normalizer_provider: &P, - ) -> Result + ) -> Option> where P: DataProvider + DataProvider @@ -422,53 +415,43 @@ impl Transliterator { { // TODO(#3909, #3910): add more match special { - "any-nfc" => Ok(InternalTransliterator::Composing( - ComposingTransliterator::try_nfc(normalizer_provider)?, - )), - "any-nfkc" => Ok(InternalTransliterator::Composing( - ComposingTransliterator::try_nfkc(normalizer_provider)?, - )), - "any-nfd" => Ok(InternalTransliterator::Decomposing( - DecomposingTransliterator::try_nfd(normalizer_provider)?, - )), - "any-nfkd" => Ok(InternalTransliterator::Decomposing( - DecomposingTransliterator::try_nfkd(normalizer_provider)?, - )), - "any-null" => Ok(InternalTransliterator::Null), - "any-remove" => Ok(InternalTransliterator::Remove), - "any-hex/unicode" => Ok(InternalTransliterator::Hex( + "any-nfc" => Some( + ComposingTransliterator::try_nfc(normalizer_provider) + .map(InternalTransliterator::Composing), + ), + "any-nfkc" => Some( + ComposingTransliterator::try_nfkc(normalizer_provider) + .map(InternalTransliterator::Composing), + ), + "any-nfd" => Some( + DecomposingTransliterator::try_nfd(normalizer_provider) + .map(InternalTransliterator::Decomposing), + ), + "any-nfkd" => Some( + DecomposingTransliterator::try_nfkd(normalizer_provider) + .map(InternalTransliterator::Decomposing), + ), + "any-null" => Some(Ok(InternalTransliterator::Null)), + "any-remove" => Some(Ok(InternalTransliterator::Remove)), + "any-hex/unicode" => Some(Ok(InternalTransliterator::Hex( hardcoded::HexTransliterator::new("U+", "", 4, Case::Upper), - )), - "any-hex/rust" => Ok(InternalTransliterator::Hex( + ))), + "any-hex/rust" => Some(Ok(InternalTransliterator::Hex( hardcoded::HexTransliterator::new("\\u{", "}", 2, Case::Lower), - )), - "any-hex/xml" => Ok(InternalTransliterator::Hex( + ))), + "any-hex/xml" => Some(Ok(InternalTransliterator::Hex( hardcoded::HexTransliterator::new("&#x", ";", 1, Case::Upper), - )), - "any-hex/perl" => Ok(InternalTransliterator::Hex( + ))), + "any-hex/perl" => Some(Ok(InternalTransliterator::Hex( hardcoded::HexTransliterator::new("\\x{", "}", 1, Case::Upper), - )), - "any-hex/plain" => Ok(InternalTransliterator::Hex( + ))), + "any-hex/plain" => Some(Ok(InternalTransliterator::Hex( hardcoded::HexTransliterator::new("", "", 4, Case::Upper), - )), - s => Err(DataError::custom("unavailable transliterator").with_debug_context(s)), + ))), + _ => None, } } - fn load_with_override( - id: &str, - lookup: &F, - ) -> Result, DataError> - where - F: Fn(&Locale) -> Option>, - { - let locale: Locale = id.parse().map_err(|e| { - DataError::custom("invalid data: transliterator dependency is not a valid Locale") - .with_debug_context(&e) - })?; - Ok(lookup(&locale).map(InternalTransliterator::Dyn)) - } - fn load_rbt

( marker_attributes: &DataMarkerAttributes, provider: &P, @@ -1418,7 +1401,11 @@ mod tests { let want_locale = "und-t-und-latn-d0-ascii".parse().unwrap(); let t = Transliterator::try_new_with_override_unstable( "de-t-de-d0-ascii".parse().unwrap(), - |locale| locale.eq(&want_locale).then_some(Box::new(MaoamTranslit)), + |locale| { + locale + .eq(&want_locale) + .then_some(Ok(Box::new(MaoamTranslit))) + }, &TestingProvider, ) .unwrap();