//! The collection of code for locale canonicalization.
usecrate::provider::*; use alloc::vec::Vec; use core::cmp::Ordering;
usecrate::LocaleExpander; usecrate::TransformResult; use icu_locale_core::extensions::Extensions; use icu_locale_core::subtags::{Language, Region, Script}; use icu_locale_core::{
extensions::unicode::key,
subtags::{language, Variant, Variants},
LanguageIdentifier, Locale,
}; use icu_provider::prelude::*; use tinystr::TinyAsciiStr;
/// Implements the algorithm defined in *[UTS #35: Annex C, LocaleId Canonicalization]*. /// /// # Examples /// /// ``` /// use icu::locale::Locale; /// use icu::locale::{LocaleCanonicalizer, TransformResult}; /// /// let lc = LocaleCanonicalizer::new_extended(); /// /// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap(); /// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified); /// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap()); /// ``` /// /// [UTS #35: Annex C, LocaleId Canonicalization]: http://unicode.org/reports/tr35/#LocaleId_Canonicalization #[derive(Debug)] pubstruct LocaleCanonicalizer<Expander = LocaleExpander> { /// Data to support canonicalization.
aliases: DataPayload<LocaleAliasesV1>, /// Likely subtags implementation for delegation.
expander: Expander,
}
fn uts35_rule_matches<'a, I>(
source: &LanguageIdentifier,
language: Language,
script: Option<Script>,
region: Option<Region>,
raw_variants: I,
) -> bool where
I: Iterator<Item = &'a str>,
{
(language.is_unknown() || language == source.language)
&& (script.is_none() || script == source.script)
&& (region.is_none() || region == source.region)
&& { // Checks if variants are a subset of source variants. // As both iterators are sorted, this can be done linearly. letmut source_variants = source.variants.iter(); 'outer: for raw_variant in raw_variants { for source_variant in source_variants.by_ref() { match source_variant.as_str().cmp(raw_variant) {
Ordering::Equal => { // The source_variant is equal, move to next raw_variant continue'outer;
}
Ordering::Less => { // The source_variant is smaller, take the next source_variant
}
Ordering::Greater => { // The source_variant is greater, // raw_variants is not a subset of source_variants returnfalse;
}
}
} // There are raw_variants left after we exhausted source_variants returnfalse;
} true
}
}
fn uts35_replacement<'a, I>(
source: &mut LanguageIdentifier,
ruletype_has_language: bool,
ruletype_has_script: bool,
ruletype_has_region: bool,
ruletype_variants: Option<I>,
replacement: &LanguageIdentifier,
) where
I: Iterator<Item = &'a str>,
{ if ruletype_has_language || (source.language.is_unknown() && !replacement.language.is_unknown())
{
source.language = replacement.language;
} if ruletype_has_script || (source.script.is_none() && replacement.script.is_some()) {
source.script = replacement.script;
} if ruletype_has_region || (source.region.is_none() && replacement.region.is_some()) {
source.region = replacement.region;
} iflet Some(skips) = ruletype_variants { // The rule matches if the ruletype variants are a subset of the source variants. // This means ja-Latn-fonipa-hepburn-heploc matches against the rule for // hepburn-heploc and is canonicalized to ja-Latn-alalc97-fonipa
// We're merging three sorted deduped iterators into a new sequence: // sources - skips + replacements
impl LocaleCanonicalizer<LocaleExpander> { /// A constructor which creates a [`LocaleCanonicalizer`] from compiled data, /// using a [`LocaleExpander`] for common locales. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubconstfn new_common() -> Self { Self::new_with_expander(LocaleExpander::new_common())
}
/// A constructor which creates a [`LocaleCanonicalizer`] from compiled data, /// using a [`LocaleExpander`] for all locales. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubconstfn new_extended() -> Self { Self::new_with_expander(LocaleExpander::new_extended())
}
/// The canonicalize method potentially updates a passed in locale in place /// depending up the results of running the canonicalization algorithm /// from <http://unicode.org/reports/tr35/#LocaleId_Canonicalization>. /// /// Some BCP47 canonicalization data is not part of the CLDR json package. Because /// of this, some canonicalizations are not performed, e.g. the canonicalization of /// `und-u-ca-islamicc` to `und-u-ca-islamic-civil`. This will be fixed in a future /// release once the missing data has been added to the CLDR json data. See: /// <https://github.com/unicode-org/icu4x/issues/746> /// /// # Examples /// /// ``` /// use icu::locale::{Locale, LocaleCanonicalizer, TransformResult}; /// /// let lc = LocaleCanonicalizer::new_extended(); /// /// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap(); /// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified); /// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap()); /// ``` pubfn canonicalize(&self, locale: &mut Locale) -> TransformResult { letmut result = TransformResult::Unmodified;
// This loops until we get a 'fixed point', where applying the rules do not // result in any more changes. loop { // These are linear searches due to the ordering imposed by the canonicalization // rules, where rules with more variants should be considered first. With the // current data in CLDR, we will only do this for locales which have variants, // or new rules which we haven't special-cased yet (of which there are fewer // than 20). let modified = if locale.id.variants.is_empty() { self.canonicalize_absolute_language_fallbacks(&mut locale.id)
} else { self.canonicalize_language_variant_fallbacks(&mut locale.id)
}; if modified {
result = TransformResult::Modified; continue;
}
if !locale.id.language.is_unknown() { // If the region is specified, check sgn-region rules first iflet Some(region) = locale.id.region { if locale.id.language == language!("sgn") { iflet Some(&sgn_lang) = self
.aliases
.get()
.sgn_region
.get(®ion.to_tinystr().to_unvalidated())
{
uts35_replacement::<core::iter::Empty<&str>>(
&mut locale.id, true, false, true,
None,
&sgn_lang.into(),
);
result = TransformResult::Modified; continue;
}
}
}
if uts35_check_language_rules(&mut locale.id, &self.aliases)
== TransformResult::Modified
{
result = TransformResult::Modified; continue;
}
}
locale.id.region = Some( match ( self.expander.as_ref().maximize(&mut maximized),
maximized.region,
) {
(TransformResult::Modified, Some(candidate)) if regions.iter().any(|x| x == candidate) =>
{
candidate
}
_ => default_region,
},
);
result = TransformResult::Modified; continue;
}
}
}
if !locale.id.variants.is_empty() { letmut modified = Vec::with_capacity(0); for (idx, &variant) in locale.id.variants.iter().enumerate() { iflet Some(&updated) = self
.aliases
.get()
.variant
.get(&variant.to_tinystr().to_unvalidated())
{ if modified.is_empty() {
modified = locale.id.variants.to_vec();
} #[expect(clippy::indexing_slicing)] let _ = core::mem::replace(&mut modified[idx], updated);
}
}
if !modified.is_empty() {
modified.sort();
modified.dedup();
locale.id.variants = Variants::from_vec_unchecked(modified);
result = TransformResult::Modified; continue;
}
}
// Nothing matched in this iteration, we're done. break;
}
if !locale.extensions.transform.is_empty() || !locale.extensions.unicode.is_empty() { self.canonicalize_extensions(&mut locale.extensions, &yle='color:red'>mut result);
}
result
}
fn canonicalize_extensions(&self, extensions: &mut Extensions, result: &mut TransformResult) { // Handle Locale extensions in their own loops, because these rules do not interact // with each other. iflet Some(refmut lang) = extensions.transform.lang { while uts35_check_language_rules(lang, &self.aliases) == TransformResult::Modified {
*result = TransformResult::Modified;
}
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.