//! Bundles the part of UTS 46 that makes sense to implement as a //! normalization. //! //! This is meant to be used as a building block of an UTS 46 //! implementation, such as the `idna` crate.
usecrate::ComposingNormalizer; usecrate::ComposingNormalizerBorrowed; usecrate::IgnorableBehavior; usecrate::IteratorPolicy; usecrate::NormalizerNfcV2; usecrate::NormalizerNfdTablesV1; usecrate::NormalizerNfkdTablesV1; usecrate::NormalizerUts46DataV1; use icu_collections::codepointtrie::CharIterWithTrie; use icu_collections::codepointtrie::CodePointTrie; use icu_provider::DataError; use icu_provider::DataProvider;
// Implementation note: Despite merely wrapping a `ComposingNormalizer`, // having a `Uts46Mapper` serves two purposes: // // 1. Denying public access to parts of the `ComposingNormalizer` API // that don't work when the data contains markers for ignorables. // 2. Providing a place where additional iterator pre-processing or // post-processing can take place if needed in the future. (When // writing this, it looked like such processing was needed but // now isn't needed after all.)
/// A borrowed version of a mapper that knows how to performs the /// subsets of UTS 46 processing documented on the methods. #[derive(Debug)] pubstruct Uts46MapperBorrowed<'a> {
normalizer: ComposingNormalizerBorrowed<'a>,
}
impl Uts46MapperBorrowed<'static> { /// Cheaply converts a [`Uts46MapperBorrowed<'static>`] into a [`Uts46Mapper`]. /// /// Note: Due to branching and indirection, using [`Uts46Mapper`] might inhibit some /// compile-time optimizations that are possible with [`Uts46MapperBorrowed`]. pubconstfn static_to_owned(self) -> Uts46Mapper {
Uts46Mapper {
normalizer: self.normalizer.static_to_owned(),
}
}
impl Uts46MapperBorrowed<'_> { /// Returns an iterator adaptor that turns an `Iterator` over `char` /// into an iterator yielding a `char` sequence that gets the following /// operations from the "Map" and "Normalize" steps of the "Processing" /// section of UTS 46 lazily applied to it: /// /// 1. The `ignored` characters are ignored. /// 2. The `mapped` characters are mapped. /// 3. The `disallowed` characters are replaced with U+FFFD, /// which itself is a disallowed character. /// 4. The `deviation` characters are treated as `mapped` or `valid` /// as appropriate. /// 5. The `disallowed_STD3_valid` characters are treated as allowed. /// 6. The `disallowed_STD3_mapped` characters are treated as /// `mapped`. /// 7. The result is normalized to NFC. /// /// Notably: /// /// * The STD3 or WHATWG ASCII deny list should be implemented as a /// post-processing step. /// * Transitional processing is not performed. Transitional mapping /// would be a pre-processing step, but transitional processing is /// deprecated, and none of Firefox, Safari, or Chrome use it. pubfn map_normalize<'delegate, I: Iterator<Item = char> + 'delegate>(
&'delegate self,
iter: I,
) -> impl Iterator<Item = char> + 'delegate { letmut ret = self.normalizer
.normalize_iter_private::<_, Trie46, Uts46MapNormalizePolicy>(
CharIterWithTrie::new(iter, self.normalizer.trie::<Trie46<'_>>()),
);
ret.decomposition.init(); // Discard the U+0000.
ret
}
/// Returns an iterator adaptor that turns an `Iterator` over `char` /// into an iterator yielding a `char` sequence that gets the following /// operations from the NFC check and statucs steps of the "Validity /// Criteria" section of UTS 46 lazily applied to it: /// /// 1. The `ignored` characters are treated as `disallowed`. /// 2. The `mapped` characters are mapped. /// 3. The `disallowed` characters are replaced with U+FFFD, /// which itself is a disallowed character. /// 4. The `deviation` characters are treated as `mapped` or `valid` /// as appropriate. /// 5. The `disallowed_STD3_valid` characters are treated as allowed. /// 6. The `disallowed_STD3_mapped` characters are treated as /// `mapped`. /// 7. The result is normalized to NFC. /// /// Notably: /// /// * The STD3 or WHATWG ASCII deny list should be implemented as a /// post-processing step. /// * Transitional processing is not performed. Transitional mapping /// would be a pre-processing step, but transitional processing is /// deprecated, and none of Firefox, Safari, or Chrome use it. /// * The output needs to be compared with input to see if anything /// changed. This check catches failures to adhere to the normalization /// and status requirements. In particular, this comparison results /// in _mapped_ characters resulting in error like "Validity Criteria" /// requires. #[inline] pubfn normalize_validate<'delegate, I: Iterator<Item = char> + 'delegate>(
&'delegate self,
iter: I,
) -> impl Iterator<Item = char> + 'delegate { letmut ret = self
.normalizer
.normalize_iter_private::<_, Trie46, Uts46NormalizeValidatePolicy>(
CharIterWithTrie::new(iter, self.normalizer.trie::<Trie46<'_>>()),
);
ret.decomposition.init(); // Discard the U+0000.
ret
}
}
/// A mapper that knows how to performs the subsets of UTS 46 processing /// documented on the methods. #[derive(Debug)] pubstruct Uts46Mapper {
normalizer: ComposingNormalizer,
}
impl Uts46Mapper { /// Constructs a borrowed version of this type for more efficient querying. pubfn as_borrowed(&self) -> Uts46MapperBorrowed<'_> {
Uts46MapperBorrowed {
normalizer: self.normalizer.as_borrowed(),
}
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.