//! The functions in this module return a [`CodePointSetData`] containing //! the set of characters with a particular Unicode property. //! //! The descriptions of most properties are taken from [`TR44`], the documentation for the //! Unicode Character Database. Some properties are instead defined in [`TR18`], the //! documentation for Unicode regular expressions. In particular, Annex C of this document //! defines properties for POSIX compatibility. //! //! [`CodePointSetData`]: crate::sets::CodePointSetData //! [`TR44`]: https://www.unicode.org/reports/tr44 //! [`TR18`]: https://www.unicode.org/reports/tr18
usecrate::error::PropertiesError; usecrate::provider::*; usecrate::*; use core::iter::FromIterator; use core::ops::RangeInclusive; use icu_collections::codepointinvlist::CodePointInversionList; use icu_collections::codepointinvliststringlist::CodePointInversionListAndStringList; use icu_provider::prelude::*;
// // CodePointSet* structs, impls, & macros // (a set with only code points) //
/// A wrapper around code point set data. It is returned by APIs that return Unicode /// property data in a set-like form, ex: a set of code points sharing the same /// value for a Unicode property. Access its data via the borrowed version, /// [`CodePointSetDataBorrowed`]. #[derive(Debug)] pubstruct CodePointSetData {
data: DataPayload<ErasedSetlikeMarker>,
}
/// Private marker type for CodePointSetData /// to work for all set properties at once #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub(crate) struct ErasedSetlikeMarker; impl DataMarker for ErasedSetlikeMarker { type Yokeable = PropertyCodePointSetV1<'static>;
}
impl CodePointSetData { /// Construct a borrowed version of this type that can be queried. /// /// This owned version if returned by functions that use a runtime data provider. #[inline] pubfn as_borrowed(&self) -> CodePointSetDataBorrowed<'_> {
CodePointSetDataBorrowed {
set: self.data.get(),
}
}
/// Construct a new one from loaded data /// /// Typically it is preferable to use getters like [`load_ascii_hex_digit()`] instead pubfn from_data<M>(data: DataPayload<M>) -> Self where
M: DataMarker<Yokeable = PropertyCodePointSetV1<'static>>,
{ Self { data: data.cast() }
}
/// Construct a new owned [`CodePointInversionList`] pubfn from_code_point_inversion_list(set: CodePointInversionList<'static>) -> Self { let set = PropertyCodePointSetV1::from_code_point_inversion_list(set);
CodePointSetData::from_data(DataPayload::<ErasedSetlikeMarker>::from_owned(set))
}
/// Convert this type to a [`CodePointInversionList`] as a borrowed value. /// /// The data backing this is extensible and supports multiple implementations. /// Currently it is always [`CodePointInversionList`]; however in the future more backends may be /// added, and users may select which at data generation time. /// /// This method returns an `Option` in order to return `None` when the backing data provider /// cannot return a [`CodePointInversionList`], or cannot do so within the expected constant time /// constraint. pubfn as_code_point_inversion_list(&self) -> Option<&CodePointInversionList<'_>> { self.data.get().as_code_point_inversion_list()
}
/// Convert this type to a [`CodePointInversionList`], borrowing if possible, /// otherwise allocating a new [`CodePointInversionList`]. /// /// The data backing this is extensible and supports multiple implementations. /// Currently it is always [`CodePointInversionList`]; however in the future more backends may be /// added, and users may select which at data generation time. /// /// The performance of the conversion to this specific return type will vary /// depending on the data structure that is backing `self`. pubfn to_code_point_inversion_list(&self) -> CodePointInversionList<'_> { self.data.get().to_code_point_inversion_list()
}
}
/// A borrowed wrapper around code point set data, returned by /// [`CodePointSetData::as_borrowed()`]. More efficient to query. #[derive(Clone, Copy, Debug)] pubstruct CodePointSetDataBorrowed<'a> {
set: &'a PropertyCodePointSetV1<'a>,
}
impl CodePointSetDataBorrowed<'static> { /// Cheaply converts a [`CodePointSetDataBorrowed<'static>`] into a [`CodePointSetData`]. /// /// Note: Due to branching and indirection, using [`CodePointSetData`] might inhibit some /// compile-time optimizations that are possible with [`CodePointSetDataBorrowed`]. pubconstfn static_to_owned(self) -> CodePointSetData {
CodePointSetData {
data: DataPayload::from_static_ref(self.set),
}
}
}
impl<'a> CodePointSetDataBorrowed<'a> { /// Check if the set contains a character /// /// ```rust /// use icu::properties::sets; /// /// let alphabetic = sets::alphabetic(); /// /// assert!(!alphabetic.contains('3')); /// assert!(!alphabetic.contains('੩')); // U+0A69 GURMUKHI DIGIT THREE /// assert!(alphabetic.contains('A')); /// assert!(alphabetic.contains('Ä')); // U+00C4 LATIN CAPITAL LETTER A WITH DIAERESIS /// ``` #[inline] pubfn contains(self, ch: char) -> bool { self.set.contains(ch)
}
/// Check if the set contains a character as a UTF32 code unit /// /// ```rust /// use icu::properties::sets; /// /// let alphabetic = sets::alphabetic(); /// /// assert!(!alphabetic.contains32(0x0A69)); // U+0A69 GURMUKHI DIGIT THREE /// assert!(alphabetic.contains32(0x00C4)); // U+00C4 LATIN CAPITAL LETTER A WITH DIAERESIS /// ``` #[inline] pubfn contains32(self, ch: u32) -> bool { self.set.contains32(ch)
}
// Yields an [`Iterator`] returning the ranges of the code points that are /// included in the [`CodePointSetData`] /// /// Ranges are returned as [`RangeInclusive`], which is inclusive of its /// `end` bound value. An end-inclusive behavior matches the ICU4C/J /// behavior of ranges, ex: `UnicodeSet::contains(UChar32 start, UChar32 end)`. /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let alphabetic = sets::alphabetic(); /// let mut ranges = alphabetic.iter_ranges(); /// /// assert_eq!(Some(0x0041..=0x005A), ranges.next()); // 'A'..'Z' /// assert_eq!(Some(0x0061..=0x007A), ranges.next()); // 'a'..'z' /// ``` #[inline] pubfn iter_ranges(self) -> impl Iterator<Item = RangeInclusive<u32>> + 'a { self.set.iter_ranges()
}
// Yields an [`Iterator`] returning the ranges of the code points that are /// *not* included in the [`CodePointSetData`] /// /// Ranges are returned as [`RangeInclusive`], which is inclusive of its /// `end` bound value. An end-inclusive behavior matches the ICU4C/J /// behavior of ranges, ex: `UnicodeSet::contains(UChar32 start, UChar32 end)`. /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let alphabetic = sets::alphabetic(); /// let mut ranges = alphabetic.iter_ranges(); /// /// assert_eq!(Some(0x0041..=0x005A), ranges.next()); // 'A'..'Z' /// assert_eq!(Some(0x0061..=0x007A), ranges.next()); // 'a'..'z' /// ``` #[inline] pubfn iter_ranges_complemented(self) -> impl Iterator<Item = RangeInclusive<u32>> + 'a { self.set.iter_ranges_complemented()
}
}
// // UnicodeSet* structs, impls, & macros // (a set with code points + strings) //
/// A wrapper around `UnicodeSet` data (characters and strings) #[derive(Debug)] pubstruct UnicodeSetData {
data: DataPayload<ErasedUnicodeSetlikeMarker>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub(crate) struct ErasedUnicodeSetlikeMarker; impl DataMarker for ErasedUnicodeSetlikeMarker { type Yokeable = PropertyUnicodeSetV1<'static>;
}
impl UnicodeSetData { /// Construct a borrowed version of this type that can be queried. /// /// This avoids a potential small underlying cost per API call (ex: `contains()`) by consolidating it /// up front. #[inline] pubfn as_borrowed(&self) -> UnicodeSetDataBorrowed<'_> {
UnicodeSetDataBorrowed {
set: self.data.get(),
}
}
/// Construct a new one from loaded data /// /// Typically it is preferable to use getters instead pubfn from_data<M>(data: DataPayload<M>) -> Self where
M: DataMarker<Yokeable = PropertyUnicodeSetV1<'static>>,
{ Self { data: data.cast() }
}
/// Construct a new owned [`CodePointInversionListAndStringList`] pubfn from_code_point_inversion_list_string_list(
set: CodePointInversionListAndStringList<'static>,
) -> Self { let set = PropertyUnicodeSetV1::from_code_point_inversion_list_string_list(set);
UnicodeSetData::from_data(DataPayload::<ErasedUnicodeSetlikeMarker>::from_owned(set))
}
/// Convert this type to a [`CodePointInversionListAndStringList`] as a borrowed value. /// /// The data backing this is extensible and supports multiple implementations. /// Currently it is always [`CodePointInversionListAndStringList`]; however in the future more backends may be /// added, and users may select which at data generation time. /// /// This method returns an `Option` in order to return `None` when the backing data provider /// cannot return a [`CodePointInversionListAndStringList`], or cannot do so within the expected constant time /// constraint. pubfn as_code_point_inversion_list_string_list(
&self,
) -> Option<&CodePointInversionListAndStringList<'_>> { self.data.get().as_code_point_inversion_list_string_list()
}
/// Convert this type to a [`CodePointInversionListAndStringList`], borrowing if possible, /// otherwise allocating a new [`CodePointInversionListAndStringList`]. /// /// The data backing this is extensible and supports multiple implementations. /// Currently it is always [`CodePointInversionListAndStringList`]; however in the future more backends may be /// added, and users may select which at data generation time. /// /// The performance of the conversion to this specific return type will vary /// depending on the data structure that is backing `self`. pubfn to_code_point_inversion_list_string_list(
&self,
) -> CodePointInversionListAndStringList<'_> { self.data.get().to_code_point_inversion_list_string_list()
}
}
/// A borrowed wrapper around code point set data, returned by /// [`UnicodeSetData::as_borrowed()`]. More efficient to query. #[derive(Clone, Copy, Debug)] pubstruct UnicodeSetDataBorrowed<'a> {
set: &'a PropertyUnicodeSetV1<'a>,
}
impl<'a> UnicodeSetDataBorrowed<'a> { /// Check if the set contains the string. Strings consisting of one character /// are treated as a character/code point. /// /// This matches ICU behavior for ICU's `UnicodeSet`. #[inline] pubfn contains(self, s: &str) -> bool { self.set.contains(s)
}
/// Check if the set contains a character as a UTF32 code unit #[inline] pubfn contains32(&self, cp: u32) -> bool { self.set.contains32(cp)
}
/// Check if the set contains the code point corresponding to the Rust character. #[inline] pubfn contains_char(&self, ch: char) -> bool { self.set.contains_char(ch)
}
}
impl UnicodeSetDataBorrowed<'static> { /// Cheaply converts a [`UnicodeSetDataBorrowed<'static>`] into a [`UnicodeSetData`]. /// /// Note: Due to branching and indirection, using [`UnicodeSetData`] might inhibit some /// compile-time optimizations that are possible with [`UnicodeSetDataBorrowed`]. pubconstfn static_to_owned(self) -> UnicodeSetData {
UnicodeSetData {
data: DataPayload::from_static_ref(self.set),
}
}
}
// // Binary property getter fns // (data as code point sets) //
macro_rules! make_code_point_set_property {
( // currently unused
property: $property:expr; // currently unused
marker: $marker_name:ident;
keyed_data_marker: $keyed_data_marker:ty;
func:
$(#[$doc:meta])+
$cvis:vis constfn $constname:ident() => $singleton_name:ident;
$vis:vis fn $funcname:ident();
) => { #[doc = concat!("A version of [`", stringify!($constname), "()`] that uses custom data provided by a [`DataProvider`].")] /// /// Note that this will return an owned version of the data. Functionality is available on /// the borrowed version, accessible through [`CodePointSetData::as_borrowed`].
$vis fn $funcname(
provider: &(impl DataProvider<$keyed_data_marker> + ?Sized)
) -> Result<CodePointSetData, PropertiesError> {
load_set_data(provider)
}
make_code_point_set_property! {
property: "ASCII_Hex_Digit";
marker: AsciiHexDigitProperty;
keyed_data_marker: AsciiHexDigitV1Marker;
func: /// ASCII characters commonly used for the representation of hexadecimal numbers /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let ascii_hex_digit = sets::ascii_hex_digit(); /// /// assert!(ascii_hex_digit.contains('3')); /// assert!(!ascii_hex_digit.contains('੩')); // U+0A69 GURMUKHI DIGIT THREE /// assert!(ascii_hex_digit.contains('A')); /// assert!(!ascii_hex_digit.contains('Ä')); // U+00C4 LATIN CAPITAL LETTER A WITH DIAERESIS /// ``` pubconstfn ascii_hex_digit() => SINGLETON_PROPS_AHEX_V1; pubfn load_ascii_hex_digit();
}
make_code_point_set_property! {
property: "Alnum";
marker: AlnumProperty;
keyed_data_marker: AlnumV1Marker;
func: /// Characters with the Alphabetic or Decimal_Number property /// This is defined for POSIX compatibility.
make_code_point_set_property! {
property: "Bidi_Control";
marker: BidiControlProperty;
keyed_data_marker: BidiControlV1Marker;
func: /// Format control characters which have specific functions in the Unicode Bidirectional /// Algorithm /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let bidi_control = sets::bidi_control(); /// /// assert!(bidi_control.contains32(0x200F)); // RIGHT-TO-LEFT MARK /// assert!(!bidi_control.contains('ش')); // U+0634 ARABIC LETTER SHEEN /// ```
make_code_point_set_property! {
property: "Full_Composition_Exclusion";
marker: FullCompositionExclusionProperty;
keyed_data_marker: FullCompositionExclusionV1Marker;
func: /// Characters that are excluded from composition /// See <https://unicode.org/Public/UNIDATA/CompositionExclusions.txt>
make_code_point_set_property! {
property: "Changes_When_Casefolded";
marker: ChangesWhenCasefoldedProperty;
keyed_data_marker: ChangesWhenCasefoldedV1Marker;
func: /// Characters whose normalized forms are not stable under case folding /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let changes_when_casefolded = sets::changes_when_casefolded(); /// /// assert!(changes_when_casefolded.contains('ß')); // U+00DF LATIN SMALL LETTER SHARP S /// assert!(!changes_when_casefolded.contains('ᜉ')); // U+1709 TAGALOG LETTER PA /// ```
make_code_point_set_property! {
property: "Changes_When_Casemapped";
marker: ChangesWhenCasemappedProperty;
keyed_data_marker: ChangesWhenCasemappedV1Marker;
func: /// Characters which may change when they undergo case mapping
make_code_point_set_property! {
property: "Changes_When_NFKC_Casefolded";
marker: ChangesWhenNfkcCasefoldedProperty;
keyed_data_marker: ChangesWhenNfkcCasefoldedV1Marker;
func: /// Characters which are not identical to their NFKC_Casefold mapping /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let changes_when_nfkc_casefolded = sets::changes_when_nfkc_casefolded(); /// /// assert!(changes_when_nfkc_casefolded.contains('')); // U+1F135 SQUARED LATIN CAPITAL LETTER F /// assert!(!changes_when_nfkc_casefolded.contains('f')); /// ```
make_code_point_set_property! {
property: "Changes_When_Lowercased";
marker: ChangesWhenLowercasedProperty;
keyed_data_marker: ChangesWhenLowercasedV1Marker;
func: /// Characters whose normalized forms are not stable under a toLowercase mapping /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let changes_when_lowercased = sets::changes_when_lowercased(); /// /// assert!(changes_when_lowercased.contains('Ⴔ')); // U+10B4 GEORGIAN CAPITAL LETTER PHAR /// assert!(!changes_when_lowercased.contains('ფ')); // U+10E4 GEORGIAN LETTER PHAR /// ```
make_code_point_set_property! {
property: "Changes_When_Titlecased";
marker: ChangesWhenTitlecasedProperty;
keyed_data_marker: ChangesWhenTitlecasedV1Marker;
func: /// Characters whose normalized forms are not stable under a toTitlecase mapping /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let changes_when_titlecased = sets::changes_when_titlecased(); /// /// assert!(changes_when_titlecased.contains('æ')); // U+00E6 LATIN SMALL LETTER AE /// assert!(!changes_when_titlecased.contains('Æ')); // U+00E6 LATIN CAPITAL LETTER AE /// ```
make_code_point_set_property! {
property: "Changes_When_Uppercased";
marker: ChangesWhenUppercasedProperty;
keyed_data_marker: ChangesWhenUppercasedV1Marker;
func: /// Characters whose normalized forms are not stable under a toUppercase mapping /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let changes_when_uppercased = sets::changes_when_uppercased(); /// /// assert!(changes_when_uppercased.contains('ւ')); // U+0582 ARMENIAN SMALL LETTER YIWN /// assert!(!changes_when_uppercased.contains('Ւ')); // U+0552 ARMENIAN CAPITAL LETTER YIWN /// ```
make_code_point_set_property! {
property: "Deprecated";
marker: DeprecatedProperty;
keyed_data_marker: DeprecatedV1Marker;
func: /// Deprecated characters. No characters will ever be removed from the standard, but the /// usage of deprecated characters is strongly discouraged. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let deprecated = sets::deprecated(); /// /// assert!(deprecated.contains('ឣ')); // U+17A3 KHMER INDEPENDENT VOWEL QAQ /// assert!(!deprecated.contains('A')); /// ```
make_code_point_set_property! {
property: "Default_Ignorable_Code_Point";
marker: DefaultIgnorableCodePointProperty;
keyed_data_marker: DefaultIgnorableCodePointV1Marker;
func: /// For programmatic determination of default ignorable code points. New characters that /// should be ignored in rendering (unless explicitly supported) will be assigned in these /// ranges, permitting programs to correctly handle the default rendering of such /// characters when not otherwise supported. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let default_ignorable_code_point = sets::default_ignorable_code_point(); /// /// assert!(default_ignorable_code_point.contains32(0x180B)); // MONGOLIAN FREE VARIATION SELECTOR ONE /// assert!(!default_ignorable_code_point.contains('E')); /// ```
make_code_point_set_property! {
property: "Diacritic";
marker: DiacriticProperty;
keyed_data_marker: DiacriticV1Marker;
func: /// Characters that linguistically modify the meaning of another character to which they apply /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let diacritic = sets::diacritic(); /// /// assert!(diacritic.contains('\u{05B3}')); // HEBREW POINT HATAF QAMATS /// assert!(!diacritic.contains('א')); // U+05D0 HEBREW LETTER ALEF /// ```
make_code_point_set_property! {
property: "Emoji_Component";
marker: EmojiComponentProperty;
keyed_data_marker: EmojiComponentV1Marker;
func: /// Characters used in emoji sequences that normally do not appear on emoji keyboards as /// separate choices, such as base characters for emoji keycaps /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let emoji_component = sets::emoji_component(); /// /// assert!(emoji_component.contains('')); // U+1F1F9 REGIONAL INDICATOR SYMBOL LETTER T /// assert!(emoji_component.contains32(0x20E3)); // COMBINING ENCLOSING KEYCAP /// assert!(emoji_component.contains('7')); /// assert!(!emoji_component.contains('T')); /// ```
make_code_point_set_property! {
property: "Extender";
marker: ExtenderProperty;
keyed_data_marker: ExtenderV1Marker;
func: /// Characters whose principal function is to extend the value of a preceding alphabetic /// character or to extend the shape of adjacent characters. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let extender = sets::extender(); /// /// assert!(extender.contains('ヾ')); // U+30FE KATAKANA VOICED ITERATION MARK /// assert!(extender.contains('ー')); // U+30FC KATAKANA-HIRAGANA PROLONGED SOUND MARK /// assert!(!extender.contains('・')); // U+30FB KATAKANA MIDDLE DOT /// ```
make_code_point_set_property! {
property: "Extended_Pictographic";
marker: ExtendedPictographicProperty;
keyed_data_marker: ExtendedPictographicV1Marker;
func: /// Pictographic symbols, as well as reserved ranges in blocks largely associated with /// emoji characters /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let extended_pictographic = sets::extended_pictographic(); /// /// assert!(extended_pictographic.contains('')); // U+1F973 FACE WITH PARTY HORN AND PARTY HAT /// assert!(!extended_pictographic.contains('')); // U+1F1EA REGIONAL INDICATOR SYMBOL LETTER E /// ```
make_code_point_set_property! {
property: "Graph";
marker: GraphProperty;
keyed_data_marker: GraphV1Marker;
func: /// Visible characters. /// This is defined for POSIX compatibility.
make_code_point_set_property! {
property: "Grapheme_Base";
marker: GraphemeBaseProperty;
keyed_data_marker: GraphemeBaseV1Marker;
func: /// Property used together with the definition of Standard Korean Syllable Block to define /// "Grapheme base". See D58 in Chapter 3, Conformance in the Unicode Standard. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let grapheme_base = sets::grapheme_base(); /// /// assert!(grapheme_base.contains('ക')); // U+0D15 MALAYALAM LETTER KA /// assert!(grapheme_base.contains('\u{0D3F}')); // U+0D3F MALAYALAM VOWEL SIGN I /// assert!(!grapheme_base.contains('\u{0D3E}')); // U+0D3E MALAYALAM VOWEL SIGN AA /// ```
make_code_point_set_property! {
property: "Grapheme_Extend";
marker: GraphemeExtendProperty;
keyed_data_marker: GraphemeExtendV1Marker;
func: /// Property used to define "Grapheme extender". See D59 in Chapter 3, Conformance in the /// Unicode Standard. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let grapheme_extend = sets::grapheme_extend(); /// /// assert!(!grapheme_extend.contains('ക')); // U+0D15 MALAYALAM LETTER KA /// assert!(!grapheme_extend.contains('\u{0D3F}')); // U+0D3F MALAYALAM VOWEL SIGN I /// assert!(grapheme_extend.contains('\u{0D3E}')); // U+0D3E MALAYALAM VOWEL SIGN AA /// ```
make_code_point_set_property! {
property: "Hex_Digit";
marker: HexDigitProperty;
keyed_data_marker: HexDigitV1Marker;
func: /// Characters commonly used for the representation of hexadecimal numbers, plus their /// compatibility equivalents /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let hex_digit = sets::hex_digit(); /// /// assert!(hex_digit.contains('0')); /// assert!(!hex_digit.contains('੩')); // U+0A69 GURMUKHI DIGIT THREE /// assert!(hex_digit.contains('f')); /// assert!(hex_digit.contains('f')); // U+FF46 FULLWIDTH LATIN SMALL LETTER F /// assert!(hex_digit.contains('F')); // U+FF26 FULLWIDTH LATIN CAPITAL LETTER F /// assert!(!hex_digit.contains('Ä')); // U+00C4 LATIN CAPITAL LETTER A WITH DIAERESIS /// ```
make_code_point_set_property! {
property: "Hyphen";
marker: HyphenProperty;
keyed_data_marker: HyphenV1Marker;
func: /// Deprecated property. Dashes which are used to mark connections between pieces of /// words, plus the Katakana middle dot.
make_code_point_set_property! {
property: "Id_Continue";
marker: IdContinueProperty;
keyed_data_marker: IdContinueV1Marker;
func: /// Characters that can come after the first character in an identifier. If using NFKC to /// fold differences between characters, use [`load_xid_continue`] instead. See /// [`Unicode Standard Annex #31`](https://www.unicode.org/reports/tr31/tr31-35.html) for /// more details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let id_continue = sets::id_continue(); /// /// assert!(id_continue.contains('x')); /// assert!(id_continue.contains('1')); /// assert!(id_continue.contains('_')); /// assert!(id_continue.contains('ߝ')); // U+07DD NKO LETTER FA /// assert!(!id_continue.contains('ⓧ')); // U+24E7 CIRCLED LATIN SMALL LETTER X /// assert!(id_continue.contains32(0xFC5E)); // ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM /// ```
make_code_point_set_property! {
property: "Id_Start";
marker: IdStartProperty;
keyed_data_marker: IdStartV1Marker;
func: /// Characters that can begin an identifier. If using NFKC to fold differences between /// characters, use [`load_xid_start`] instead. See [`Unicode Standard Annex /// #31`](https://www.unicode.org/reports/tr31/tr31-35.html) for more details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let id_start = sets::id_start(); /// /// assert!(id_start.contains('x')); /// assert!(!id_start.contains('1')); /// assert!(!id_start.contains('_')); /// assert!(id_start.contains('ߝ')); // U+07DD NKO LETTER FA /// assert!(!id_start.contains('ⓧ')); // U+24E7 CIRCLED LATIN SMALL LETTER X /// assert!(id_start.contains32(0xFC5E)); // ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM /// ```
make_code_point_set_property! {
property: "Ids_Trinary_Operator";
marker: IdsTrinaryOperatorProperty;
keyed_data_marker: IdsTrinaryOperatorV1Marker;
func: /// Characters used in Ideographic Description Sequences /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let ids_trinary_operator = sets::ids_trinary_operator(); /// /// assert!(ids_trinary_operator.contains32(0x2FF2)); // IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO MIDDLE AND RIGHT /// assert!(ids_trinary_operator.contains32(0x2FF3)); // IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO MIDDLE AND BELOW /// assert!(!ids_trinary_operator.contains32(0x2FF4)); /// assert!(!ids_trinary_operator.contains32(0x2FF5)); // IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM ABOVE /// assert!(!ids_trinary_operator.contains32(0x3006)); // IDEOGRAPHIC CLOSING MARK /// ```
make_code_point_set_property! {
property: "Join_Control";
marker: JoinControlProperty;
keyed_data_marker: JoinControlV1Marker;
func: /// Format control characters which have specific functions for control of cursive joining /// and ligation /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let join_control = sets::join_control(); /// /// assert!(join_control.contains32(0x200C)); // ZERO WIDTH NON-JOINER /// assert!(join_control.contains32(0x200D)); // ZERO WIDTH JOINER /// assert!(!join_control.contains32(0x200E)); /// ```
make_code_point_set_property! {
property: "Logical_Order_Exception";
marker: LogicalOrderExceptionProperty;
keyed_data_marker: LogicalOrderExceptionV1Marker;
func: /// A small number of spacing vowel letters occurring in certain Southeast Asian scripts such as Thai and Lao /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let logical_order_exception = sets::logical_order_exception(); /// /// assert!(logical_order_exception.contains('ແ')); // U+0EC1 LAO VOWEL SIGN EI /// assert!(!logical_order_exception.contains('ະ')); // U+0EB0 LAO VOWEL SIGN A /// ```
make_code_point_set_property! {
property: "NFC_Inert";
marker: NfcInertProperty;
keyed_data_marker: NfcInertV1Marker;
func: /// Characters that are inert under NFC, i.e., they do not interact with adjacent characters
make_code_point_set_property! {
property: "NFD_Inert";
marker: NfdInertProperty;
keyed_data_marker: NfdInertV1Marker;
func: /// Characters that are inert under NFD, i.e., they do not interact with adjacent characters
make_code_point_set_property! {
property: "NFKC_Inert";
marker: NfkcInertProperty;
keyed_data_marker: NfkcInertV1Marker;
func: /// Characters that are inert under NFKC, i.e., they do not interact with adjacent characters
make_code_point_set_property! {
property: "NFKD_Inert";
marker: NfkdInertProperty;
keyed_data_marker: NfkdInertV1Marker;
func: /// Characters that are inert under NFKD, i.e., they do not interact with adjacent characters
make_code_point_set_property! {
property: "Pattern_White_Space";
marker: PatternWhiteSpaceProperty;
keyed_data_marker: PatternWhiteSpaceV1Marker;
func: /// Characters used as whitespace in patterns (such as regular expressions). See /// [`Unicode Standard Annex #31`](https://www.unicode.org/reports/tr31/tr31-35.html) for /// more details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let pattern_white_space = sets::pattern_white_space(); /// /// assert!(pattern_white_space.contains(' ')); /// assert!(pattern_white_space.contains32(0x2029)); // PARAGRAPH SEPARATOR /// assert!(pattern_white_space.contains32(0x000A)); // NEW LINE /// assert!(!pattern_white_space.contains32(0x00A0)); // NO-BREAK SPACE /// ```
make_code_point_set_property! {
property: "Prepended_Concatenation_Mark";
marker: PrependedConcatenationMarkProperty;
keyed_data_marker: PrependedConcatenationMarkV1Marker;
func: /// A small class of visible format controls, which precede and then span a sequence of /// other characters, usually digits.
make_code_point_set_property! {
property: "Print";
marker: PrintProperty;
keyed_data_marker: PrintV1Marker;
func: /// Printable characters (visible characters and whitespace). /// This is defined for POSIX compatibility.
make_code_point_set_property! {
property: "Regional_Indicator";
marker: RegionalIndicatorProperty;
keyed_data_marker: RegionalIndicatorV1Marker;
func: /// Regional indicator characters, U+1F1E6..U+1F1FF /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let regional_indicator = sets::regional_indicator(); /// /// assert!(regional_indicator.contains('')); // U+1F1F9 REGIONAL INDICATOR SYMBOL LETTER T /// assert!(!regional_indicator.contains('Ⓣ')); // U+24C9 CIRCLED LATIN CAPITAL LETTER T /// assert!(!regional_indicator.contains('T')); /// ```
make_code_point_set_property! {
property: "Soft_Dotted";
marker: SoftDottedProperty;
keyed_data_marker: SoftDottedV1Marker;
func: /// Characters with a "soft dot", like i or j. An accent placed on these characters causes /// the dot to disappear. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let soft_dotted = sets::soft_dotted(); /// /// assert!(soft_dotted.contains('і')); //U+0456 CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I /// assert!(!soft_dotted.contains('ı')); // U+0131 LATIN SMALL LETTER DOTLESS I /// ```
make_code_point_set_property! {
property: "Segment_Starter";
marker: SegmentStarterProperty;
keyed_data_marker: SegmentStarterV1Marker;
func: /// Characters that are starters in terms of Unicode normalization and combining character /// sequences
make_code_point_set_property! {
property: "Case_Sensitive";
marker: CaseSensitiveProperty;
keyed_data_marker: CaseSensitiveV1Marker;
func: /// Characters that are either the source of a case mapping or in the target of a case /// mapping
make_code_point_set_property! {
property: "Sentence_Terminal";
marker: SentenceTerminalProperty;
keyed_data_marker: SentenceTerminalV1Marker;
func: /// Punctuation characters that generally mark the end of sentences /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let sentence_terminal = sets::sentence_terminal(); /// /// assert!(sentence_terminal.contains('.')); /// assert!(sentence_terminal.contains('?')); /// assert!(sentence_terminal.contains('᪨')); // U+1AA8 TAI THAM SIGN KAAN /// assert!(!sentence_terminal.contains(',')); /// assert!(!sentence_terminal.contains('¿')); // U+00BF INVERTED QUESTION MARK /// ```
make_code_point_set_property! {
property: "Terminal_Punctuation";
marker: TerminalPunctuationProperty;
keyed_data_marker: TerminalPunctuationV1Marker;
func: /// Punctuation characters that generally mark the end of textual units /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let terminal_punctuation = sets::terminal_punctuation(); /// /// assert!(terminal_punctuation.contains('.')); /// assert!(terminal_punctuation.contains('?')); /// assert!(terminal_punctuation.contains('᪨')); // U+1AA8 TAI THAM SIGN KAAN /// assert!(terminal_punctuation.contains(',')); /// assert!(!terminal_punctuation.contains('¿')); // U+00BF INVERTED QUESTION MARK /// ```
make_code_point_set_property! {
property: "White_Space";
marker: WhiteSpaceProperty;
keyed_data_marker: WhiteSpaceV1Marker;
func: /// Spaces, separator characters and other control characters which should be treated by /// programming languages as "white space" for the purpose of parsing elements /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let white_space = sets::white_space(); /// /// assert!(white_space.contains(' ')); /// assert!(white_space.contains32(0x000A)); // NEW LINE /// assert!(white_space.contains32(0x00A0)); // NO-BREAK SPACE /// assert!(!white_space.contains32(0x200B)); // ZERO WIDTH SPACE /// ```
make_code_point_set_property! {
property: "Xdigit";
marker: XdigitProperty;
keyed_data_marker: XdigitV1Marker;
func: /// Hexadecimal digits /// This is defined for POSIX compatibility.
make_code_point_set_property! {
property: "XID_Continue";
marker: XidContinueProperty;
keyed_data_marker: XidContinueV1Marker;
func: /// Characters that can come after the first character in an identifier. See [`Unicode Standard Annex /// #31`](https://www.unicode.org/reports/tr31/tr31-35.html) for more details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let xid_continue = sets::xid_continue(); /// /// assert!(xid_continue.contains('x')); /// assert!(xid_continue.contains('1')); /// assert!(xid_continue.contains('_')); /// assert!(xid_continue.contains('ߝ')); // U+07DD NKO LETTER FA /// assert!(!xid_continue.contains('ⓧ')); // U+24E7 CIRCLED LATIN SMALL LETTER X /// assert!(!xid_continue.contains32(0xFC5E)); // ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM /// ```
make_code_point_set_property! {
property: "XID_Start";
marker: XidStartProperty;
keyed_data_marker: XidStartV1Marker;
func: /// Characters that can begin an identifier. See [`Unicode /// Standard Annex #31`](https://www.unicode.org/reports/tr31/tr31-35.html) for more /// details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let xid_start = sets::xid_start(); /// /// assert!(xid_start.contains('x')); /// assert!(!xid_start.contains('1')); /// assert!(!xid_start.contains('_')); /// assert!(xid_start.contains('ߝ')); // U+07DD NKO LETTER FA /// assert!(!xid_start.contains('ⓧ')); // U+24E7 CIRCLED LATIN SMALL LETTER X /// assert!(!xid_start.contains32(0xFC5E)); // ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM /// ```
// // Binary property getter fns // (data as sets of strings + code points) //
macro_rules! make_unicode_set_property {
( // currently unused
property: $property:expr; // currently unused
marker: $marker_name:ident;
keyed_data_marker: $keyed_data_marker:ty;
func:
$(#[$doc:meta])+
$cvis:vis constfn $constname:ident() => $singleton:ident;
$vis:vis fn $funcname:ident();
) => { #[doc = concat!("A version of [`", stringify!($constname), "()`] that uses custom data provided by a [`DataProvider`].")]
$vis fn $funcname(
provider: &(impl DataProvider<$keyed_data_marker> + ?Sized)
) -> Result<UnicodeSetData, PropertiesError> {
Ok(provider.load(Default::default()).and_then(DataResponse::take_payload).map(UnicodeSetData::from_data)?)
}
$(#[$doc])* #[cfg(feature = "compiled_data")]
$cvis constfn $constname() -> UnicodeSetDataBorrowed<'static> {
UnicodeSetDataBorrowed {
set: crate::provider::Baked::$singleton
}
}
}
}
make_unicode_set_property! {
property: "Basic_Emoji";
marker: BasicEmojiProperty;
keyed_data_marker: BasicEmojiV1Marker;
func: /// Characters and character sequences intended for general-purpose, independent, direct input. /// See [`Unicode Technical Standard #51`](https://unicode.org/reports/tr51/) for more /// details. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// # Example /// /// ``` /// use icu::properties::sets; /// /// let basic_emoji = sets::basic_emoji(); /// /// assert!(!basic_emoji.contains32(0x0020)); /// assert!(!basic_emoji.contains_char('\n')); /// assert!(basic_emoji.contains_char('')); // U+1F983 TURKEY /// assert!(basic_emoji.contains("\u{1F983}")); /// assert!(basic_emoji.contains("\u{1F6E4}\u{FE0F}")); // railway track /// assert!(!basic_emoji.contains("\u{0033}\u{FE0F}\u{20E3}")); // Emoji_Keycap_Sequence, keycap 3 /// ``` pubconstfn basic_emoji() => SINGLETON_PROPS_BASIC_EMOJI_V1; pubfn load_basic_emoji();
}
// // Enumerated property getter fns //
/// A version of [`for_general_category_group()`] that uses custom data provided by a [`DataProvider`]. /// /// [ Help choosing a constructor](icu_provider::constructors) pubfn load_for_general_category_group(
provider: &(impl DataProvider<GeneralCategoryV1Marker> + ?Sized),
enum_val: GeneralCategoryGroup,
) -> Result<CodePointSetData, PropertiesError> { let gc_map_payload = maps::load_general_category(provider)?; let gc_map = gc_map_payload.as_borrowed(); let matching_gc_ranges = gc_map
.iter_ranges()
.filter(|cpm_range| (1 << cpm_range.value as u32) & enum_val.0 != 0)
.map(|cpm_range| cpm_range.range); let set = CodePointInversionList::from_iter(matching_gc_ranges);
Ok(CodePointSetData::from_code_point_inversion_list(set))
}
/// Return a [`CodePointSetData`] for a value or a grouping of values of the General_Category property. See [`GeneralCategoryGroup`]. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubfn for_general_category_group(enum_val: GeneralCategoryGroup) -> CodePointSetData { let matching_gc_ranges = maps::general_category()
.iter_ranges()
.filter(|cpm_range| (1 << cpm_range.value as u32) & enum_val.0 != 0)
.map(|cpm_range| cpm_range.range); let set = CodePointInversionList::from_iter(matching_gc_ranges);
CodePointSetData::from_code_point_inversion_list(set)
}
/// Returns a type capable of looking up values for a property specified as a string, as long as it is a /// [binary property listed in ECMA-262][ecma], using strict matching on the names in the spec. /// /// This handles every property required by ECMA-262 `/u` regular expressions, except for: /// /// - `Script` and `General_Category`: handle these directly with [`maps::load_general_category()`] and /// [`maps::load_script()`]. /// using property values parsed via [`GeneralCategory::get_name_to_enum_mapper()`] and [`Script::get_name_to_enum_mapper()`] /// if necessary. /// - `Script_Extensions`: handle this directly using APIs from [`crate::script`], like [`script::load_script_with_extensions_unstable()`] /// - `General_Category` mask values: Handle this alongside `General_Category` using [`GeneralCategoryGroup`], /// using property values parsed via [`GeneralCategoryGroup::get_name_to_enum_mapper()`] if necessary /// - `Assigned`, `All`, and `ASCII` pseudoproperties: Handle these using their equivalent sets: /// - `Any` can be expressed as the range `[\u{0}-\u{10FFFF}]` /// - `Assigned` can be expressed as the inverse of the set `gc=Cn` (i.e., `\P{gc=Cn}`). /// - `ASCII` can be expressed as the range `[\u{0}-\u{7F}]` /// - `General_Category` property values can themselves be treated like properties using a shorthand in ECMA262, /// simply create the corresponding `GeneralCategory` set. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [ Help choosing a constructor](icu_provider::constructors) /// /// ``` /// use icu::properties::sets; /// /// let emoji = sets::load_for_ecma262("Emoji").expect("loading data failed"); /// /// assert!(emoji.contains('')); // U+1F525 FIRE /// assert!(!emoji.contains('V')); /// ``` /// /// [ecma]: https://tc39.es/ecma262/#table-binary-unicode-properties #[cfg(feature = "compiled_data")] pubfn load_for_ecma262(name: &str) -> Result<CodePointSetDataBorrowed<'static>, PropertiesError> { usecrate::runtime::UnicodeProperty;
#[test] fn test_general_category() { use icu::properties::sets; use icu::properties::GeneralCategoryGroup;
let digits_data = sets::for_general_category_group(GeneralCategoryGroup::Number); let digits = digits_data.as_borrowed();
assert!(digits.contains('5'));
assert!(digits.contains('\u{0665}')); // U+0665 ARABIC-INDIC DIGIT FIVE
assert!(digits.contains('\u{096b}')); // U+0969 DEVANAGARI DIGIT FIVE
assert!(!digits.contains('A'));
}
#[test] fn test_script() { use icu::properties::maps; use icu::properties::Script;
let thai_data = maps::script().get_set_for_value(Script::Thai); let thai = thai_data.as_borrowed();
assert!(thai.contains('\u{0e01}')); // U+0E01 THAI CHARACTER KO KAI
assert!(thai.contains('\u{0e50}')); // U+0E50 THAI DIGIT ZERO
assert!(!thai.contains('A'));
assert!(!thai.contains('\u{0e3f}')); // U+0E50 THAI CURRENCY SYMBOL BAHT
}
#[test] fn test_gc_groupings() { use icu::properties::{maps, sets}; use icu::properties::{GeneralCategory, GeneralCategoryGroup}; use icu_collections::codepointinvlist::CodePointInversionListBuilder;
let test_group = |category: GeneralCategoryGroup, subcategories: &[GeneralCategory]| { let category_set = sets::for_general_category_group(category); let category_set = category_set
.as_code_point_inversion_list()
.expect("The data should be valid");
letmut builder = CodePointInversionListBuilder::new(); for subcategory in subcategories { let gc_set_data = &maps::general_category().get_set_for_value(*subcategory); let gc_set = gc_set_data.as_borrowed(); for range in gc_set.iter_ranges() {
builder.add_range32(&range);
}
} let combined_set = builder.build();
println!("{category:?} {subcategories:?}");
assert_eq!(
category_set.get_inversion_list_vec(),
combined_set.get_inversion_list_vec()
);
};
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.