use alloc::{
string::{String, ToString},
vec::Vec,
};
usecrate::hir;
/// An inclusive range of codepoints from a generated file (hence the static /// lifetime). type Range = &'static [(char, char)];
/// An error that occurs when dealing with Unicode. /// /// We don't impl the Error trait here because these always get converted /// into other public errors. (This error type isn't exported.) #[derive(Debug)] pubenum Error {
PropertyNotFound,
PropertyValueNotFound, // Not used when unicode-perl is enabled. #[allow(dead_code)]
PerlClassNotFound,
}
/// An error that occurs when Unicode-aware simple case folding fails. /// /// This error can occur when the case mapping tables necessary for Unicode /// aware case folding are unavailable. This only occurs when the /// `unicode-case` feature is disabled. (The feature is enabled by default.) #[derive(Debug)] pubstruct CaseFoldError(());
#[cfg(feature = "std")] impl std::error::Error for CaseFoldError {}
impl core::fmt::Display for CaseFoldError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
f, "Unicode-aware case folding is not available \
(probably because the unicode-case feature is not enabled)"
)
}
}
/// An error that occurs when the Unicode-aware `\w` class is unavailable. /// /// This error can occur when the data tables necessary for the Unicode aware /// Perl character class `\w` are unavailable. This only occurs when the /// `unicode-perl` feature is disabled. (The feature is enabled by default.) #[derive(Debug)] pubstruct UnicodeWordError(());
#[cfg(feature = "std")] impl std::error::Error for UnicodeWordError {}
impl core::fmt::Display for UnicodeWordError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
f, "Unicode-aware \\w class is not available \
(probably because the unicode-perl feature is not enabled)"
)
}
}
/// A state oriented traverser of the simple case folding table. /// /// A case folder can be constructed via `SimpleCaseFolder::new()`, which will /// return an error if the underlying case folding table is unavailable. /// /// After construction, it is expected that callers will use /// `SimpleCaseFolder::mapping` by calling it with codepoints in strictly /// increasing order. For example, calling it on `b` and then on `a` is illegal /// and will result in a panic. /// /// The main idea of this type is that it tries hard to make mapping lookups /// fast by exploiting the structure of the underlying table, and the ordering /// assumption enables this. #[derive(Debug)] pubstruct SimpleCaseFolder { /// The simple case fold table. It's a sorted association list, where the /// keys are Unicode scalar values and the values are the corresponding /// equivalence class (not including the key) of the "simple" case folded /// Unicode scalar values.
table: &'static [(char, &'static [char])], /// The last codepoint that was used for a lookup.
last: Option<char>, /// The index to the entry in `table` corresponding to the smallest key `k` /// such that `k > k0`, where `k0` is the most recent key lookup. Note that /// in particular, `k0` may not be in the table!
next: usize,
}
impl SimpleCaseFolder { /// Create a new simple case folder, returning an error if the underlying /// case folding table is unavailable. pubfn new() -> Result<SimpleCaseFolder, CaseFoldError> { #[cfg(not(feature = "unicode-case"))]
{
Err(CaseFoldError(()))
} #[cfg(feature = "unicode-case")]
{
Ok(SimpleCaseFolder {
table: crate::unicode_tables::case_folding_simple::CASE_FOLDING_SIMPLE,
last: None,
next: 0,
})
}
}
/// Return the equivalence class of case folded codepoints for the given /// codepoint. The equivalence class returned never includes the codepoint /// given. If the given codepoint has no case folded codepoints (i.e., /// no entry in the underlying case folding table), then this returns an /// empty slice. /// /// # Panics /// /// This panics when called with a `c` that is less than or equal to the /// previous call. In other words, callers need to use this method with /// strictly increasing values of `c`. pubfn mapping(&mutself, c: char) -> &'static [char] { iflet Some(last) = self.last {
assert!(
last < c, "got codepoint U+{:X} which occurs before \
last codepoint U+{:X}",
u32::from(c),
u32::from(last),
);
} self.last = Some(c); ifself.next >= self.table.len() { return &[];
} let (k, v) = self.table[self.next]; if k == c { self.next += 1; return v;
} matchself.get(c) {
Err(i) => { self.next = i;
&[]
}
Ok(i) => { // Since we require lookups to proceed // in order, anything we find should be // after whatever we thought might be // next. Otherwise, the caller is either // going out of order or we would have // found our next key at 'self.next'.
assert!(i > self.next); self.next = i + 1; self.table[i].1
}
}
}
/// Returns true if and only if the given range overlaps with any region /// of the underlying case folding table. That is, when true, there exists /// at least one codepoint in the inclusive range `[start, end]` that has /// a non-trivial equivalence class of case folded codepoints. Conversely, /// when this returns false, all codepoints in the range `[start, end]` /// correspond to the trivial equivalence class of case folded codepoints, /// i.e., itself. /// /// This is useful to call before iterating over the codepoints in the /// range and looking up the mapping for each. If you know none of the /// mappings will return anything, then you might be able to skip doing it /// altogether. /// /// # Panics /// /// This panics when `end < start`. pubfn overlaps(&self, start: char, end: char) -> bool { use core::cmp::Ordering;
assert!(start <= end); self.table
.binary_search_by(|&(c, _)| { if start <= c && c <= end {
Ordering::Equal
} elseif c > end {
Ordering::Greater
} else {
Ordering::Less
}
})
.is_ok()
}
/// Returns the index at which `c` occurs in the simple case fold table. If /// `c` does not occur, then this returns an `i` such that `table[i-1].0 < /// c` and `table[i].0 > c`. fn get(&self, c: char) -> Result<usize, usize> { self.table.binary_search_by_key(&c, |&(c1, _)| c1)
}
}
/// A query for finding a character class defined by Unicode. This supports /// either use of a property name directly, or lookup by property value. The /// former generally refers to Binary properties (see UTS#44, Table 8), but /// as a special exception (see UTS#18, Section 1.2) both general categories /// (an enumeration) and scripts (a catalog) are supported as if each of their /// possible values were a binary property. /// /// In all circumstances, property names and values are normalized and /// canonicalized. That is, `GC == gc == GeneralCategory == general_category`. /// /// The lifetime `'a` refers to the shorter of the lifetimes of property name /// and property value. #[derive(Debug)] pubenum ClassQuery<'a> { /// Return a class corresponding to a Unicode binary property, named by /// a single letter.
OneLetter(char), /// Return a class corresponding to a Unicode binary property. /// /// Note that, by special exception (see UTS#18, Section 1.2), both /// general category values and script values are permitted here as if /// they were a binary property.
Binary(&'a str), /// Return a class corresponding to all codepoints whose property /// (identified by `property_name`) corresponds to the given value /// (identified by `property_value`).
ByValue { /// A property name.
property_name: &'a str, /// A property value.
property_value: &'a str,
},
}
// This is a special case where 'cf' refers to the 'Format' general // category, but where the 'cf' abbreviation is also an abbreviation // for the 'Case_Folding' property. But we want to treat it as // a general category. (Currently, we don't even support the // 'Case_Folding' property. But if we do in the future, users will be // required to spell it out.) // // Also 'sc' refers to the 'Currency_Symbol' general category, but is // also the abbreviation for the 'Script' property. So we avoid calling // 'canonical_prop' for it too, which would erroneously normalize it // to 'Script'. // // Another case: 'lc' is an abbreviation for the 'Cased_Letter' // general category, but is also an abbreviation for the 'Lowercase_Mapping' // property. We don't currently support the latter, so as with 'cf' // above, we treat 'lc' as 'Cased_Letter'. if norm != "cf" && norm != "sc" && norm != "lc" { iflet Some(canon) = canonical_prop(&norm)? { return Ok(CanonicalClassQuery::Binary(canon));
}
} iflet Some(canon) = canonical_gencat(&norm)? { return Ok(CanonicalClassQuery::GeneralCategory(canon));
} iflet Some(canon) = canonical_script(&norm)? { return Ok(CanonicalClassQuery::Script(canon));
}
Err(Error::PropertyNotFound)
}
}
/// Like ClassQuery, but its parameters have been canonicalized. This also /// differentiates binary properties from flattened general categories and /// scripts. #[derive(Debug, Eq, PartialEq)] enum CanonicalClassQuery { /// The canonical binary property name.
Binary(&'static str), /// The canonical general category name.
GeneralCategory(&'static str), /// The canonical script name.
Script(&'static str), /// An arbitrary association between property and value, both of which /// have been canonicalized. /// /// Note that by construction, the property name of ByValue will never /// be General_Category or Script. Those two cases are subsumed by the /// eponymous variants.
ByValue { /// The canonical property name.
property_name: &'static str, /// The canonical property value.
property_value: &'static str,
},
}
/// Looks up a Unicode class given a query. If one doesn't exist, then /// `None` is returned. pubfn class(query: ClassQuery<'_>) -> Result<hir::ClassUnicode, Error> { useself::CanonicalClassQuery::*;
/// Returns a Unicode aware class for \w. /// /// This returns an error if the data is not available for \w. pubfn perl_word() -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-perl"))] fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
/// Returns a Unicode aware class for \s. /// /// This returns an error if the data is not available for \s. pubfn perl_space() -> Result<hir::ClassUnicode, Error> { #[cfg(not(any(feature = "unicode-perl", feature = "unicode-bool")))] fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
/// Returns a Unicode aware class for \d. /// /// This returns an error if the data is not available for \d. pubfn perl_digit() -> Result<hir::ClassUnicode, Error> { #[cfg(not(any(feature = "unicode-perl", feature = "unicode-gencat")))] fn imp() -> Result<hir::ClassUnicode, Error> {
Err(Error::PerlClassNotFound)
}
/// Build a Unicode HIR class from a sequence of Unicode scalar value ranges. pubfn hir_class(ranges: &[(char, char)]) -> hir::ClassUnicode { let hir_ranges: Vec<hir::ClassUnicodeRange> = ranges
.iter()
.map(|&(s, e)| hir::ClassUnicodeRange::new(s, e))
.collect();
hir::ClassUnicode::new(hir_ranges)
}
/// Returns true only if the given codepoint is in the `\w` character class. /// /// If the `unicode-perl` feature is not enabled, then this returns an error. pubfn is_word_character(c: char) -> Result<bool, UnicodeWordError> { #[cfg(not(feature = "unicode-perl"))] fn imp(_: char) -> Result<bool, UnicodeWordError> {
Err(UnicodeWordError(()))
}
if u8::try_from(c).map_or(false, is_word_byte) { return Ok(true);
}
Ok(PERL_WORD
.binary_search_by(|&(start, end)| { use core::cmp::Ordering;
if start <= c && c <= end {
Ordering::Equal
} elseif start > c {
Ordering::Greater
} else {
Ordering::Less
}
})
.is_ok())
}
imp(c)
}
/// A mapping of property values for a specific property. /// /// The first element of each tuple is a normalized property value while the /// second element of each tuple is the corresponding canonical property /// value. type PropertyValues = &'static [(&'static str, &'static str)];
/// Find the canonical property name for the given normalized property name. /// /// If no such property exists, then `None` is returned. /// /// The normalized property name must have been normalized according to /// UAX44 LM3, which can be done using `symbolic_name_normalize`. /// /// If the property names data is not available, then an error is returned. fn canonical_prop(
normalized_name: &str,
) -> Result<Option<&'static str>, Error> { #[cfg(not(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
)))] fn imp(_: &str) -> Result<Option<&'static str>, Error> {
Err(Error::PropertyNotFound)
}
/// Find the canonical property value for the given normalized property /// value. /// /// The given property values should correspond to the values for the property /// under question, which can be found using `property_values`. /// /// If no such property value exists, then `None` is returned. /// /// The normalized property value must have been normalized according to /// UAX44 LM3, which can be done using `symbolic_name_normalize`. fn canonical_value(
vals: PropertyValues,
normalized_value: &str,
) -> Option<&'static str> {
vals.binary_search_by_key(&normalized_value, |&(n, _)| n)
.ok()
.map(|i| vals[i].1)
}
/// Return the table of property values for the given property name. /// /// If the property values data is not available, then an error is returned. fn property_values(
canonical_property_name: &'static str,
) -> Result<Option<PropertyValues>, Error> { #[cfg(not(any(
feature = "unicode-age",
feature = "unicode-bool",
feature = "unicode-gencat",
feature = "unicode-perl",
feature = "unicode-script",
feature = "unicode-segment",
)))] fn imp(_: &'static str) -> Result<Option<PropertyValues>, Error> {
Err(Error::PropertyValueNotFound)
}
// This is only used in some cases, but small enough to just let it be dead // instead of figuring out (and maintaining) the right set of features. #[allow(dead_code)] fn property_set(
name_map: &'static [(&'static str, Range)],
canonical: &'static str,
) -> Option<Range> {
name_map
.binary_search_by_key(&canonical, |x| x.0)
.ok()
.map(|i| name_map[i].1)
}
/// Returns an iterator over Unicode Age sets. Each item corresponds to a set /// of codepoints that were added in a particular revision of Unicode. The /// iterator yields items in chronological order. /// /// If the given age value isn't valid or if the data isn't available, then an /// error is returned instead. fn ages(canonical_age: &str) -> Result<impl Iterator<Item = Range>, Error> { #[cfg(not(feature = "unicode-age"))] fn imp(_: &str) -> Result<impl Iterator<Item = Range>, Error> { use core::option::IntoIter;
Err::<IntoIter<Range>, _>(Error::PropertyNotFound)
}
/// Returns the Unicode HIR class corresponding to the given general category. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given general category could not be found, or if the general /// category data is not available, then an error is returned. fn gencat(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-gencat"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
match canonical_name { "Decimal_Number" => perl_digit(),
name => imp(name),
}
}
/// Returns the Unicode HIR class corresponding to the given script. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given script could not be found, or if the script data is not /// available, then an error is returned. fn script(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-script"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
/// Returns the Unicode HIR class corresponding to the given script extension. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given script extension could not be found, or if the script data is /// not available, then an error is returned. fn script_extension(
canonical_name: &'static str,
) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-script"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
/// Returns the Unicode HIR class corresponding to the given Unicode boolean /// property. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given boolean property could not be found, or if the boolean /// property data is not available, then an error is returned. fn bool_property(
canonical_name: &'static str,
) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-bool"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
match canonical_name { "Decimal_Number" => perl_digit(), "White_Space" => perl_space(),
name => imp(name),
}
}
/// Returns the Unicode HIR class corresponding to the given grapheme cluster /// break property. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given property could not be found, or if the corresponding data is /// not available, then an error is returned. fn gcb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-segment"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
/// Returns the Unicode HIR class corresponding to the given word break /// property. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given property could not be found, or if the corresponding data is /// not available, then an error is returned. fn wb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-segment"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
/// Returns the Unicode HIR class corresponding to the given sentence /// break property. /// /// Name canonicalization is assumed to be performed by the caller. /// /// If the given property could not be found, or if the corresponding data is /// not available, then an error is returned. fn sb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { #[cfg(not(feature = "unicode-segment"))] fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> {
Err(Error::PropertyNotFound)
}
/// Like symbolic_name_normalize_bytes, but operates on a string. fn symbolic_name_normalize(x: &str) -> String { letmut tmp = x.as_bytes().to_vec(); let len = symbolic_name_normalize_bytes(&mut tmp).len();
tmp.truncate(len); // This should always succeed because `symbolic_name_normalize_bytes` // guarantees that `&tmp[..len]` is always valid UTF-8. // // N.B. We could avoid the additional UTF-8 check here, but it's unlikely // to be worth skipping the additional safety check. A benchmark must // justify it first.
String::from_utf8(tmp).unwrap()
}
/// Normalize the given symbolic name in place according to UAX44-LM3. /// /// A "symbolic name" typically corresponds to property names and property /// value aliases. Note, though, that it should not be applied to property /// string values. /// /// The slice returned is guaranteed to be valid UTF-8 for all possible values /// of `slice`. /// /// See: https://unicode.org/reports/tr44/#UAX44-LM3 fn symbolic_name_normalize_bytes(slice: &mut [u8]) -> &='color:red'>mut [u8] { // I couldn't find a place in the standard that specified that property // names/aliases had a particular structure (unlike character names), but // we assume that it's ASCII only and drop anything that isn't ASCII. letmut start = 0; letmut starts_with_is = false; if slice.len() >= 2 { // Ignore any "is" prefix.
starts_with_is = slice[0..2] == b"is"[..]
|| slice[0..2] == b"IS"[..]
|| slice[0..2] == b"iS"[..]
|| slice[0..2] == b"Is"[..]; if starts_with_is {
start = 2;
}
} letmut next_write = 0; for i in start..slice.len() { // VALIDITY ARGUMENT: To guarantee that the resulting slice is valid // UTF-8, we ensure that the slice contains only ASCII bytes. In // particular, we drop every non-ASCII byte from the normalized string. let b = slice[i]; if b == b' ' || b == b'_' || b == b'-' { continue;
} elseif b'A' <= b && b <= b'Z' {
slice[next_write] = b + (b'a' - b'A');
next_write += 1;
} elseif b <= 0x7F {
slice[next_write] = b;
next_write += 1;
}
} // Special case: ISO_Comment has a 'isc' abbreviation. Since we generally // ignore 'is' prefixes, the 'isc' abbreviation gets caught in the cross // fire and ends up creating an alias for 'c' to 'ISO_Comment', but it // is actually an alias for the 'Other' general category. if starts_with_is && next_write == 1 && slice[0] == b'c' {
slice[0] = b'i';
slice[1] = b's';
slice[2] = b'c';
next_write = 3;
}
&mut slice[..next_write]
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.