usecrate::complex::ComplexPayloadsBorrowed; usecrate::indices::{Latin1Indices, Utf16Indices}; usecrate::options::WordType; usecrate::provider::*; use core::str::CharIndices; use utf8_iter::Utf8CharIndices;
/// A trait allowing for RuleBreakIterator to be generalized to multiple string /// encoding methods and granularity such as grapheme cluster, word, etc. /// /// <div class="stab unstable"> /// This trait is sealed; it cannot be implemented by user code. If an API requests an item that implements this /// trait, please consider using a type from the implementors listed below. /// </div> pubtrait RuleBreakType: crate::private::Sealed + Sized { /// The iterator over characters. type IterAttr<'s>: Iterator<Item = (usize, Self::CharType)> + Clone + core::fmt::Debug;
/// The character type. type CharType: Copy + Into<u32> + core::fmt::Debug;
/// Implements the [`Iterator`] trait over the segmenter boundaries of the given string. /// /// Lifetimes: /// /// - `'l` = lifetime of the segmenter object from which this iterator was created /// - `'data` = lifetime of data borrowed by segmenter object /// (this largely exists because segmenter data is invariant due to ZeroMap constraints, /// think of it as a second 'l) /// - `'s` = lifetime of the string being segmented /// /// The [`Iterator::Item`] is an [`usize`] representing index of a code unit /// _after_ the boundary (for a boundary at the end of text, this index is the length /// of the [`str`] or array of code units). #[derive(Debug)] pubstruct RuleBreakIterator<'data, 's, Y: RuleBreakType> { pub(crate) iter: Y::IterAttr<'s>, pub(crate) len: usize, pub(crate) current_pos_data: Option<(usize, Y::CharType)>, pub(crate) result_cache: alloc::vec::Vec<usize>, pub(crate) data: &'data RuleBreakData<'data>, pub(crate) complex: Option<ComplexPayloadsBorrowed<'data>>, pub(crate) boundary_property: u8, pub(crate) locale_override: Option<&'data RuleBreakDataOverride<'data>>, // Should return None if there is no complex language handling pub(crate) handle_complex_language: fn(&mut RuleBreakIterator<'data, 's, Y>, Y::CharType) -> Option<usize>,
}
pub(crate) fn empty_handle_complex_language<Y: RuleBreakType>(
_i: &mut RuleBreakIterator<'_, '_, Y>,
_c: Y::CharType,
) -> Option<usize> {
debug_assert!( false, "grapheme/sentence segmenters should never need complex language handling"
);
None
}
impl<Y: RuleBreakType> Iterator for RuleBreakIterator<'_, '_, Y> { type Item = usize;
fn next(&mutself) -> Option<Self::Item> { // If we have break point cache by previous run, return this result iflet Some(&first_result) = self.result_cache.first() { letmut i = 0; loop { if i == first_result { self.result_cache = self.result_cache.iter().skip(1).map(|r| r - i).collect(); returnself.get_current_position();
}
i += self.get_current_codepoint().map_or(0, Y::char_len); self.advance_iter(); ifself.is_eof() { self.result_cache.clear(); self.boundary_property = self.data.complex_property; return Some(self.len);
}
}
}
ifself.is_eof() { self.advance_iter(); ifself.is_eof() && self.len == 0 { // Empty string. Since `self.current_pos_data` is always going to be empty, // we never read `self.len` except for here, so we can use it to mark that // we have already returned the single empty-string breakpoint. self.len = 1; return Some(0);
} let Some(right_prop) = self.get_current_break_property() else { // iterator already reaches to EOT. Reset boundary property for word-like. self.boundary_property = 0; return None;
}; // SOT x anything if matches!( self.get_break_state_from_table(self.data.sot_property, right_prop),
BreakState::Break | BreakState::NoMatch
) { self.boundary_property = 0; // SOT is special type returnself.get_current_position();
}
}
'a: loop {
debug_assert!(!self.is_eof()); let left_codepoint = self.get_current_codepoint()?; let left_prop = self.get_break_property(left_codepoint); self.advance_iter();
// Some segmenter rules doesn't have language-specific rules, we have to use LSTM (or dictionary) segmenter. // If property is marked as SA, use it if right_prop == self.data.complex_property { if left_prop != self.data.complex_property { // break before SA self.boundary_property = left_prop; returnself.get_current_position();
} let break_offset = (self.handle_complex_language)(self, left_codepoint); if break_offset.is_some() { return break_offset;
}
}
matchself.get_break_state_from_table(left_prop, right_prop) {
BreakState::Keep => continue,
BreakState::Break | BreakState::NoMatch => { self.boundary_property = left_prop; returnself.get_current_position();
}
BreakState::Index(mut index) | BreakState::Intermediate(mut index) => { // This isn't simple rule set. We need marker to restore iterator to previous position. letmut previous_iter = self.iter.clone(); letmut previous_pos_data = self.current_pos_data; letmut previous_left_prop = left_prop;
loop { self.advance_iter();
let Some(prop) = self.get_current_break_property() else { // Reached EOF. But we are analyzing multiple characters now, so next break may be previous point. self.boundary_property = index; ifself.get_break_state_from_table(index, self.data.eot_property)
== BreakState::NoMatch
{ self.boundary_property = previous_left_prop; self.iter = previous_iter; self.current_pos_data = previous_pos_data; returnself.get_current_position();
} // EOF return Some(self.len);
};
let previous_break_state_is_cp_prop =
index <= self.data.last_codepoint_property;
fn get_break_property(&self, codepoint: Y::CharType) -> u8 { // Note: Default value is 0 == UNKNOWN iflet Some(locale_override) = &self.locale_override { let property = locale_override
.property_table_override
.get32(codepoint.into()); if property != 0 { return property;
}
} self.data.property_table.get32(codepoint.into())
}
fn get_break_state_from_table(&self, left: u8, right: u8) -> BreakState { let idx = left as usize * self.data.property_count as usize + right as usize; // We use unwrap_or to fall back to the base case and prevent panics on bad data. self.data
.break_state_table
.get(idx)
.unwrap_or(BreakState::Keep)
}
/// Return the status value of break boundary. /// If segmenter isn't word, always return WordType::None pubfn word_type(&self) -> WordType { if !self.result_cache.is_empty() { // Dictionary type (CJ and East Asian) is letter. return WordType::Letter;
} ifself.boundary_property == 0 { // break position is SOT / Any return WordType::None;
} self.data
.word_type_table
.get((self.boundary_property - 1) as usize)
.unwrap_or(WordType::None)
}
/// Return true when break boundary is word-like such as letter/number/CJK /// If segmenter isn't word, return false pubfn is_word_like(&self) -> bool { self.word_type().is_word_like()
}
}
#[derive(Debug)] #[non_exhaustive] /// [`RuleBreakType`] for UTF-8 strings pubstruct Utf8;
implcrate::private::Sealed for Utf8 {}
impl RuleBreakType for Utf8 { type IterAttr<'s> = CharIndices<'s>; type CharType = char;
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.