usecrate::complex::*; usecrate::indices::*; usecrate::provider::*; usecrate::SegmenterError; use alloc::string::String; use alloc::vec; use alloc::vec::Vec; use core::char; use core::str::CharIndices; use icu_provider::prelude::*; use utf8_iter::Utf8CharIndices;
/// An enum specifies the strictness of line-breaking rules. It can be passed as /// an argument when creating a line segmenter. /// /// Each enum value has the same meaning with respect to the `line-break` /// property values in the CSS Text spec. See the details in /// <https://drafts.csswg.org/css-text-3/#line-break-property>. #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug)] pubenum LineBreakStrictness { /// Breaks text using the least restrictive set of line-breaking rules. /// Typically used for short lines, such as in newspapers. /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-loose>
Loose,
/// Breaks text assuming there is a soft wrap opportunity around every /// typographic character unit, disregarding any prohibition against line /// breaks. See more details in /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-anywhere>.
Anywhere,
}
/// An enum specifies the line break opportunities between letters. It can be /// passed as an argument when creating a line segmenter. /// /// Each enum value has the same meaning with respect to the `word-break` /// property values in the CSS Text spec. See the details in /// <https://drafts.csswg.org/css-text-3/#word-break-property> #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug)] pubenum LineBreakWordOption { /// Words break according to their customary rules. See the details in /// <https://drafts.csswg.org/css-text-3/#valdef-word-break-normal>.
Normal,
/// Options to tailor line-breaking behavior. #[non_exhaustive] #[derive(Copy, Clone, PartialEq, Eq, Debug)] pubstruct LineBreakOptions { /// Strictness of line-breaking rules. See [`LineBreakStrictness`]. pub strictness: LineBreakStrictness,
/// Line break opportunities between letters. See [`LineBreakWordOption`]. pub word_option: LineBreakWordOption,
/// Use `true` as a hint to the line segmenter that the writing /// system is Chinese or Japanese. This allows more break opportunities when /// `LineBreakStrictness` is `Normal` or `Loose`. See /// <https://drafts.csswg.org/css-text-3/#line-break-property> for details. /// /// This option has no effect in Latin-1 mode. pub ja_zh: bool,
}
/// Line break iterator for an `str` (a UTF-8 string). /// /// For examples of use, see [`LineSegmenter`]. pubtype LineBreakIteratorUtf8<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeUtf8>;
/// Line break iterator for a potentially invalid UTF-8 string. /// /// For examples of use, see [`LineSegmenter`]. pubtype LineBreakIteratorPotentiallyIllFormedUtf8<'l, 's> =
LineBreakIterator<'l, 's, LineBreakTypePotentiallyIllFormedUtf8>;
/// Line break iterator for a Latin-1 (8-bit) string. /// /// For examples of use, see [`LineSegmenter`]. pubtype LineBreakIteratorLatin1<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeLatin1>;
/// Line break iterator for a UTF-16 string. /// /// For examples of use, see [`LineSegmenter`]. pubtype LineBreakIteratorUtf16<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeUtf16>;
/// Supports loading line break data, and creating line break iterators for different string /// encodings. /// /// The segmenter returns mandatory breaks (as defined by [definition LD7][LD7] of /// Unicode Standard Annex #14, _Unicode Line Breaking Algorithm_) as well as /// line break opportunities ([definition LD3][LD3]). /// It does not distinguish them. Callers requiring that distinction can check /// the Line_Break property of the code point preceding the break against those /// listed in rules [LB4][LB4] and [LB5][LB5], special-casing the end of text /// according to [LB3][LB3]. /// /// For consistency with the grapheme, word, and sentence segmenters, there is /// always a breakpoint returned at index 0, but this breakpoint is not a /// meaningful line break opportunity. /// /// [LD3]: https://www.unicode.org/reports/tr14/#LD3 /// [LD7]: https://www.unicode.org/reports/tr14/#LD7 /// [LB3]: https://www.unicode.org/reports/tr14/#LB3 /// [LB4]: https://www.unicode.org/reports/tr14/#LB4 /// [LB5]: https://www.unicode.org/reports/tr14/#LB5 /// /// ```rust /// # use icu::segmenter::LineSegmenter; /// # /// # let segmenter = LineSegmenter::new_auto(); /// # /// let text = "Summary\r\nThis annex…"; /// let breakpoints: Vec<usize> = segmenter.segment_str(text).collect(); /// // 9 and 22 are mandatory breaks, 14 is a line break opportunity. /// assert_eq!(&breakpoints, &[0, 9, 14, 22]); /// /// // There is a break opportunity between emoji, but not within the ZWJ sequence ��️��. /// let flag_equation = "��️➕������️\u{200D}��"; /// let possible_first_lines: Vec<&str> = /// segmenter.segment_str(flag_equation).skip(1).map(|i| &flag_equation[..i]).collect(); /// assert_eq!( /// &possible_first_lines, /// &[ /// "��️", /// "��️➕", /// "��️➕��", /// "��️➕����", /// "��️➕������️��" /// ] /// ); /// ``` /// /// # Examples /// /// Segment a string with default options: /// /// ```rust /// use icu::segmenter::LineSegmenter; /// /// let segmenter = LineSegmenter::new_auto(); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_str("Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 6, 11]); /// ``` /// /// Segment a string with CSS option overrides: /// /// ```rust /// use icu::segmenter::{ /// LineBreakOptions, LineBreakStrictness, LineBreakWordOption, /// LineSegmenter, /// }; /// /// let mut options = LineBreakOptions::default(); /// options.strictness = LineBreakStrictness::Strict; /// options.word_option = LineBreakWordOption::BreakAll; /// options.ja_zh = false; /// let segmenter = LineSegmenter::new_auto_with_options(options); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_str("Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11]); /// ``` /// /// Segment a Latin1 byte string: /// /// ```rust /// use icu::segmenter::LineSegmenter; /// /// let segmenter = LineSegmenter::new_auto(); /// /// let breakpoints: Vec<usize> = /// segmenter.segment_latin1(b"Hello World").collect(); /// assert_eq!(&breakpoints, &[0, 6, 11]); /// ``` /// /// Separate mandatory breaks from the break opportunities: /// /// ```rust /// use icu::properties::{maps, LineBreak}; /// use icu::segmenter::LineSegmenter; /// /// # let segmenter = LineSegmenter::new_auto(); /// # /// let text = "Summary\r\nThis annex…"; /// /// let mandatory_breaks: Vec<usize> = segmenter /// .segment_str(text) /// .into_iter() /// .filter(|&i| { /// text[..i].chars().next_back().map_or(false, |c| { /// matches!( /// maps::line_break().get(c), /// LineBreak::MandatoryBreak /// | LineBreak::CarriageReturn /// | LineBreak::LineFeed /// | LineBreak::NextLine /// ) || i == text.len() /// }) /// }) /// .collect(); /// assert_eq!(&mandatory_breaks, &[9, 22]); /// ``` #[derive(Debug)] pubstruct LineSegmenter {
options: LineBreakOptions,
payload: DataPayload<LineBreakDataV1Marker>,
complex: ComplexPayloads,
}
impl LineSegmenter { /// Constructs a [`LineSegmenter`] with an invariant locale and the best available compiled data for /// complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The current behavior, which is subject to change, is to use the LSTM model when available. /// /// See also [`Self::new_auto_with_options`]. /// /// ✨ *Enabled with the `compiled_data` and `auto` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] #[cfg(feature = "auto")] pubfn new_auto() -> Self { Self::new_auto_with_options(Default::default())
}
/// Constructs a [`LineSegmenter`] with an invariant locale and compiled LSTM data for /// complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The LSTM, or Long Term Short Memory, is a machine learning model. It is smaller than /// the full dictionary but more expensive during segmentation (inference). /// /// See also [`Self::new_lstm_with_options`]. /// /// ✨ *Enabled with the `compiled_data` and `lstm` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] #[cfg(feature = "lstm")] pubfn new_lstm() -> Self { Self::new_lstm_with_options(Default::default())
}
/// Constructs a [`LineSegmenter`] with an invariant locale and compiled dictionary data for /// complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The dictionary model uses a list of words to determine appropriate breakpoints. It is /// faster than the LSTM model but requires more data. /// /// See also [`Self::new_dictionary_with_options`]. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubfn new_dictionary() -> Self { Self::new_dictionary_with_options(Default::default())
}
/// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// the best available compiled data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The current behavior, which is subject to change, is to use the LSTM model when available. /// /// See also [`Self::new_auto`]. /// /// ✨ *Enabled with the `compiled_data` and `auto` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "auto")] #[cfg(feature = "compiled_data")] pubfn new_auto_with_options(options: LineBreakOptions) -> Self { Self::new_lstm_with_options(options)
}
/// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// compiled LSTM data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The LSTM, or Long Term Short Memory, is a machine learning model. It is smaller than /// the full dictionary but more expensive during segmentation (inference). /// /// See also [`Self::new_dictionary`]. /// /// ✨ *Enabled with the `compiled_data` and `lstm` Cargo features.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "lstm")] #[cfg(feature = "compiled_data")] pubfn new_lstm_with_options(options: LineBreakOptions) -> Self { Self {
options,
payload: DataPayload::from_static_ref( crate::provider::Baked::SINGLETON_SEGMENTER_LINE_V1,
),
complex: ComplexPayloads::new_lstm(),
}
}
/// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and /// compiled dictionary data for complex scripts (Khmer, Lao, Myanmar, and Thai). /// /// The dictionary model uses a list of words to determine appropriate breakpoints. It is /// faster than the LSTM model but requires more data. /// /// See also [`Self::new_dictionary`]. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* /// /// [�� Help choosing a constructor](icu_provider::constructors) #[cfg(feature = "compiled_data")] pubfn new_dictionary_with_options(options: LineBreakOptions) -> Self { Self {
options,
payload: DataPayload::from_static_ref( crate::provider::Baked::SINGLETON_SEGMENTER_LINE_V1,
), // Line segmenter doesn't need to load CJ dictionary because UAX 14 rules handles CJK // characters [1]. Southeast Asian languages however require complex context analysis // [2]. // // [1]: https://www.unicode.org/reports/tr14/#ID // [2]: https://www.unicode.org/reports/tr14/#SA
complex: ComplexPayloads::new_southeast_asian(),
}
}
#[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_dictionary_with_options)] pubfn try_new_dictionary_with_options_unstable<D>(
provider: &D,
options: LineBreakOptions,
) -> Result<Self, SegmenterError> where
D: DataProvider<LineBreakDataV1Marker>
+ DataProvider<DictionaryForWordLineExtendedV1Marker>
+ DataProvider<GraphemeClusterBreakDataV1Marker>
+ ?Sized,
{
Ok(Self {
options,
payload: provider.load(Default::default())?.take_payload()?, // Line segmenter doesn't need to load CJ dictionary because UAX 14 rules handles CJK // characters [1]. Southeast Asian languages however require complex context analysis // [2]. // // [1]: https://www.unicode.org/reports/tr14/#ID // [2]: https://www.unicode.org/reports/tr14/#SA
complex: ComplexPayloads::try_new_southeast_asian(provider)?,
})
}
/// Creates a line break iterator for an `str` (a UTF-8 string). /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_str<'l, 's>(&'l self, input: &'s str) -> LineBreakIteratorUtf8<'l, 's> {
LineBreakIterator {
iter: input.char_indices(),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.payload.get(),
options: &self.options,
complex: &self.complex,
}
} /// Creates a line break iterator for a potentially ill-formed UTF8 string /// /// Invalid characters are treated as REPLACEMENT CHARACTER /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_utf8<'l, 's>(
&'l self,
input: &'s [u8],
) -> LineBreakIteratorPotentiallyIllFormedUtf8<'l, 's> {
LineBreakIterator {
iter: Utf8CharIndices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.payload.get(),
options: &self.options,
complex: &self.complex,
}
} /// Creates a line break iterator for a Latin-1 (8-bit) string. /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_latin1<'l, 's>(&'l self, input: &'s [u8]) -> LineBreakIteratorLatin1<'l, 's> {
LineBreakIterator {
iter: Latin1Indices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.payload.get(),
options: &self.options,
complex: &self.complex,
}
}
/// Creates a line break iterator for a UTF-16 string. /// /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string. pubfn segment_utf16<'l, 's>(&'l self, input: &'s [u16]) -> LineBreakIteratorUtf16<'l, 's> {
LineBreakIterator {
iter: Utf16Indices::new(input),
len: input.len(),
current_pos_data: None,
result_cache: Vec::new(),
data: self.payload.get(),
options: &self.options,
complex: &self.complex,
}
}
}
impl RuleBreakDataV1<'_> { fn get_linebreak_property_utf32_with_rule(
&self,
codepoint: u32,
strictness: LineBreakStrictness,
word_option: LineBreakWordOption,
) -> u8 { // Note: Default value is 0 == UNKNOWN let prop = self.property_table.get32(codepoint);
if word_option == LineBreakWordOption::BreakAll
|| strictness == LineBreakStrictness::Loose
|| strictness == LineBreakStrictness::Normal
{ returnmatch prop {
CJ => ID, // All CJ's General_Category is Other_Letter (Lo).
_ => prop,
};
}
#[inline] fn get_break_state_from_table(&self, left: u8, right: u8) -> BreakState { let idx = (left as usize) * (self.property_count as usize) + (right as usize); // We use unwrap_or to fall back to the base case and prevent panics on bad data. self.break_state_table.get(idx).unwrap_or(BreakState::Keep)
}
// breaks before certain centered punctuation marks: if right_codepoint == 0x30FB
|| right_codepoint == 0xFF1A
|| right_codepoint == 0xFF1B
|| right_codepoint == 0xFF65
|| right_codepoint == 0x203C
|| (0x2047..=0x2049).contains(&right_codepoint)
{ return Some(ja_zh);
}
} elseif right_prop == IN { // breaks between inseparable characters such as U+2025, U+2026 i.e. characters with the Unicode Line Break property IN return Some(true);
} elseif right_prop == EX { // breaks before certain centered punctuation marks: if right_codepoint == 0xFF01 || right_codepoint == 0xFF1F { return Some(ja_zh);
}
}
// breaks before suffixes: // Characters with the Unicode Line Break property PO and the East Asian Width property if right_prop == PO_EAW { return Some(ja_zh);
} // breaks after prefixes: // Characters with the Unicode Line Break property PR and the East Asian Width property if left_prop == PR_EAW { return Some(ja_zh);
}
None
}
/// A trait allowing for LineBreakIterator to be generalized to multiple string iteration methods. /// /// This is implemented by ICU4X for several common string types. pubtrait LineBreakType<'l, 's> { /// The iterator over characters. type IterAttr: Iterator<Item = (usize, Self::CharType)> + Clone;
/// The character type. type CharType: Copy + Into<u32>;
/// Implements the [`Iterator`] trait over the line break opportunities of the given string. /// /// Lifetimes: /// /// - `'l` = lifetime of the [`LineSegmenter`] object from which this iterator was created /// - `'s` = lifetime of the string being segmented /// /// The [`Iterator::Item`] is an [`usize`] representing index of a code unit /// _after_ the break (for a break at the end of text, this index is the length /// of the [`str`] or array of code units). /// /// For examples of use, see [`LineSegmenter`]. #[derive(Debug)] pubstruct LineBreakIterator<'l, 's, Y: LineBreakType<'l, 's> + ?Sized> {
iter: Y::IterAttr,
len: usize,
current_pos_data: Option<(usize, Y::CharType)>,
result_cache: Vec<usize>,
data: &'l RuleBreakDataV1<'l>,
options: &'l LineBreakOptions,
complex: &'l ComplexPayloads,
}
impl<'l, 's, Y: LineBreakType<'l, 's>> Iterator for LineBreakIterator<'l, 's, Y> { type Item = usize;
// If we have break point cache by previous run, return this result iflet Some(&first_pos) = self.result_cache.first() { letmut i = 0; loop { if i == first_pos { self.result_cache = self.result_cache.iter().skip(1).map(|r| r - i).collect(); returnself.get_current_position();
}
i += Y::get_current_position_character_len(self); self.advance_iter(); ifself.is_eof() { self.result_cache.clear(); return Some(self.len);
}
}
}
let Some(right_codepoint) = self.get_current_codepoint() else { return Some(self.len);
}; let right_prop = self.get_linebreak_property(right_codepoint);
// CSS word-break property handling match (self.options.word_option, left_prop, right_prop) {
(LineBreakWordOption::BreakAll, AL | NU | SA, _) => {
left_prop = ID;
} // typographic letter units shouldn't be break
(
LineBreakWordOption::KeepAll,
AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,
AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,
) => { continue;
}
_ => (),
}
// UAX14 doesn't have Thai etc, so use another way. ifself.options.word_option != LineBreakWordOption::BreakAll
&& Y::use_complex_breaking(self, left_codepoint)
&& Y::use_complex_breaking(self, right_codepoint)
{ let result = Y::handle_complex_language(self, left_codepoint); if result.is_some() { return result;
} // I may have to fetch text until non-SA character?.
}
// If break_state is equals or grater than 0, it is alias of property. letmut index = matchself.data.get_break_state_from_table(left_prop, right_prop) {
BreakState::Index(index) => index, // Line break uses more that 64 states, so they spill over into the intermediate range, // and we cannot change that at the moment
BreakState::Intermediate(index) => index + 64,
BreakState::Break | BreakState::NoMatch => returnself.get_current_position(),
BreakState::Keep => continue,
};
let Some(prop) = self.get_current_linebreak_property() else { // Reached EOF. But we are analyzing multiple characters now, so next break may be previous point. let break_state = self
.data
.get_break_state_from_table(index, self.data.eot_property); if break_state == BreakState::NoMatch { self.iter = previous_iter; self.current_pos_data = previous_pos_data; returnself.get_current_position();
} // EOF return Some(self.len);
};
#[inline] fn check_eof(&mutself) -> StringBoundaryPosType { ifself.is_eof() { self.advance_iter(); ifself.is_eof() { ifself.len == 0 { // Empty string. Since `self.current_pos_data` is always going to be empty, // we never read `self.len` except for here, so we can use it to mark that // we have already returned the single empty-string breakpoint. self.len = 1;
StringBoundaryPosType::Start
} else {
StringBoundaryPosType::End
}
} else {
StringBoundaryPosType::Start
}
} else {
StringBoundaryPosType::Middle
}
}
fn handle_complex_language(
iter: &mut LineBreakIterator<'l, 's, Self>,
left_codepoint: char,
) -> Option<usize> {
handle_complex_language_utf8(iter, left_codepoint)
}
} /// handle_complex_language impl for UTF8 iterators fn handle_complex_language_utf8<'l, 's, T>(
iter: &mut LineBreakIterator<'l, 's, T>,
left_codepoint: char,
) -> Option<usize> where
T: LineBreakType<'l, 's, CharType = char>,
{ // word segmenter doesn't define break rules for some languages such as Thai. let start_iter = iter.iter.clone(); let start_point = iter.current_pos_data; letmut s = String::new();
s.push(left_codepoint); loop {
debug_assert!(!iter.is_eof());
s.push(iter.get_current_codepoint()?);
iter.advance_iter(); iflet Some(current_codepoint) = iter.get_current_codepoint() { if !T::use_complex_breaking(iter, current_codepoint) { break;
}
} else { // EOF break;
}
}
// Restore iterator to move to head of complex string
iter.iter = start_iter;
iter.current_pos_data = start_point; let breaks = complex_language_segment_str(iter.complex, &s);
iter.result_cache = breaks; let first_pos = *iter.result_cache.first()?; letmut i = left_codepoint.len_utf8(); loop { if i == first_pos { // Re-calculate breaking offset
iter.result_cache = iter.result_cache.iter().skip(1).map(|r| r - i).collect(); return iter.get_current_position();
}
debug_assert!(
i < first_pos, "we should always arrive at first_pos: near index {:?}",
iter.get_current_position()
);
i += T::get_current_position_character_len(iter);
iter.advance_iter(); if iter.is_eof() {
iter.result_cache.clear(); return Some(iter.len);
}
}
}
#[derive(Debug)] pubstruct LineBreakTypeLatin1;
impl<'l, 's> LineBreakType<'l, 's> for LineBreakTypeLatin1 { type IterAttr = Latin1Indices<'s>; type CharType = u8;
fn get_linebreak_property_with_rule(iterator: &LineBreakIterator<Self>, c: u8) -> u8 { // No CJ on Latin1 // Note: Default value is 0 == UNKNOWN
iterator.data.property_table.get32(c as u32)
}
fn handle_complex_language(
iterator: &mut LineBreakIterator<Self>,
left_codepoint: Self::CharType,
) -> Option<usize> { // word segmenter doesn't define break rules for some languages such as Thai. let start_iter = iterator.iter.clone(); let start_point = iterator.current_pos_data; letmut s = vec![left_codepoint as u16]; loop {
debug_assert!(!iterator.is_eof());
s.push(iterator.get_current_codepoint()? as u16);
iterator.advance_iter(); iflet Some(current_codepoint) = iterator.get_current_codepoint() { if !Self::use_complex_breaking(iterator, current_codepoint) { break;
}
} else { // EOF break;
}
}
// Restore iterator to move to head of complex string
iterator.iter = start_iter;
iterator.current_pos_data = start_point; let breaks = complex_language_segment_utf16(iterator.complex, &s);
iterator.result_cache = breaks; // result_cache vector is utf-16 index that is in BMP. let first_pos = *iterator.result_cache.first()?; letmut i = 1; loop { if i == first_pos { // Re-calculate breaking offset
iterator.result_cache = iterator
.result_cache
.iter()
.skip(1)
.map(|r| r - i)
.collect(); return iterator.get_current_position();
}
debug_assert!(
i < first_pos, "we should always arrive at first_pos: near index {:?}",
iterator.get_current_position()
);
i += 1;
iterator.advance_iter(); if iterator.is_eof() {
iterator.result_cache.clear(); return Some(iterator.len);
}
}
}
}
#[cfg(test)] #[cfg(feature = "serde")] mod tests { usesuper::*; usecrate::LineSegmenter;
#[test] fn linebreak_property() { let payload = DataProvider::<LineBreakDataV1Marker>::load(
&crate::provider::Baked,
Default::default(),
)
.expect("Loading should succeed!")
.take_payload()
.expect("Data should be present!");
let get_linebreak_property = |codepoint| {
payload.get().get_linebreak_property_utf32_with_rule(
codepoint as u32,
LineBreakStrictness::Strict,
LineBreakWordOption::Normal,
)
};
let segmenter = LineSegmenter::new_lstm(); let breaks: Vec<usize> = segmenter.segment_str(TEST_STR).collect(); // LSTM model breaks more characters, but it is better to return [30].
assert_eq!(breaks, [0, 12, 18, 30, TEST_STR.len()], "Burmese test");
let utf16: Vec<u16> = TEST_STR.encode_utf16().collect(); let breaks: Vec<usize> = segmenter.segment_utf16(&utf16).collect(); // LSTM model breaks more characters, but it is better to return [10].
assert_eq!(breaks, [0, 4, 6, 10, utf16.len()], "Burmese utf-16 test");
}
let segmenter = LineSegmenter::new_lstm(); let breaks: Vec<usize> = segmenter.segment_str(TEST_STR).collect(); // Note: LSTM finds a break at '12' that the dictionary does not find
assert_eq!(breaks, [0, 12, 21, 30, 39, TEST_STR.len()], "Lao test");
#[test] fn empty_string() { let segmenter = LineSegmenter::new_auto(); let breaks: Vec<usize> = segmenter.segment_str("").collect();
assert_eq!(breaks, [0]);
}
}
Messung V0.5 in Prozent
¤ Diese beiden folgenden Angebotsgruppen bietet das Unternehmen0.26Angebot
(Wie Sie bei der Firma Beratungs- und Dienstleistungen beauftragen können 2026-06-19)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
