|
|
|
|
Quelle lib.rs
Sprache: unbekannt
|
|
// Copyright 2019 Mozilla Foundation. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! `chardetng` is a character encoding detector for legacy Web content.
//!
//! It is optimized for binary size in applications that already depend
//! on `encoding_rs` for other reasons.
use encoding_rs::Decoder;
use encoding_rs::DecoderResult;
use encoding_rs::Encoding;
use encoding_rs::BIG5;
use encoding_rs::EUC_JP;
use encoding_rs::EUC_KR;
use encoding_rs::GBK;
use encoding_rs::ISO_2022_JP;
use encoding_rs::ISO_8859_8;
use encoding_rs::SHIFT_JIS;
use encoding_rs::UTF_8;
use encoding_rs::WINDOWS_1255;
mod data;
mod tld;
use data::*;
use tld::classify_tld;
use tld::Tld;
const LATIN_ADJACENCY_PENALTY: i64 = -50;
const IMPLAUSIBILITY_PENALTY: i64 = -220;
const ORDINAL_BONUS: i64 = 300;
/// Must match the ISO-8859-2 score for " Š ". Note: There
/// are four Slovenian Wikipedia list page titles where the
/// list is split by letter so that Š stands alone for the
/// list part for Š. Let's assume that's a special case not
/// worth detecting even though the copyright sign detection
/// makes Slovenian title detection round to one percentage
/// point worse.
const COPYRIGHT_BONUS: i64 = 222;
const IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY: i64 = -180;
const NON_LATIN_CAPITALIZATION_BONUS: i64 = 40;
const NON_LATIN_ALL_CAPS_PENALTY: i64 = -40;
const NON_LATIN_MIXED_CASE_PENALTY: i64 = -20;
// Manually calibrated relative to windows-1256 Arabic
const CJK_BASE_SCORE: i64 = 41;
const CJK_SECONDARY_BASE_SCORE: i64 = 20; // Was 20
const SHIFT_JIS_SCORE_PER_KANA: i64 = 20;
const SHIFT_JIS_SCORE_PER_LEVEL_1_KANJI: i64 = CJK_BASE_SCORE;
const SHIFT_JIS_SCORE_PER_LEVEL_2_KANJI: i64 = CJK_SECONDARY_BASE_SCORE;
// Manually calibrated relative to windows-1256 Persian and Urdu
const SHIFT_JIS_INITIAL_HALF_WIDTH_KATAKANA_PENALTY: i64 = -75;
const HALF_WIDTH_KATAKANA_SCORE: i64 = 1;
// Unclear if this is a good idea; seems not harmful, but can't be sure.
const HALF_WIDTH_KATAKANA_VOICING_SCORE: i64 = 10;
const SHIFT_JIS_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 10); // Should this be larger?
const SHIFT_JIS_EXTENSION_PENALTY: i64 = SHIFT_JIS_PUA_PENALTY * 2;
const SHIFT_JIS_SINGLE_BYTE_EXTENSION_PENALTY: i64 = SHIFT_JIS_EXTENSION_PENALTY;
const EUC_JP_SCORE_PER_KANA: i64 = CJK_BASE_SCORE + (CJK_BASE_SCORE / 3); // Relative to Big 5
const EUC_JP_SCORE_PER_NEAR_OBSOLETE_KANA: i64 = CJK_BASE_SCORE - 1;
const EUC_JP_SCORE_PER_LEVEL_1_KANJI: i64 = CJK_BASE_SCORE;
const EUC_JP_SCORE_PER_LEVEL_2_KANJI: i64 = CJK_SECONDARY_BASE_SCORE;
const EUC_JP_SCORE_PER_OTHER_KANJI: i64 = CJK_SECONDARY_BASE_SCORE / 4;
const EUC_JP_INITIAL_KANA_PENALTY: i64 = -((CJK_BASE_SCORE / 3) + 1);
const EUC_JP_EXTENSION_PENALTY: i64 = -(CJK_BASE_SCORE * 50); // Needs to be more severe than for Shift_JIS to avoid misdetecting EUC-KR!
const BIG5_SCORE_PER_LEVEL_1_HANZI: i64 = CJK_BASE_SCORE;
const BIG5_SCORE_PER_OTHER_HANZI: i64 = CJK_SECONDARY_BASE_SCORE;
const BIG5_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 30); // More severe than other PUA penalties to avoid misdetecting EUC-KR! (25 as the multiplier is too little)
const BIG5_SINGLE_BYTE_EXTENSION_PENALTY: i64 = -(CJK_BASE_SCORE * 40);
const EUC_KR_SCORE_PER_EUC_HANGUL: i64 = CJK_BASE_SCORE + 1;
const EUC_KR_SCORE_PER_NON_EUC_HANGUL: i64 = CJK_SECONDARY_BASE_SCORE / 5;
const EUC_KR_SCORE_PER_HANJA: i64 = CJK_SECONDARY_BASE_SCORE / 2;
const EUC_KR_HANJA_AFTER_HANGUL_PENALTY: i64 = -(CJK_BASE_SCORE * 10);
const EUC_KR_LONG_WORD_PENALTY: i64 = -6;
const EUC_KR_PUA_PENALTY: i64 = GBK_PUA_PENALTY - 1; // Break tie in favor of GBK
const EUC_KR_MAC_KOREAN_PENALTY: i64 = EUC_KR_PUA_PENALTY * 2;
const EUC_KR_SINGLE_BYTE_EXTENSION_PENALTY: i64 = EUC_KR_MAC_KOREAN_PENALTY;
const GBK_SCORE_PER_LEVEL_1: i64 = CJK_BASE_SCORE;
const GBK_SCORE_PER_LEVEL_2: i64 = CJK_SECONDARY_BASE_SCORE;
const GBK_SCORE_PER_NON_EUC: i64 = CJK_SECONDARY_BASE_SCORE / 4;
const GBK_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 10); // Factor should be at least 2, but should it be larger?
const GBK_SINGLE_BYTE_EXTENSION_PENALTY: i64 = GBK_PUA_PENALTY * 4;
const CJK_LATIN_ADJACENCY_PENALTY: i64 = -CJK_BASE_SCORE; // smaller penalty than LATIN_ADJACENCY_PENALTY
const CJ_PUNCTUATION: i64 = CJK_BASE_SCORE / 2;
const CJK_OTHER: i64 = CJK_SECONDARY_BASE_SCORE / 4;
/// Latin letter caseless class
const LATIN_LETTER: u8 = 1;
fn contains_upper_case_period_or_non_ascii(label: &[u8]) -> bool {
for &b in label.into_iter() {
if b >= 0x80 {
return true;
}
if b == b'.' {
return true;
}
if b >= b'A' && b <= b'Z' {
return true;
}
}
false
}
// For Latin, we only penalize pairwise bad transitions
// if one participant is non-ASCII. This avoids violating
// the principle that ASCII pairs never contribute to the
// score. (Maybe that's a bad principle, though!)
#[derive(PartialEq)]
enum LatinCaseState {
Space,
Upper,
Lower,
AllCaps,
}
// Fon non-Latin, we calculate case-related penalty
// or bonus on a per-non-Latin-word basis.
#[derive(PartialEq)]
enum NonLatinCaseState {
Space,
Upper,
Lower,
UpperLower,
AllCaps,
Mix,
}
struct NonLatinCasedCandidate {
data: &'static SingleByteData,
prev: u8,
case_state: NonLatinCaseState,
prev_ascii: bool,
current_word_len: u64,
longest_word: u64,
ibm866: bool,
prev_was_a0: bool, // Only used with IBM866
}
impl NonLatinCasedCandidate {
fn new(data: &'static SingleByteData) -> Self {
NonLatinCasedCandidate {
data: data,
prev: 0,
case_state: NonLatinCaseState::Space,
prev_ascii: true,
current_word_len: 0,
longest_word: 0,
ibm866: data == &SINGLE_BYTE_DATA[IBM866_INDEX],
prev_was_a0: false,
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_ascii && ascii;
let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);
// The purpose of this state machine is to avoid misdetecting Greek as
// Cyrillic by:
//
// * Giving a small bonus to words that start with an upper-case letter
// and are lower-case for the rest.
// * Giving a large penalty to start with one lower-case letter followed
// by all upper-case (obviously upper and lower case inverted, which
// unfortunately is possible due to KOI8-U).
// * Giving a small per-word penalty to all-uppercase KOI8-U (to favor
// all-lowercase Greek over all-caps KOI8-U).
// * Giving large penalties for mixed-case other than initial upper-case.
// This also helps relative to non-cased encodings.
// ASCII doesn't participate in non-Latin casing.
if caseless_class == LATIN_LETTER {
// Latin
// Mark this word as a mess. If there end up being non-Latin
// letters in this word, the ASCII-adjacency penalty gets
// applied to Latin/non-Latin pairs and the mix penalty
// to non-Latin/non-Latin pairs.
// XXX Apply penalty here
self.case_state = NonLatinCaseState::Mix;
} else if !non_ascii_alphabetic {
// Space
match self.case_state {
NonLatinCaseState::Space
| NonLatinCaseState::Upper
| NonLatinCaseState::Lower => {}
NonLatinCaseState::UpperLower => {
// Intentionally applied only once per word.
score += NON_LATIN_CAPITALIZATION_BONUS;
}
NonLatinCaseState::AllCaps => {
// Intentionally applied only once per word.
if self.data == &SINGLE_BYTE_DATA[KOI8_U_INDEX] {
// Apply only to KOI8-U.
score += NON_LATIN_ALL_CAPS_PENALTY;
}
}
NonLatinCaseState::Mix => {
// Per letter
score += NON_LATIN_MIXED_CASE_PENALTY * (self.current_word_len as i64);
}
}
self.case_state = NonLatinCaseState::Space;
} else if (class >> 7) == 0 {
// Lower case
match self.case_state {
NonLatinCaseState::Space => {
self.case_state = NonLatinCaseState::Lower;
}
NonLatinCaseState::Upper => {
self.case_state = NonLatinCaseState::UpperLower;
}
NonLatinCaseState::Lower
| NonLatinCaseState::UpperLower
| NonLatinCaseState::Mix => {}
NonLatinCaseState::AllCaps => {
self.case_state = NonLatinCaseState::Mix;
}
}
} else {
// Upper case
match self.case_state {
NonLatinCaseState::Space => {
self.case_state = NonLatinCaseState::Upper;
}
NonLatinCaseState::Upper => {
self.case_state = NonLatinCaseState::AllCaps;
}
NonLatinCaseState::Lower | NonLatinCaseState::UpperLower => {
self.case_state = NonLatinCaseState::Mix;
}
NonLatinCaseState::AllCaps | NonLatinCaseState::Mix => {}
}
}
// XXX Apply penalty if > 16
if non_ascii_alphabetic {
self.current_word_len += 1;
} else {
if self.current_word_len > self.longest_word {
self.longest_word = self.current_word_len;
}
self.current_word_len = 0;
}
let is_a0 = b == 0xA0;
if !ascii_pair {
// 0xA0 is no-break space in many other encodings, so avoid
// assigning score to IBM866 when 0xA0 occurs next to itself
// or a space-like byte.
if !(self.ibm866
&& ((is_a0 && (self.prev_was_a0 || self.prev == 0))
|| caseless_class == 0 && self.prev_was_a0))
{
score += self.data.score(caseless_class, self.prev, false);
}
if self.prev == LATIN_LETTER && non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
} else if caseless_class == LATIN_LETTER
&& self.data.is_non_latin_alphabetic(self.prev, false)
{
score += LATIN_ADJACENCY_PENALTY;
}
}
self.prev_ascii = ascii;
self.prev = caseless_class;
self.prev_was_a0 = is_a0;
}
Some(score)
}
}
enum OrdinalState {
Other,
Space,
PeriodAfterN,
OrdinalExpectingSpace,
OrdinalExpectingSpaceUndoImplausibility,
OrdinalExpectingSpaceOrDigit,
OrdinalExpectingSpaceOrDigitUndoImplausibily,
UpperN,
LowerN,
FeminineAbbreviationStartLetter,
Digit,
Roman,
Copyright,
}
struct LatinCandidate {
data: &'static SingleByteData,
prev: u8,
case_state: LatinCaseState,
prev_non_ascii: u32,
ordinal_state: OrdinalState, // Used only when `windows1252 == true`
windows1252: bool,
}
impl LatinCandidate {
fn new(data: &'static SingleByteData) -> Self {
LatinCandidate {
data: data,
prev: 0,
case_state: LatinCaseState::Space,
prev_non_ascii: 0,
ordinal_state: OrdinalState::Space,
windows1252: data == &SINGLE_BYTE_DATA[WINDOWS_1252_INDEX],
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_non_ascii == 0 && ascii;
let non_ascii_penalty = match self.prev_non_ascii {
0 | 1 | 2 => 0,
3 => -5,
4 => -20,
_ => -200,
};
score += non_ascii_penalty;
// XXX if has Vietnamese-only characters and word length > 7,
// apply penalty
if !self.data.is_latin_alphabetic(caseless_class) {
self.case_state = LatinCaseState::Space;
} else if (class >> 7) == 0 {
// Penalizing lower case after two upper case
// is important for avoiding misdetecting
// windows-1250 as windows-1252 (byte 0x9F).
if self.case_state == LatinCaseState::AllCaps && !ascii_pair {
score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;
}
self.case_state = LatinCaseState::Lower;
} else {
match self.case_state {
LatinCaseState::Space => {
self.case_state = LatinCaseState::Upper;
}
LatinCaseState::Upper | LatinCaseState::AllCaps => {
self.case_state = LatinCaseState::AllCaps;
}
LatinCaseState::Lower => {
if !ascii_pair {
// XXX How bad is this for Irish Gaelic?
score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;
}
self.case_state = LatinCaseState::Upper;
}
}
}
// Treat pairing space-like, which can be non-ASCII, with ASCII as
// ASCIIish enough not to get a score in order to avoid giving
// ASCII i and I in windows-1254 next to windows-125x apostrophe/quote
// a score. This avoids detecting English I’ as Turkish.
let ascii_ish_pair = ascii_pair
|| (ascii && self.prev == 0)
|| (caseless_class == 0 && self.prev_non_ascii == 0);
if !ascii_ish_pair {
score += self.data.score(caseless_class, self.prev, false);
}
if self.windows1252 {
// This state machine assigns score to the sequences
// * " º " (Spanish)
// * " ª " (Spanish)
// * ".ª " (Spanish)
// * ".º " (Spanish)
// * "n.º1" (Spanish)
// * " Mª " (Spanish)
// * " Dª " (Spanish)
// * " Nª " (Spanish)
// * " Sª " (Spanish)
// * " 3º " (Italian, where 3 is an ASCII digit)
// * " 3ª " (Italian, where 3 is an ASCII digit)
// * " Xº " (Italian, where X is a small Roman numeral)
// * " Xª " (Italian, where X is a small Roman numeral)
// * " Nº1" (Italian, where 1 is an ASCII digit)
// * " Nº " (Italian)
// * " © " (otherwise ASCII-only)
// which are problematic to deal with by pairwise scoring
// without messing up Romanian detection.
// Initial sc
match self.ordinal_state {
OrdinalState::Other => {
if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
}
}
OrdinalState::Space => {
if caseless_class == 0 {
// pass
} else if b == 0xAA || b == 0xBA {
self.ordinal_state = OrdinalState::OrdinalExpectingSpace;
} else if b == b'M' || b == b'D' || b == b'S' {
self.ordinal_state = OrdinalState::FeminineAbbreviationStartLetter;
} else if b == b'N' {
// numero or Nuestra
self.ordinal_state = OrdinalState::UpperN;
} else if b == b'n' {
// numero
self.ordinal_state = OrdinalState::LowerN;
} else if caseless_class == (ASCII_DIGIT as u8) {
self.ordinal_state = OrdinalState::Digit;
} else if caseless_class == 9 /* I */ || caseless_class == 22 /* V */ || caseless_class == 24
/* X */
{
self.ordinal_state = OrdinalState::Roman;
} else if b == 0xA9 {
self.ordinal_state = OrdinalState::Copyright;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::OrdinalExpectingSpace => {
if caseless_class == 0 {
score += ORDINAL_BONUS;
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::OrdinalExpectingSpaceUndoImplausibility => {
if caseless_class == 0 {
score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::OrdinalExpectingSpaceOrDigit => {
if caseless_class == 0 {
score += ORDINAL_BONUS;
self.ordinal_state = OrdinalState::Space;
} else if caseless_class == (ASCII_DIGIT as u8) {
score += ORDINAL_BONUS;
// Deliberately set to `Other`
self.ordinal_state = OrdinalState::Other;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily => {
if caseless_class == 0 {
score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;
self.ordinal_state = OrdinalState::Space;
} else if caseless_class == (ASCII_DIGIT as u8) {
score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;
// Deliberately set to `Other`
self.ordinal_state = OrdinalState::Other;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::UpperN => {
if b == 0xAA {
self.ordinal_state =
OrdinalState::OrdinalExpectingSpaceUndoImplausibility;
} else if b == 0xBA {
self.ordinal_state =
OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily;
} else if b == b'.' {
self.ordinal_state = OrdinalState::PeriodAfterN;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::LowerN => {
if b == 0xBA {
self.ordinal_state =
OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily;
} else if b == b'.' {
self.ordinal_state = OrdinalState::PeriodAfterN;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::FeminineAbbreviationStartLetter => {
if b == 0xAA {
self.ordinal_state =
OrdinalState::OrdinalExpectingSpaceUndoImplausibility;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::Digit => {
if b == 0xAA || b == 0xBA {
self.ordinal_state = OrdinalState::OrdinalExpectingSpace;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else if caseless_class == (ASCII_DIGIT as u8) {
// pass
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::Roman => {
if b == 0xAA || b == 0xBA {
self.ordinal_state =
OrdinalState::OrdinalExpectingSpaceUndoImplausibility;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else if caseless_class == 9 /* I */ || caseless_class == 22 /* V */ || caseless_class == 24
/* X */
{
// pass
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::PeriodAfterN => {
if b == 0xBA {
self.ordinal_state = OrdinalState::OrdinalExpectingSpaceOrDigit;
} else if caseless_class == 0 {
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
OrdinalState::Copyright => {
if caseless_class == 0 {
score += COPYRIGHT_BONUS;
self.ordinal_state = OrdinalState::Space;
} else {
self.ordinal_state = OrdinalState::Other;
}
}
}
}
if ascii {
self.prev_non_ascii = 0;
} else {
self.prev_non_ascii += 1;
}
self.prev = caseless_class;
}
Some(score)
}
}
struct ArabicFrenchCandidate {
data: &'static SingleByteData,
prev: u8,
case_state: LatinCaseState,
prev_ascii: bool,
current_word_len: u64,
longest_word: u64,
}
impl ArabicFrenchCandidate {
fn new(data: &'static SingleByteData) -> Self {
ArabicFrenchCandidate {
data: data,
prev: 0,
case_state: LatinCaseState::Space,
prev_ascii: true,
current_word_len: 0,
longest_word: 0,
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_ascii && ascii;
if caseless_class != LATIN_LETTER {
// We compute case penalties for French only
self.case_state = LatinCaseState::Space;
} else if (class >> 7) == 0 {
if self.case_state == LatinCaseState::AllCaps && !ascii_pair {
score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;
}
self.case_state = LatinCaseState::Lower;
} else {
match self.case_state {
LatinCaseState::Space => {
self.case_state = LatinCaseState::Upper;
}
LatinCaseState::Upper | LatinCaseState::AllCaps => {
self.case_state = LatinCaseState::AllCaps;
}
LatinCaseState::Lower => {
if !ascii_pair {
score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;
}
self.case_state = LatinCaseState::Upper;
}
}
}
// Count only Arabic word length and ignore French
let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, true);
// XXX apply penalty if > 23
if non_ascii_alphabetic {
self.current_word_len += 1;
} else {
if self.current_word_len > self.longest_word {
self.longest_word = self.current_word_len;
}
self.current_word_len = 0;
}
if !ascii_pair {
score += self.data.score(caseless_class, self.prev, true);
if self.prev == LATIN_LETTER && non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
} else if caseless_class == LATIN_LETTER
&& self.data.is_non_latin_alphabetic(self.prev, true)
{
score += LATIN_ADJACENCY_PENALTY;
}
}
self.prev_ascii = ascii;
self.prev = caseless_class;
}
Some(score)
}
}
struct CaselessCandidate {
data: &'static SingleByteData,
prev: u8,
prev_ascii: bool,
current_word_len: u64,
longest_word: u64,
}
impl CaselessCandidate {
fn new(data: &'static SingleByteData) -> Self {
CaselessCandidate {
data: data,
prev: 0,
prev_ascii: true,
current_word_len: 0,
longest_word: 0,
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_ascii && ascii;
let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);
// Apply penalty if > 23 and not Thai
if non_ascii_alphabetic {
self.current_word_len += 1;
} else {
if self.current_word_len > self.longest_word {
self.longest_word = self.current_word_len;
}
self.current_word_len = 0;
}
if !ascii_pair {
score += self.data.score(caseless_class, self.prev, false);
if self.prev == LATIN_LETTER && non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
} else if caseless_class == LATIN_LETTER
&& self.data.is_non_latin_alphabetic(self.prev, false)
{
score += LATIN_ADJACENCY_PENALTY;
}
}
self.prev_ascii = ascii;
self.prev = caseless_class;
}
Some(score)
}
}
fn is_ascii_punctuation(byte: u8) -> bool {
match byte {
b'.' | b',' | b':' | b';' | b'?' | b'!' => true,
_ => false,
}
}
struct LogicalCandidate {
data: &'static SingleByteData,
prev: u8,
prev_ascii: bool,
plausible_punctuation: u64,
current_word_len: u64,
longest_word: u64,
}
impl LogicalCandidate {
fn new(data: &'static SingleByteData) -> Self {
LogicalCandidate {
data: data,
prev: 0,
prev_ascii: true,
plausible_punctuation: 0,
current_word_len: 0,
longest_word: 0,
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_ascii && ascii;
let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);
// XXX apply penalty if > 22
if non_ascii_alphabetic {
self.current_word_len += 1;
} else {
if self.current_word_len > self.longest_word {
self.longest_word = self.current_word_len;
}
self.current_word_len = 0;
}
if !ascii_pair {
score += self.data.score(caseless_class, self.prev, false);
let prev_non_ascii_alphabetic = self.data.is_non_latin_alphabetic(self.prev, false);
if caseless_class == 0 && prev_non_ascii_alphabetic && is_ascii_punctuation(b) {
self.plausible_punctuation += 1;
}
if self.prev == LATIN_LETTER && non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
} else if caseless_class == LATIN_LETTER && prev_non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
}
}
self.prev_ascii = ascii;
self.prev = caseless_class;
}
Some(score)
}
}
struct VisualCandidate {
data: &'static SingleByteData,
prev: u8,
prev_ascii: bool,
prev_punctuation: bool,
plausible_punctuation: u64,
current_word_len: u64,
longest_word: u64,
}
impl VisualCandidate {
fn new(data: &'static SingleByteData) -> Self {
VisualCandidate {
data: data,
prev: 0,
prev_ascii: true,
prev_punctuation: false,
plausible_punctuation: 0,
current_word_len: 0,
longest_word: 0,
}
}
fn feed(&mut self, buffer: &[u8]) -> Option<i64> {
let mut score = 0i64;
for &b in buffer {
let class = self.data.classify(b);
if class == 255 {
return None;
}
let caseless_class = class & 0x7F;
let ascii = b < 0x80;
let ascii_pair = self.prev_ascii && ascii;
let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);
// XXX apply penalty if > 22
if non_ascii_alphabetic {
self.current_word_len += 1;
} else {
if self.current_word_len > self.longest_word {
self.longest_word = self.current_word_len;
}
self.current_word_len = 0;
}
if !ascii_pair {
score += self.data.score(caseless_class, self.prev, false);
if non_ascii_alphabetic && self.prev_punctuation {
self.plausible_punctuation += 1;
}
if self.prev == LATIN_LETTER && non_ascii_alphabetic {
score += LATIN_ADJACENCY_PENALTY;
} else if caseless_class == LATIN_LETTER
&& self.data.is_non_latin_alphabetic(self.prev, false)
{
score += LATIN_ADJACENCY_PENALTY;
}
}
self.prev_ascii = ascii;
self.prev = caseless_class;
self.prev_punctuation = caseless_class == 0 && is_ascii_punctuation(b);
}
Some(score)
}
}
struct Utf8Candidate {
decoder: Decoder,
}
impl Utf8Candidate {
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut dst = [0u8; 1024];
let mut total_read = 0;
loop {
let (result, read, _) = self.decoder.decode_to_utf8_without_replacement(
&buffer[total_read..],
&mut dst,
last,
);
total_read += read;
match result {
DecoderResult::InputEmpty => {
return Some(0);
}
DecoderResult::Malformed(_, _) => {
return None;
}
DecoderResult::OutputFull => {
continue;
}
}
}
}
}
struct Iso2022Candidate {
decoder: Decoder,
}
impl Iso2022Candidate {
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut dst = [0u16; 1024];
let mut total_read = 0;
loop {
let (result, read, _) = self.decoder.decode_to_utf16_without_replacement(
&buffer[total_read..],
&mut dst,
last,
);
total_read += read;
match result {
DecoderResult::InputEmpty => {
return Some(0);
}
DecoderResult::Malformed(_, _) => {
return None;
}
DecoderResult::OutputFull => {
continue;
}
}
}
}
}
#[derive(PartialEq)]
enum LatinCj {
AsciiLetter,
Cj,
Other,
}
#[derive(PartialEq, Copy, Clone)]
enum HalfWidthKatakana {
DakutenForbidden,
DakutenAllowed,
DakutenOrHandakutenAllowed,
}
#[derive(PartialEq)]
enum LatinKorean {
AsciiLetter,
Hangul,
Hanja,
Other,
}
fn cjk_extra_score(u: u16, table: &'static [u16; 128]) -> i64 {
if let Some(pos) = table.iter().position(|&x| x == u) {
((128 - pos) / 16) as i64
} else {
0
}
}
struct GbkCandidate {
decoder: Decoder,
prev_byte: u8,
prev: LatinCj,
pending_score: Option<i64>,
}
impl GbkCandidate {
fn maybe_set_as_pending(&mut self, s: i64) -> i64 {
assert!(self.pending_score.is_none());
if self.prev == LatinCj::Cj || !more_problematic_lead(self.prev_byte) {
s
} else {
self.pending_score = Some(s);
0
}
}
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut score = 0i64;
let mut src = [0u8];
let mut dst = [0u16; 2];
for &b in buffer {
src[0] = b;
let (result, read, written) = self
.decoder
.decode_to_utf16_without_replacement(&src, &mut dst, false);
if written == 1 {
let u = dst[0];
if (u >= u16::from(b'a') && u <= u16::from(b'z'))
|| (u >= u16::from(b'A') && u <= u16::from(b'Z'))
{
self.pending_score = None; // Discard pending score
if self.prev == LatinCj::Cj {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::AsciiLetter;
} else if u == 0x20AC {
// euro sign
self.pending_score = None; // Discard pending score
// Should there even be a penalty?
self.prev = LatinCj::Other;
} else if u >= 0x4E00 && u <= 0x9FA5 {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
if b >= 0xA1 && b <= 0xFE {
match self.prev_byte {
0xA1..=0xD7 => {
score += GBK_SCORE_PER_LEVEL_1;
score +=
cjk_extra_score(u, &data::DETECTOR_DATA.frequent_simplified);
}
0xD8..=0xFE => score += GBK_SCORE_PER_LEVEL_2,
_ => {
score += GBK_SCORE_PER_NON_EUC;
}
}
} else {
score += self.maybe_set_as_pending(GBK_SCORE_PER_NON_EUC);
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
// XXX score?
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if u >= 0xE000 && u < 0xF900 {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
// Treat the GB18030-required PUA mappings as non-EUC ideographs.
match u {
0xE78D..=0xE796
| 0xE816..=0xE818
| 0xE81E
| 0xE826
| 0xE82B
| 0xE82C
| 0xE831
| 0xE832
| 0xE83B
| 0xE843
| 0xE854
| 0xE855
| 0xE864 => {
score += GBK_SCORE_PER_NON_EUC;
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
}
_ => {
score += GBK_PUA_PENALTY;
self.prev = LatinCj::Other;
}
}
} else {
match u {
0x3000 // Distinct from Korean, space
| 0x3001 // Distinct from Korean, enumeration comma
| 0x3002 // Distinct from Korean, full stop
| 0xFF08 // Distinct from Korean, parenthesis
| 0xFF09 // Distinct from Korean, parenthesis
| 0xFF01 // Distinct from Japanese, exclamation
| 0xFF0C // Distinct from Japanese, comma
| 0xFF1B // Distinct from Japanese, semicolon
| 0xFF1F // Distinct from Japanese, question
=> {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += CJ_PUNCTUATION;
}
0..=0x7F => {
self.pending_score = None; // Discard pending score
}
_ => {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += CJK_OTHER;
}
}
self.prev = LatinCj::Other;
}
} else if written == 2 {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
let u = dst[0];
if u >= 0xDB80 && u <= 0xDBFF {
score += GBK_PUA_PENALTY;
self.prev = LatinCj::Other;
} else if u >= 0xD480 && u < 0xD880 {
score += GBK_SCORE_PER_NON_EUC;
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else {
score += CJK_OTHER;
self.prev = LatinCj::Other;
}
}
match result {
DecoderResult::InputEmpty => {
assert_eq!(read, 1);
}
DecoderResult::Malformed(malformed_len, _) => {
if (self.prev_byte == 0xA0 || self.prev_byte == 0xFE || self.prev_byte == 0xFD)
&& (b < 0x80 || b == 0xFF)
{
// Mac OS Chinese Simplified single-byte that conflicts with code page GBK lead byte
// followed by ASCII or a non-conflicting single-byte extension.
self.pending_score = None; // Just in case
score += GBK_SINGLE_BYTE_EXTENSION_PENALTY;
if (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') {
self.prev = LatinCj::AsciiLetter;
} else if b == 0xFF {
score += GBK_SINGLE_BYTE_EXTENSION_PENALTY;
self.prev = LatinCj::Other;
} else {
self.prev = LatinCj::Other;
}
// The GBK decoder has the pending ASCII concept, which is
// a problem with this trickery, so let's reset the state.
self.decoder = GBK.new_decoder_without_bom_handling();
} else if malformed_len == 1 && b == 0xFF {
// Mac OS Chinese Simplified single-byte extension that doesn't conflict with lead bytes
self.pending_score = None; // Just in case
score += GBK_SINGLE_BYTE_EXTENSION_PENALTY;
self.prev = LatinCj::Other;
// The GBK decoder has the pending ASCII concept, which is
// a problem with this trickery, so let's reset the state.
self.decoder = GBK.new_decoder_without_bom_handling();
} else {
return None;
}
}
DecoderResult::OutputFull => {
unreachable!();
}
}
self.prev_byte = b;
}
if last {
let (result, _, _) = self
.decoder
.decode_to_utf16_without_replacement(b"", &mut dst, true);
match result {
DecoderResult::InputEmpty => {}
DecoderResult::Malformed(_, _) => {
return None;
}
DecoderResult::OutputFull => {
unreachable!();
}
}
}
Some(score)
}
}
// Shift_JIS and Big5
fn problematic_lead(b: u8) -> bool {
match b {
0x91..=0x97 | 0x9A | 0x8A | 0x9B | 0x8B | 0x9E | 0x8E | 0xB0 => true,
_ => false,
}
}
// GBK and EUC-KR
fn more_problematic_lead(b: u8) -> bool {
problematic_lead(b) || b == 0x82 || b == 0x84 || b == 0x85 || b == 0xA0
}
struct ShiftJisCandidate {
decoder: Decoder,
half_width_katakana_seen: bool,
half_width_katakana_state: HalfWidthKatakana,
prev: LatinCj,
prev_byte: u8,
pending_score: Option<i64>,
}
impl ShiftJisCandidate {
fn maybe_set_as_pending(&mut self, s: i64) -> i64 {
assert!(self.pending_score.is_none());
if self.prev == LatinCj::Cj || !problematic_lead(self.prev_byte) {
s
} else {
self.pending_score = Some(s);
0
}
}
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut score = 0i64;
let mut src = [0u8];
let mut dst = [0u16; 2];
for &b in buffer {
src[0] = b;
let (result, read, written) = self
.decoder
.decode_to_utf16_without_replacement(&src, &mut dst, false);
if written > 0 {
let half_width_katakana_state = self.half_width_katakana_state;
self.half_width_katakana_state = HalfWidthKatakana::DakutenForbidden;
let u = dst[0];
if (u >= u16::from(b'a') && u <= u16::from(b'z'))
|| (u >= u16::from(b'A') && u <= u16::from(b'Z'))
{
self.pending_score = None; // Discard pending score
if self.prev == LatinCj::Cj {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::AsciiLetter;
} else if u >= 0xFF61 && u <= 0xFF9F {
if !self.half_width_katakana_seen {
self.half_width_katakana_seen = true;
// To avoid misdetecting title-length inputs
score += SHIFT_JIS_INITIAL_HALF_WIDTH_KATAKANA_PENALTY;
}
self.pending_score = None; // Discard pending score
score += HALF_WIDTH_KATAKANA_SCORE;
if (u >= 0xFF76 && u <= 0xFF84) || u == 0xFF73 {
self.half_width_katakana_state = HalfWidthKatakana::DakutenAllowed;
} else if u >= 0xFF8A && u <= 0xFF8E {
self.half_width_katakana_state =
HalfWidthKatakana::DakutenOrHandakutenAllowed;
} else if u == 0xFF9E {
if half_width_katakana_state == HalfWidthKatakana::DakutenForbidden {
score += IMPLAUSIBILITY_PENALTY;
} else {
score += HALF_WIDTH_KATAKANA_VOICING_SCORE;
}
} else if u == 0xFF9F {
if half_width_katakana_state
!= HalfWidthKatakana::DakutenOrHandakutenAllowed
{
score += IMPLAUSIBILITY_PENALTY;
} else {
score += HALF_WIDTH_KATAKANA_VOICING_SCORE;
}
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if u >= 0x3040 && u < 0x3100 {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += SHIFT_JIS_SCORE_PER_KANA;
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
if self.prev_byte < 0x98 || (self.prev_byte == 0x98 && b < 0x73) {
score += self.maybe_set_as_pending(
SHIFT_JIS_SCORE_PER_LEVEL_1_KANJI
+ cjk_extra_score(u, &data::DETECTOR_DATA.frequent_kanji),
);
} else {
score += self.maybe_set_as_pending(SHIFT_JIS_SCORE_PER_LEVEL_2_KANJI);
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if u >= 0xE000 && u < 0xF900 {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += SHIFT_JIS_PUA_PENALTY;
self.prev = LatinCj::Other;
} else {
match u {
0x3000 // Distinct from Korean, space
| 0x3001 // Distinct from Korean, enumeration comma
| 0x3002 // Distinct from Korean, full stop
| 0xFF08 // Distinct from Korean, parenthesis
| 0xFF09 // Distinct from Korean, parenthesis
=> {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
// Not really needed for CJK distinction
// but let's give non-zero score for these
// common byte pairs anyway.
score += CJ_PUNCTUATION;
}
0..=0x7F => {
self.pending_score = None; // Discard pending score
}
0x80 => {
// This is a control character that overlaps euro
// in windows-1252 and happens to be a non-error
// is Shift_JIS.
self.pending_score = None; // Discard pending score
score += IMPLAUSIBILITY_PENALTY;
}
_ => {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += CJK_OTHER;
}
}
self.prev = LatinCj::Other;
}
}
match result {
DecoderResult::InputEmpty => {
assert_eq!(read, 1);
}
DecoderResult::Malformed(malformed_len, _) => {
if (((self.prev_byte >= 0x81 && self.prev_byte <= 0x9F)
|| (self.prev_byte >= 0xE0 && self.prev_byte <= 0xFC))
&& ((b >= 0x40 && b <= 0x7E) || (b >= 0x80 && b <= 0xFC)))
&& !((self.prev_byte == 0x82 && b >= 0xFA)
|| (self.prev_byte == 0x84 && ((b >= 0xDD && b <= 0xE4) || b >= 0xFB))
|| (self.prev_byte == 0x86 && b >= 0xF2 && b <= 0xFA)
|| (self.prev_byte == 0x87 && b >= 0x77 && b <= 0x7D)
|| (self.prev_byte == 0xFC && b >= 0xF5))
{
// Shift_JIS2004 or MacJapanese
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
score += SHIFT_JIS_EXTENSION_PENALTY;
// Approximate boundary
if self.prev_byte < 0x87 {
self.prev = LatinCj::Other;
} else {
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
}
} else if malformed_len == 1 && (b == 0xA0 || b >= 0xFD) {
self.pending_score = None; // Just in case
score += SHIFT_JIS_SINGLE_BYTE_EXTENSION_PENALTY;
self.prev = LatinCj::Other;
} else {
return None;
}
}
DecoderResult::OutputFull => {
unreachable!();
}
}
self.prev_byte = b;
}
if last {
let (result, _, _) = self
.decoder
.decode_to_utf16_without_replacement(b"", &mut dst, true);
match result {
DecoderResult::InputEmpty => {}
DecoderResult::Malformed(_, _) => {
return None;
}
DecoderResult::OutputFull => {
unreachable!();
}
}
}
Some(score)
}
}
struct EucJpCandidate {
decoder: Decoder,
non_ascii_seen: bool,
half_width_katakana_state: HalfWidthKatakana,
prev: LatinCj,
prev_byte: u8,
prev_prev_byte: u8,
}
impl EucJpCandidate {
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut score = 0i64;
let mut src = [0u8];
let mut dst = [0u16; 2];
for &b in buffer {
src[0] = b;
let (result, read, written) = self
.decoder
.decode_to_utf16_without_replacement(&src, &mut dst, false);
if written > 0 {
let half_width_katakana_state = self.half_width_katakana_state;
self.half_width_katakana_state = HalfWidthKatakana::DakutenForbidden;
let u = dst[0];
if !self.non_ascii_seen && u >= 0x80 {
self.non_ascii_seen = true;
if u >= 0x3040 && u < 0x3100 {
// Remove the kana advantage over initial Big5
// hanzi.
score += EUC_JP_INITIAL_KANA_PENALTY;
}
}
if (u >= u16::from(b'a') && u <= u16::from(b'z'))
|| (u >= u16::from(b'A') && u <= u16::from(b'Z'))
{
if self.prev == LatinCj::Cj {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::AsciiLetter;
} else if u >= 0xFF61 && u <= 0xFF9F {
score += HALF_WIDTH_KATAKANA_SCORE;
if (u >= 0xFF76 && u <= 0xFF84) || u == 0xFF73 {
self.half_width_katakana_state = HalfWidthKatakana::DakutenAllowed;
} else if u >= 0xFF8A && u <= 0xFF8E {
self.half_width_katakana_state =
HalfWidthKatakana::DakutenOrHandakutenAllowed;
} else if u == 0xFF9E {
if half_width_katakana_state == HalfWidthKatakana::DakutenForbidden {
score += IMPLAUSIBILITY_PENALTY;
} else {
score += HALF_WIDTH_KATAKANA_VOICING_SCORE;
}
} else if u == 0xFF9F {
if half_width_katakana_state
!= HalfWidthKatakana::DakutenOrHandakutenAllowed
{
score += IMPLAUSIBILITY_PENALTY;
} else {
score += HALF_WIDTH_KATAKANA_VOICING_SCORE;
}
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Other;
} else if (u >= 0x3041 && u <= 0x3093) || (u >= 0x30A1 && u <= 0x30F6) {
match u {
0x3090 // hiragana wi
| 0x3091 // hiragana we
| 0x30F0 // katakana wi
| 0x30F1 // katakana we
=> {
// Remove advantage over Big5 Hanzi
score += EUC_JP_SCORE_PER_NEAR_OBSOLETE_KANA;
}
_ => {
score += EUC_JP_SCORE_PER_KANA;
}
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {
if self.prev_prev_byte == 0x8F {
score += EUC_JP_SCORE_PER_OTHER_KANJI;
} else if self.prev_byte < 0xD0 {
score += EUC_JP_SCORE_PER_LEVEL_1_KANJI;
score += cjk_extra_score(u, &data::DETECTOR_DATA.frequent_kanji);
} else {
score += EUC_JP_SCORE_PER_LEVEL_2_KANJI;
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else {
match u {
0x3000 // Distinct from Korean, space
| 0x3001 // Distinct from Korean, enumeration comma
| 0x3002 // Distinct from Korean, full stop
| 0xFF08 // Distinct from Korean, parenthesis
| 0xFF09 // Distinct from Korean, parenthesis
=> {
score += CJ_PUNCTUATION;
}
0..=0x7F => {}
_ => {
score += CJK_OTHER;
}
}
self.prev = LatinCj::Other;
}
}
match result {
DecoderResult::InputEmpty => {
assert_eq!(read, 1);
}
DecoderResult::Malformed(_, _) => {
if b >= 0xA1
&& b <= 0xFE
&& self.prev_byte >= 0xA1
&& self.prev_byte <= 0xFE
&& ((self.prev_prev_byte != 0x8F
&& !(self.prev_byte == 0xA8 && b >= 0xDF && b <= 0xE6)
&& !(self.prev_byte == 0xAC && b >= 0xF4 && b <= 0xFC)
&& !(self.prev_byte == 0xAD && b >= 0xD8 && b <= 0xDE))
|| (self.prev_prev_byte == 0x8F
&& self.prev_byte != 0xA2
&& self.prev_byte != 0xA6
&& self.prev_byte != 0xA7
&& self.prev_byte != 0xA9
&& self.prev_byte != 0xAA
&& self.prev_byte != 0xAB
&& self.prev_byte != 0xED
&& !(self.prev_byte == 0xFE && b >= 0xF7)))
{
score += EUC_JP_EXTENSION_PENALTY;
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else {
return None;
}
}
DecoderResult::OutputFull => {
unreachable!();
}
}
self.prev_prev_byte = self.prev_byte;
self.prev_byte = b;
}
if last {
let (result, _, _) = self
.decoder
.decode_to_utf16_without_replacement(b"", &mut dst, true);
match result {
DecoderResult::InputEmpty => {}
DecoderResult::Malformed(_, _) => {
return None;
}
DecoderResult::OutputFull => {
unreachable!();
}
}
}
Some(score)
}
}
struct Big5Candidate {
decoder: Decoder,
prev: LatinCj,
prev_byte: u8,
pending_score: Option<i64>,
}
impl Big5Candidate {
fn maybe_set_as_pending(&mut self, s: i64) -> i64 {
assert!(self.pending_score.is_none());
if self.prev == LatinCj::Cj || !problematic_lead(self.prev_byte) {
s
} else {
self.pending_score = Some(s);
0
}
}
fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {
let mut score = 0i64;
let mut src = [0u8];
let mut dst = [0u16; 2];
for &b in buffer {
src[0] = b;
let (result, read, written) = self
.decoder
.decode_to_utf16_without_replacement(&src, &mut dst, false);
if written == 1 {
let u = dst[0];
if (u >= u16::from(b'a') && u <= u16::from(b'z'))
|| (u >= u16::from(b'A') && u <= u16::from(b'Z'))
{
self.pending_score = None; // Discard pending score
if self.prev == LatinCj::Cj {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::AsciiLetter;
} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {
if let Some(pending) = self.pending_score {
score += pending;
self.pending_score = None;
}
match self.prev_byte {
0xA4..=0xC6 => {
score += self.maybe_set_as_pending(BIG5_SCORE_PER_LEVEL_1_HANZI);
// score += cjk_extra_score(u, &data::DETECTOR_DATA.frequent_traditional);
}
_ => {
score += self.maybe_set_as_pending(BIG5_SCORE_PER_OTHER_HANZI);
}
}
if self.prev == LatinCj::AsciiLetter {
score += CJK_LATIN_ADJACENCY_PENALTY;
}
self.prev = LatinCj::Cj;
} else {
match u {
0x3000 // Distinct from Korean, space
| 0x3001 // Distinct from Korean, enumeration comma
| 0x3002 // Distinct from Korean, full stop
| 0xFF08 // Distinct from Korean, parenthesis
| 0xFF09 // Distinct from Korean, parenthesis
| 0xFF01 // Distinct from Japanese, exclamation
--> --------------------
--> maximum size reached
--> --------------------
[ Verzeichnis aufwärts0.78unsichere Verbindung
Übersetzung europäischer Sprachen durch Browser
]
|
2026-04-04
|
|
|
|
|
