// Copyright Mozilla Foundation. See the COPYRIGHT // file at the top-level directory of this distribution. // // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or // https://www.apache.org/licenses/LICENSE-2.0> or the MIT license // <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your // option. This file may not be copied, modified, or distributed // except according to those terms.
pubfn utf8_valid_up_to(src: &[u8]) -> usize { letmut read = 0; 'outer: loop { letmut byte = { let src_remaining = &src[read..]; match validate_ascii(src_remaining) {
None => { return src.len();
}
Some((non_ascii, consumed)) => {
read += consumed;
non_ascii
}
}
}; // Check for the longest sequence to avoid checking twice for the // multi-byte sequences. This can't overflow with 64-bit address space, // because full 64 bits aren't in use. In the 32-bit PAE case, for this // to overflow would mean that the source slice would be so large that // the address space of the process would not have space for any code. // Therefore, the slice cannot be so long that this would overflow. if likely(read + 4 <= src.len()) { 'inner: loop { // At this point, `byte` is not included in `read`, because we // don't yet know that a) the UTF-8 sequence is valid and b) that there // is output space if it is an astral sequence. // Inspecting the lead byte directly is faster than what the // std lib does! if likely(in_inclusive_range8(byte, 0xC2, 0xDF)) { // Two-byte let second = unsafe { *(src.get_unchecked(read + 1)) }; if !in_inclusive_range8(second, 0x80, 0xBF) { break'outer;
}
read += 2;
// Next lead (manually inlined) if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if byte < 0x80 {
read += 1; continue'outer;
} continue'inner;
} break'inner;
} if likely(byte < 0xF0) { 'three: loop { // Three-byte let second = unsafe { *(src.get_unchecked(read + 1)) }; let third = unsafe { *(src.get_unchecked(read + 2)) }; if ((UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
| (third >> 6))
!= 2
{ break'outer;
}
read += 3;
// Next lead (manually inlined) if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if in_inclusive_range8(byte, 0xE0, 0xEF) { continue'three;
} if likely(byte < 0x80) {
read += 1; continue'outer;
} continue'inner;
} break'inner;
}
} // Four-byte let second = unsafe { *(src.get_unchecked(read + 1)) }; let third = unsafe { *(src.get_unchecked(read + 2)) }; let fourth = unsafe { *(src.get_unchecked(read + 3)) }; if (u16::from(
UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) },
) | u16::from(third >> 6)
| (u16::from(fourth & 0xC0) << 2))
!= 0x202
{ break'outer;
}
read += 4;
// Next lead if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if byte < 0x80 {
read += 1; continue'outer;
} continue'inner;
} break'inner;
}
} // We can't have a complete 4-byte sequence, but we could still have // one to three shorter sequences. 'tail: loop { // >= is better for bound check elision than == if read >= src.len() { break'outer;
}
byte = src[read]; // At this point, `byte` is not included in `read`, because we // don't yet know that a) the UTF-8 sequence is valid and b) that there // is output space if it is an astral sequence. // Inspecting the lead byte directly is faster than what the // std lib does! if byte < 0x80 {
read += 1; continue'tail;
} if in_inclusive_range8(byte, 0xC2, 0xDF) { // Two-byte let new_read = read + 2; if new_read > src.len() { break'outer;
} let second = src[read + 1]; if !in_inclusive_range8(second, 0x80, 0xBF) { break'outer;
}
read += 2; continue'tail;
} // We need to exclude valid four byte lead bytes, because // `UTF8_DATA.second_mask` covers if byte < 0xF0 { // Three-byte let new_read = read + 3; if new_read > src.len() { break'outer;
} let second = src[read + 1]; let third = src[read + 2]; if ((UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
| (third >> 6))
!= 2
{ break'outer;
}
read += 3; // `'tail` handles sequences shorter than 4, so // there can't be another sequence after this one. break'outer;
} break'outer;
}
}
read
}
#[cfg_attr(feature = "cargo-clippy", allow(never_loop, cyclomatic_complexity))] pubfn convert_utf8_to_utf16_up_to_invalid(src: &[u8], dst: &mut[u16]) -> (usize, usize) { letmut read = 0; letmut written = 0; 'outer: loop { letmut byte = { let src_remaining = &src[read..]; let dst_remaining = &mut dst[written..]; let length = ::core::cmp::min(src_remaining.len(), dst_remaining.len()); matchunsafe {
ascii_to_basic_latin(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
None => {
read += length;
written += length; break'outer;
}
Some((non_ascii, consumed)) => {
read += consumed;
written += consumed;
non_ascii
}
}
}; // Check for the longest sequence to avoid checking twice for the // multi-byte sequences. This can't overflow with 64-bit address space, // because full 64 bits aren't in use. In the 32-bit PAE case, for this // to overflow would mean that the source slice would be so large that // the address space of the process would not have space for any code. // Therefore, the slice cannot be so long that this would overflow. if likely(read + 4 <= src.len()) { 'inner: loop { // At this point, `byte` is not included in `read`, because we // don't yet know that a) the UTF-8 sequence is valid and b) that there // is output space if it is an astral sequence. // We know, thanks to `ascii_to_basic_latin` that there is output // space for at least one UTF-16 code unit, so no need to check // for output space in the BMP cases. // Inspecting the lead byte directly is faster than what the // std lib does! if likely(in_inclusive_range8(byte, 0xC2, 0xDF)) { // Two-byte let second = unsafe { *(src.get_unchecked(read + 1)) }; if !in_inclusive_range8(second, 0x80, 0xBF) { break'outer;
} unsafe {
*(dst.get_unchecked_mut(written)) =
((u16::from(byte) & 0x1F) << 6) | (u16::from(second) & 0x3F)
};
read += 2;
written += 1;
// Next lead (manually inlined) if written == dst.len() { break'outer;
} if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if byte < 0x80 { unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
read += 1;
written += 1; continue'outer;
} continue'inner;
} break'inner;
} if likely(byte < 0xF0) { 'three: loop { // Three-byte let second = unsafe { *(src.get_unchecked(read + 1)) }; let third = unsafe { *(src.get_unchecked(read + 2)) }; if ((UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
| (third >> 6))
!= 2
{ break'outer;
} let point = ((u16::from(byte) & 0xF) << 12)
| ((u16::from(second) & 0x3F) << 6)
| (u16::from(third) & 0x3F); unsafe { *(dst.get_unchecked_mut(written)) = point };
read += 3;
written += 1;
// Next lead (manually inlined) if written == dst.len() { break'outer;
} if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if in_inclusive_range8(byte, 0xE0, 0xEF) { continue'three;
} if likely(byte < 0x80) { unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
read += 1;
written += 1; continue'outer;
} continue'inner;
} break'inner;
}
} // Four-byte if written + 1 == dst.len() { break'outer;
} let second = unsafe { *(src.get_unchecked(read + 1)) }; let third = unsafe { *(src.get_unchecked(read + 2)) }; let fourth = unsafe { *(src.get_unchecked(read + 3)) }; if (u16::from(
UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) },
) | u16::from(third >> 6)
| (u16::from(fourth & 0xC0) << 2))
!= 0x202
{ break'outer;
} let point = ((u32::from(byte) & 0x7) << 18)
| ((u32::from(second) & 0x3F) << 12)
| ((u32::from(third) & 0x3F) << 6)
| (u32::from(fourth) & 0x3F); unsafe { *(dst.get_unchecked_mut(written)) = (0xD7C0 + (point >> 10)) as u16 }; unsafe {
*(dst.get_unchecked_mut(written + 1)) = (0xDC00 + (point & 0x3FF)) as u16
};
read += 4;
written += 2;
// Next lead if written == dst.len() { break'outer;
} if likely(read + 4 <= src.len()) {
byte = unsafe { *(src.get_unchecked(read)) }; if byte < 0x80 { unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
read += 1;
written += 1; continue'outer;
} continue'inner;
} break'inner;
}
} // We can't have a complete 4-byte sequence, but we could still have // one to three shorter sequences. 'tail: loop { // >= is better for bound check elision than == if read >= src.len() || written >= dst.len() { break'outer;
}
byte = src[read]; // At this point, `byte` is not included in `read`, because we // don't yet know that a) the UTF-8 sequence is valid and b) that there // is output space if it is an astral sequence. // Inspecting the lead byte directly is faster than what the // std lib does! if byte < 0x80 {
dst[written] = u16::from(byte);
read += 1;
written += 1; continue'tail;
} if in_inclusive_range8(byte, 0xC2, 0xDF) { // Two-byte let new_read = read + 2; if new_read > src.len() { break'outer;
} let second = src[read + 1]; if !in_inclusive_range8(second, 0x80, 0xBF) { break'outer;
}
dst[written] = ((u16::from(byte) & 0x1F) << 6) | (u16::from(second) & 0x3F);
read += 2;
written += 1; continue'tail;
} // We need to exclude valid four byte lead bytes, because // `UTF8_DATA.second_mask` covers if byte < 0xF0 { // Three-byte let new_read = read + 3; if new_read > src.len() { break'outer;
} let second = src[read + 1]; let third = src[read + 2]; if ((UTF8_DATA.table[usize::from(second)]
& unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
| (third >> 6))
!= 2
{ break'outer;
} let point = ((u16::from(byte) & 0xF) << 12)
| ((u16::from(second) & 0x3F) << 6)
| (u16::from(third) & 0x3F);
dst[written] = point;
read += 3;
written += 1; // `'tail` handles sequences shorter than 4, so // there can't be another sequence after this one. break'outer;
} break'outer;
}
}
(read, written)
}
pubstruct Utf8Decoder {
code_point: u32,
bytes_seen: usize, // 1, 2 or 3: counts continuations only
bytes_needed: usize, // 1, 2 or 3: counts continuations only
lower_boundary: u8,
upper_boundary: u8,
}
decoder_functions!(
{},
{ // This is the fast path. The rest runs only at the // start and end for partial sequences. ifself.bytes_needed == 0 {
dest.copy_utf8_up_to_invalid_from(&mut source);
}
},
{ ifself.bytes_needed != 0 { let bad_bytes = (self.bytes_seen + 1) as u8; self.code_point = 0; self.bytes_needed = 0; self.bytes_seen = 0; return (
DecoderResult::Malformed(bad_bytes, 0),
src_consumed,
dest.written(),
);
}
},
{ ifself.bytes_needed == 0 { if b < 0x80u8 {
destination_handle.write_ascii(b); continue;
} if b < 0xC2u8 { return (
DecoderResult::Malformed(1, 0),
unread_handle.consumed(),
destination_handle.written(),
);
} if b < 0xE0u8 { self.bytes_needed = 1; self.code_point = u32::from(b) & 0x1F; continue;
} if b < 0xF0u8 { if b == 0xE0u8 { self.lower_boundary = 0xA0u8;
} elseif b == 0xEDu8 { self.upper_boundary = 0x9Fu8;
} self.bytes_needed = 2; self.code_point = u32::from(b) & 0xF; continue;
} if b < 0xF5u8 { if b == 0xF0u8 { self.lower_boundary = 0x90u8;
} elseif b == 0xF4u8 { self.upper_boundary = 0x8Fu8;
} self.bytes_needed = 3; self.code_point = u32::from(b) & 0x7; continue;
} return (
DecoderResult::Malformed(1, 0),
unread_handle.consumed(),
destination_handle.written(),
);
} // self.bytes_needed != 0 if !(b >= self.lower_boundary && b <= self.upper_boundary) { let bad_bytes = (self.bytes_seen + 1) as u8; self.code_point = 0; self.bytes_needed = 0; self.bytes_seen = 0; self.lower_boundary = 0x80u8; self.upper_boundary = 0xBFu8; return (
DecoderResult::Malformed(bad_bytes, 0),
unread_handle.unread(),
destination_handle.written(),
);
} self.lower_boundary = 0x80u8; self.upper_boundary = 0xBFu8; self.code_point = (self.code_point << 6) | (u32::from(b) & 0x3F); self.bytes_seen += 1; ifself.bytes_seen != self.bytes_needed { continue;
} ifself.bytes_needed == 3 {
destination_handle.write_astral(self.code_point);
} else {
destination_handle.write_bmp_excl_ascii(self.code_point as u16);
} self.code_point = 0; self.bytes_needed = 0; self.bytes_seen = 0; continue;
}, self,
src_consumed,
dest,
source,
b,
destination_handle,
unread_handle,
check_space_astral
);
}
#[cfg_attr(feature = "cargo-clippy", allow(never_loop))] #[inline(never)] pubfn convert_utf16_to_utf8_partial_inner(src: &[u16], dst: &mut[u8]) -> (usize, usize) { letmut read = 0; letmut written = 0; 'outer: loop { letmut unit = { let src_remaining = &src[read..]; let dst_remaining = &mut dst[written..]; let length = if dst_remaining.len() < src_remaining.len() {
dst_remaining.len()
} else {
src_remaining.len()
}; matchunsafe {
basic_latin_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
None => {
read += length;
written += length; return (read, written);
}
Some((non_ascii, consumed)) => {
read += consumed;
written += consumed;
non_ascii
}
}
}; 'inner: loop { // The following loop is only broken out of as a goto forward. loop { // Unfortunately, this check isn't enough for the compiler to elide // the bound checks on writes to dst, which is why they are manually // elided, which makes a measurable difference. if written.checked_add(4).unwrap() > dst.len() { return (read, written);
}
read += 1; if unit < 0x800 { unsafe {
*(dst.get_unchecked_mut(written)) = (unit >> 6) as u8 | 0xC0u8;
written += 1;
*(dst.get_unchecked_mut(written)) = (unit & 0x3F) as u8 | 0x80u8;
written += 1;
} break;
} let unit_minus_surrogate_start = unit.wrapping_sub(0xD800); if likely(unit_minus_surrogate_start > (0xDFFF - 0xD800)) { unsafe {
*(dst.get_unchecked_mut(written)) = (unit >> 12) as u8 | 0xE0u8;
written += 1;
*(dst.get_unchecked_mut(written)) = ((unit & 0xFC0) >> 6) as u8 | 0x80u8;
written += 1;
*(dst.get_unchecked_mut(written)) = (unit & 0x3F) as u8 | 0x80u8;
written += 1;
} break;
} if likely(unit_minus_surrogate_start <= (0xDBFF - 0xD800)) { // high surrogate // read > src.len() is impossible, but using // >= instead of == allows the compiler to elide a bound check. if read >= src.len() {
debug_assert_eq!(read, src.len()); // Unpaired surrogate at the end of the buffer. unsafe {
*(dst.get_unchecked_mut(written)) = 0xEFu8;
written += 1;
*(dst.get_unchecked_mut(written)) = 0xBFu8;
written += 1;
*(dst.get_unchecked_mut(written)) = 0xBDu8;
written += 1;
} return (read, written);
} let second = src[read]; let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00); if likely(second_minus_low_surrogate_start <= (0xDFFF - 0xDC00)) { // The next code unit is a low surrogate. Advance position.
read += 1; let astral = (u32::from(unit) << 10) + u32::from(second)
- (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32); unsafe {
*(dst.get_unchecked_mut(written)) = (astral >> 18) as u8 | 0xF0u8;
written += 1;
*(dst.get_unchecked_mut(written)) =
((astral & 0x3F000u32) >> 12) as u8 | 0x80u8;
written += 1;
*(dst.get_unchecked_mut(written)) =
((astral & 0xFC0u32) >> 6) as u8 | 0x80u8;
written += 1;
*(dst.get_unchecked_mut(written)) = (astral & 0x3F) as u8 | 0x80u8;
written += 1;
} break;
} // The next code unit is not a low surrogate. Don't advance // position and treat the high surrogate as unpaired. // Fall through
} // Unpaired low surrogate unsafe {
*(dst.get_unchecked_mut(written)) = 0xEFu8;
written += 1;
*(dst.get_unchecked_mut(written)) = 0xBFu8;
written += 1;
*(dst.get_unchecked_mut(written)) = 0xBDu8;
written += 1;
} break;
} // Now see if the next unit is Basic Latin // read > src.len() is impossible, but using // >= instead of == allows the compiler to elide a bound check. if read >= src.len() {
debug_assert_eq!(read, src.len()); return (read, written);
}
unit = src[read]; if unlikely(unit < 0x80) { // written > dst.len() is impossible, but using // >= instead of == allows the compiler to elide a bound check. if written >= dst.len() {
debug_assert_eq!(written, dst.len()); return (read, written);
}
dst[written] = unit as u8;
read += 1;
written += 1; // Mysteriously, adding a punctuation check here makes // the expected benificiary cases *slower*! continue'outer;
} continue'inner;
}
}
}
#[inline(never)] pubfn convert_utf16_to_utf8_partial_tail(src: &[u16], dst: &mut [u8]) -> (usize, usize) { // Everything below is cold code! letmut read = 0; letmut written = 0; letmut unit = src[read]; // We now have up to 3 output slots, so an astral character // will not fit. if unit < 0x800 { loop { if unit < 0x80 { if written >= dst.len() { return (read, written);
}
read += 1;
dst[written] = unit as u8;
written += 1;
} elseif unit < 0x800 { if written + 2 > dst.len() { return (read, written);
}
read += 1;
dst[written] = (unit >> 6) as u8 | 0xC0u8;
written += 1;
dst[written] = (unit & 0x3F) as u8 | 0x80u8;
written += 1;
} else { return (read, written);
} // read > src.len() is impossible, but using // >= instead of == allows the compiler to elide a bound check. if read >= src.len() {
debug_assert_eq!(read, src.len()); return (read, written);
}
unit = src[read];
}
} // Could be an unpaired surrogate, but we'll need 3 output // slots in any case. if written + 3 > dst.len() { return (read, written);
}
read += 1; let unit_minus_surrogate_start = unit.wrapping_sub(0xD800); if unit_minus_surrogate_start <= (0xDFFF - 0xD800) { // Got surrogate if unit_minus_surrogate_start <= (0xDBFF - 0xD800) { // Got high surrogate if read >= src.len() { // Unpaired high surrogate
unit = 0xFFFD;
} else { let second = src[read]; if in_inclusive_range16(second, 0xDC00, 0xDFFF) { // Valid surrogate pair, but we know it won't fit.
read -= 1; return (read, written);
} // Unpaired high
unit = 0xFFFD;
}
} else { // Unpaired low
unit = 0xFFFD;
}
}
dst[written] = (unit >> 12) as u8 | 0xE0u8;
written += 1;
dst[written] = ((unit & 0xFC0) >> 6) as u8 | 0x80u8;
written += 1;
dst[written] = (unit & 0x3F) as u8 | 0x80u8;
written += 1;
debug_assert_eq!(written, dst.len());
(read, written)
}
// Highest four-byte with last byte replaced with 0xFF
decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xFF", "a\u{FFFD}\u{FFFD}");
decode_utf8_to_utf8(b"a\xF4\x8F\xBF\xFFZ", "a\u{FFFD}\u{FFFD}Z");
}
¤ Diese beiden folgenden Angebotsgruppen bietet das Unternehmen0.30Angebot
(Wie Sie bei der Firma Beratungs- und Dienstleistungen beauftragen können 2026-06-20)
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
