/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef nsUTF8Utils_h_
#define nsUTF8Utils_h_
// NB: This code may be used from non-XPCOM code, in particular, the
// standalone updater executable. That is, this file may be used in
// two ways: if MOZILLA_INTERNAL_API is defined, this file will
// provide signatures for the Mozilla abstract string types. It will
// use XPCOM assertion/debugging macros, etc.
#include <type_traits>
#include "mozilla/Assertions.h"
#include "nsCharTraits.h"
#ifdef MOZILLA_INTERNAL_API
# define UTF8UTILS_WARNING(msg) NS_WARNING(msg)
#else
# define UTF8UTILS_WARNING(msg)
#endif
class UTF8traits {
public:
static bool isASCII(
char aChar) {
return (aChar &
0x80) ==
0x00; }
static bool isInSeq(
char aChar) {
return (aChar &
0xC0) ==
0x80; }
static bool is2byte(
char aChar) {
return (aChar &
0xE0) ==
0xC0; }
static bool is3byte(
char aChar) {
return (aChar &
0xF0) ==
0xE0; }
static bool is4byte(
char aChar) {
return (aChar &
0xF8) ==
0xF0; }
static bool is5byte(
char aChar) {
return (aChar &
0xFC) ==
0xF8; }
static bool is6byte(
char aChar) {
return (aChar &
0xFE) ==
0xFC; }
// return the number of bytes in a sequence beginning with aChar
static int bytes(
char aChar) {
if (isASCII(aChar)) {
return 1;
}
if (is2byte(aChar)) {
return 2;
}
if (is3byte(aChar)) {
return 3;
}
if (is4byte(aChar)) {
return 4;
}
MOZ_ASSERT_UNREACHABLE(
"should not be used for in-sequence characters");
return 1;
}
};
/**
* Extract the next Unicode scalar value from the buffer and return it. The
* pointer passed in is advanced to the start of the next character in the
* buffer. Upon error, the return value is 0xFFFD, *aBuffer is advanced
* over the maximal valid prefix and *aErr is set to true (if aErr is not
* null).
*
* Note: This method never sets *aErr to false to allow error accumulation
* across multiple calls.
*
* Precondition: *aBuffer < aEnd
*/
class UTF8CharEnumerator {
public:
static inline char32_t NextChar(
const char** aBuffer,
const char* aEnd,
bool* aErr = nullptr) {
MOZ_ASSERT(aBuffer,
"null buffer pointer pointer");
MOZ_ASSERT(aEnd,
"null end pointer");
const unsigned char* p = reinterpret_cast<
const unsigned char*>(*aBuffer);
const unsigned char* end = reinterpret_cast<
const unsigned char*>(aEnd);
MOZ_ASSERT(p,
"null buffer");
MOZ_ASSERT(p < end,
"Bogus range");
unsigned char first = *p;
++p;
if (MOZ_LIKELY(first <
0x80U)) {
*aBuffer = reinterpret_cast<
const char*>(p);
return first;
}
// Unsigned underflow is defined behavior
if (MOZ_UNLIKELY((p == end) || ((first -
0xC2U) >= (
0xF5U -
0xC2U)))) {
*aBuffer = reinterpret_cast<
const char*>(p);
if (aErr) {
*aErr = true;
}
return 0xFFFDU;
}
unsigned char second = *p;
if (first <
0xE0U) {
// Two-byte
if (MOZ_LIKELY((second &
0xC0U) ==
0x80U)) {
++p;
*aBuffer = reinterpret_cast<
const char*>(p);
return ((uint32_t(first) &
0x1FU) <<
6) | (uint32_t(second) &
0x3FU);
}
*aBuffer = reinterpret_cast<
const char*>(p);
if (aErr) {
*aErr = true;
}
return 0xFFFDU;
}
if (MOZ_LIKELY(first <
0xF0U)) {
// Three-byte
unsigned char lower =
0x80U;
unsigned char upper =
0xBFU;
if (first ==
0xE0U) {
lower =
0xA0U;
}
else if (first ==
0xEDU) {
upper =
0x9FU;
}
if (MOZ_LIKELY(second >= lower && second <= upper)) {
++p;
if (MOZ_LIKELY(p != end)) {
unsigned char third = *p;
if (MOZ_LIKELY((third &
0xC0U) ==
0x80U)) {
++p;
*aBuffer = reinterpret_cast<
const char*>(p);
return ((uint32_t(first) &
0xFU) <<
12) |
((uint32_t(second) &
0x3FU) <<
6) |
(uint32_t(third) &
0x3FU);
}
}
}
*aBuffer = reinterpret_cast<
const char*>(p);
if (aErr) {
*aErr = true;
}
return 0xFFFDU;
}
// Four-byte
unsigned char lower =
0x80U;
unsigned char upper =
0xBFU;
if (first ==
0xF0U) {
lower =
0x90U;
}
else if (first ==
0xF4U) {
upper =
0x8FU;
}
if (MOZ_LIKELY(second >= lower && second <= upper)) {
++p;
if (MOZ_LIKELY(p != end)) {
unsigned char third = *p;
if (MOZ_LIKELY((third &
0xC0U) ==
0x80U)) {
++p;
if (MOZ_LIKELY(p != end)) {
unsigned char fourth = *p;
if (MOZ_LIKELY((fourth &
0xC0U) ==
0x80U)) {
++p;
*aBuffer = reinterpret_cast<
const char*>(p);
return ((uint32_t(first) &
0x7U) <<
18) |
((uint32_t(second) &
0x3FU) <<
12) |
((uint32_t(third) &
0x3FU) <<
6) |
(uint32_t(fourth) &
0x3FU);
}
}
}
}
}
*aBuffer = reinterpret_cast<
const char*>(p);
if (aErr) {
*aErr = true;
}
return 0xFFFDU;
}
};
/**
* Extract the next Unicode scalar value from the buffer and return it. The
* pointer passed in is advanced to the start of the next character in the
* buffer. Upon error, the return value is 0xFFFD, *aBuffer is advanced over
* the unpaired surrogate and *aErr is set to true (if aErr is not null).
*
* Note: This method never sets *aErr to false to allow error accumulation
* across multiple calls.
*
* Precondition: *aBuffer < aEnd
*/
class UTF16CharEnumerator {
public:
static inline char32_t NextChar(
const char16_t** aBuffer,
const char16_t* aEnd,
bool* aErr = nullptr) {
MOZ_ASSERT(aBuffer,
"null buffer pointer pointer");
MOZ_ASSERT(aEnd,
"null end pointer");
const char16_t* p = *aBuffer;
MOZ_ASSERT(p,
"null buffer");
MOZ_ASSERT(p < aEnd,
"Bogus range");
char16_t c = *p++;
// Let's use encoding_rs-style code golf here.
// Unsigned underflow is defined behavior
char16_t cMinusSurrogateStart = c -
0xD800U;
if (MOZ_LIKELY(cMinusSurrogateStart > (
0xDFFFU -
0xD800U))) {
*aBuffer = p;
return c;
}
if (MOZ_LIKELY(cMinusSurrogateStart <= (
0xDBFFU -
0xD800U))) {
// High surrogate
if (MOZ_LIKELY(p != aEnd)) {
char16_t second = *p;
// Unsigned underflow is defined behavior
if (MOZ_LIKELY((second -
0xDC00U) <= (
0xDFFFU -
0xDC00U))) {
*aBuffer = ++p;
return (uint32_t(c) <<
10) + uint32_t(second) -
(((
0xD800U <<
10) -
0x10000U) +
0xDC00U);
}
}
}
// Unpaired surrogate
*aBuffer = p;
if (aErr) {
*aErr = true;
}
return 0xFFFDU;
}
};
template <typename
Char, typename UnsignedT>
inline UnsignedT RewindToPriorUTF8Codepoint(
const Char* utf8Chars,
UnsignedT index) {
static_assert(std::is_same_v<
Char,
char> ||
std::is_same_v<
Char,
unsigned char> ||
std::is_same_v<
Char,
signed char>,
"UTF-8 data must be in 8-bit units");
static_assert(std::is_unsigned_v<UnsignedT>,
"index type must be unsigned");
while (index >
0 && (utf8Chars[index] &
0xC0) ==
0x80) --index;
return index;
}
#undef UTF8UTILS_WARNING
#endif /* !defined(nsUTF8Utils_h_) */