/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this http://mozilla.org/MPL/2.0/. */ #define MOZ_PRETEND_NO_JSRUST 1#include "mozilla/Utf8.h" #include "mozilla/ArrayUtils.h" #include "mozilla/Assertions.h" #include "mozilla/EnumSet.h" #include "mozilla/IntegerRange.h" #include "mozilla/Span.h" using mozilla::AsChars;using mozilla::DecodeOneUtf8CodePoint;using mozilla::EnumSet;using mozilla::IntegerRange;using mozilla::IsAscii;using mozilla::IsUtf8;using mozilla::Span;using mozilla::Utf8Unit;static void TestUtf8Unit() {'A' );'A' );'A' ));'B' ));unsigned char asUnsigned = 'A' ;'B' ).toUnsignedChar() != asUnsigned);'@' );'#' );template <typename Char >struct ToUtf8Units {public :explicit ToUtf8Units(const Char * aStart, const Char * aEnd)const Utf8Unit lead;const Char * iter;const Char * const end;class AssertIfCalled {public :template <typename ... Args>void operator ()(Args&&... aArgs) {false , "AssertIfCalled instance was called" );// NOTE: For simplicity in treating |aCharN| identically regardless whether it's // a string literal or a more-generalized array, we require |aCharN| be // null-terminated. template <typename Char , size_t N>static void ExpectValidCodePoint(const Char (&aCharN)[N],"array must be null-terminated for |aCharN + N - 1| to " "compute the value of |aIter| as altered by " "DecodeOneUtf8CodePoint" );Char > simpleUnit(aCharN, aCharN + N - 1);auto simple =Char > complexUnit(aCharN, aCharN + N - 1);auto complex = DecodeOneUtf8CodePoint(enum class InvalidUtf8Reason {template <typename Char , size_t N>static void ExpectInvalidCodePointHelper(const Char (&aCharN)[N],"array must be null-terminated for |aCharN + N - 1| to " "compute the value of |aIter| as altered by " "DecodeOneUtf8CodePoint" );Char > simpleUnit(aCharN, aCharN + N - 1);auto simple =static_cast <const void *>(simpleUnit.iter) == aCharN);struct OnNotShortestForm {void operator ()(char32_t aBadCodePoint, uint8_t aUnitsObserved) {Char > complexUnit(aCharN, aCharN + N - 1);auto complex = DecodeOneUtf8CodePoint(static_cast <const void *>(complexUnit.iter) == aCharN);bool alreadyIterated = false ;for (InvalidUtf8Reason reason : reasons) {true ;switch (reason) {case InvalidUtf8Reason::BadLeadUnit:break ;case InvalidUtf8Reason::NotEnoughUnits:break ;case InvalidUtf8Reason::BadTrailingUnit:break ;case InvalidUtf8Reason::BadCodePoint:break ;case InvalidUtf8Reason::NotShortestForm:break ;// NOTE: For simplicity in treating |aCharN| identically regardless whether it's // a string literal or a more-generalized array, we require |aCharN| be // null-terminated in all these functions. template <typename Char , size_t N>static void ExpectBadLeadUnit(const Char (&aCharN)[N]) {template <typename Char , size_t N>static void ExpectNotEnoughUnits(const Char (&aCharN)[N],template <typename Char , size_t N>static void ExpectBadTrailingUnit(const Char (&aCharN)[N],template <typename Char , size_t N>static void ExpectNotShortestForm(const Char (&aCharN)[N],template <typename Char , size_t N>static void ExpectBadCodePoint(const Char (&aCharN)[N],static void TestIsUtf8() {// Note we include the U+0000 NULL in this one -- and that's fine. static const char asciiBytes[] = u8"How about a nice game of chess?" ;static const char endNonAsciiBytes[] = u8"Life is like a 🌯" ;static const unsigned char badLeading[] = {0x80};// Byte-counts // 1 static const char oneBytes[] = u8"A" ; // U+0041 LATIN CAPITAL LETTER A "U+0041 plus nul" );// 2 static const char twoBytes[] = u8"؆" ; // U+0606 ARABIC-INDIC CUBE ROOT "U+0606 in two bytes plus nul" );// 3 static const char threeBytes[] = u8"᨞" ; // U+1A1E BUGINESE PALLAWA "U+1A1E in three bytes plus nul" );// 4 static const char fourBytes[] ="🁡" ; // U+1F061 DOMINO TILE HORIZONTAL-06-06 "U+1F061 in four bytes plus nul" );// Max code point static const char maxCodePoint[] = u8"" ; // U+10FFFF "U+10FFFF in four bytes plus nul" );// One past max code point static const unsigned char onePastMaxCodePoint[] = {0xF4, 0x90, 0x80, 0x80,// Surrogate-related testing // (Note that the various code unit sequences here are null-terminated to // simplify life for ExpectValidCodePoint, which presumes null termination.) static const unsigned char justBeforeSurrogates[] = {0xED, 0x9F, 0xBF, 0x0};static const unsigned char leastSurrogate[] = {0xED, 0xA0, 0x80, 0x0};static const unsigned char arbitraryHighSurrogate[] = {0xED, 0xA2, 0x87, 0x0};static const unsigned char arbitraryLowSurrogate[] = {0xED, 0xB7, 0xAF, 0x0};static const unsigned char greatestSurrogate[] = {0xED, 0xBF, 0xBF, 0x0};static const unsigned char justAfterSurrogates[] = {0xEE, 0x80, 0x80, 0x0};static void TestDecodeOneValidUtf8CodePoint() {// NOTE: DecodeOneUtf8CodePoint decodes only *non*-ASCII code points that // consist of multiple code units, so there are no ASCII tests below. // Length two. "" , 0x80); // <control> "©" , 0xA9); // COPYRIGHT SIGN "¶" , 0xB6); // PILCROW SIGN "¾" , 0xBE); // VULGAR FRACTION THREE QUARTERS "÷" , 0xF7); // DIVISION SIGN "ÿ" , 0xFF); // LATIN SMALL LETTER Y WITH DIAERESIS "Ā" , 0x100); // LATIN CAPITAL LETTER A WITH MACRON "IJ" , 0x132); // LATIN CAPITAL LETTER LIGATURE IJ "ͼ" , 0x37C); // GREEK SMALL DOTTED LUNATE SIGMA SYMBOL "Ӝ" ,// CYRILLIC CAPITAL LETTER ZHE WITTH DIAERESIS "۩" , 0x6E9); // ARABIC PLACE OF SAJDAH "߿" , 0x7FF); // <not assigned> // Length three. "ࠀ" , 0x800); // SAMARITAN LETTER ALAF "ࡁ" , 0x841); // MANDAIC LETTER AB "ࣿ" , 0x8FF); // ARABIC MARK SIDEWAYS NOON GHUNNA "ஆ" , 0xB86); // TAMIL LETTER AA "༃" ,// TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA "࿉" ,// TIBETAN SYMBOL NOR BU (but on my system it really looks like // SOFT-SERVE ICE CREAM FROM ABOVE THE PLANE if you ask me) "ဪ" , 0x102A); // MYANMAR LETTER AU "ᚏ" , 0x168F); // OGHAM LETTER RUIS "\xE2\x80\xA8" , 0x2028); // (the hated) LINE SEPARATOR "\xE2\x80\xA9" ,// (the hated) PARAGRAPH SEPARATOR "☬" , 0x262C); // ADI SHAKTI "㊮" , 0x32AE); // CIRCLED IDEOGRAPH RESOURCE "㏖" , 0x33D6); // SQUARE MOL "ꔄ" , 0xA504); // VAI SYLLABLE WEEN "ퟕ" , 0xD7D5); // HANGUL JONGSEONG RIEUL-SSANGKIYEOK "" , 0xD7FF); // <not assigned> "" , 0xE000); // <Private Use> "鱗" , 0xF9F2); // CJK COMPATIBILITY IDEOGRAPH-F9F "﷽" , 0xFDFD); // ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHHHEEEEM "" , 0xFFFF); // <not assigned> // Length four. "𐀀" , 0x10000); // LINEAR B SYLLABLE B008 A "𔑀" , 0x14440); // ANATOLIAN HIEROGLYPH A058 "𝛗" , 0x1D6D7); // MATHEMATICAL BOLD SMALL PHI "💩" , 0x1F4A9); // PILE OF POO "🔫" , 0x1F52B); // PISTOL "🥌" , 0x1F94C); // CURLING STONE "🥏" , 0x1F94F); // FLYING DISC "𠍆" , 0x20346); // CJK UNIFIED IDEOGRAPH-20346 "𡠺" , 0x2183A); // CJK UNIFIED IDEOGRAPH-2183A "" , 0x417F6); // <not assigned> "" , 0x7E836); // <not assigned> "" , 0xFEF67); // <Plane 15 Private Use> "" , 0x10FFFF); // static void TestDecodeBadLeadUnit() {// These tests are actually exhaustive. unsigned char badLead[] = {'\0' , '\0' };for (uint8_t lead : IntegerRange(0b1000'0000, 0b1100' 0000)) {'1000; do {if (lead == 0b1111'1111) { break ;while (true );static void TestTooFewOrBadTrailingUnits() {// Lead unit indicates a two-byte code point. char truncatedTwo[] = {'\0' , '\0' };char badTrailTwo[] = {'\0' , '\0' , '\0' };for (uint8_t lead : IntegerRange(0b1100'0000, 0b1110' 0000)) {for (uint8_t trail : IntegerRange(0b0000'0000, 0b1000' 0000)) {for (uint8_t trail : IntegerRange(0b1100'0000, 0b1111' 1111)) {// Lead unit indicates a three-byte code point. char truncatedThreeOne[] = {'\0' , '\0' };char truncatedThreeTwo[] = {'\0' , '\0' , '\0' };unsigned char badTrailThree[] = {'\0' , '\0' , '\0' , '\0' };for (uint8_t lead : IntegerRange(0b1110'0000, 0b1111' 0000)) {'1111; // make valid to test overreads for (uint8_t mid : IntegerRange(0b0000'0000, 0b1000' 0000)) {'0000; do {if (mid == 0b1111'1111) { break ;while (true );'1111; for (uint8_t last : IntegerRange(0b0000'0000, 0b1000' 0000)) {'0000; do {if (last == 0b1111'1111) { break ;while (true );// Lead unit indicates a four-byte code point. char truncatedFourOne[] = {'\0' , '\0' };char truncatedFourTwo[] = {'\0' , '\0' , '\0' };char truncatedFourThree[] = {'\0' , '\0' , '\0' , '\0' };unsigned char badTrailFour[] = {'\0' , '\0' , '\0' , '\0' , '\0' };for (uint8_t lead : IntegerRange(0b1111'0000, 0b1111' 1000)) {'1111; // test for overreads for (uint8_t second : IntegerRange(0b0000'0000, 0b1000' 0000)) {'0000; do {if (second == 0b1111'1111) { break ;while (true );'1111; // test for overreads for (uint8_t third : IntegerRange(0b0000'0000, 0b1000' 0000)) {'0000; do {if (third == 0b1111'1111) { break ;while (true );'1111; for (uint8_t fourth : IntegerRange(0b0000'0000, 0b1000' 0000)) {'0000; do {if (fourth == 0b1111'1111) { break ;while (true );static void TestBadSurrogate() {// These tests are actually exhaustive. "\xED\x9F\xBF" , 0xD7FF); // last before surrogates "\xEE\x80\x80" , 0xE000); // first after surrogates // First invalid surrogate encoding is { 0xED, 0xA0, 0x80 }. Last invalid // surrogate encoding is { 0xED, 0xBF, 0xBF }. char badSurrogate[] = {'\xED' , '\0' , '\0' , '\0' };for (char32_t c = 0xD800; c < 0xE000; c++) {'0000 ^ ((c & 0b1111' 1100'0000) >> 6); '0000 ^ ((c & 0b0000' 0011'1111)); static void TestBadTooBig() {// These tests are actually exhaustive. "\xF4\x8F\xBF\xBF" , 0x10'FFFF); // last code point // Four-byte code points are // // 0b1111'0xxx 0b10xx'xxxx 0b10xx'xxxx 0b10xx'xxxx // // with 3 + 6 + 6 + 6 == 21 unconstrained bytes, so the structurally // representable limit (exclusive) is 2**21 - 1 == 2097152. char tooLargeCodePoint[] = {'\0' , '\0' , '\0' , '\0' , '\0' };for (char32_t c = 0x11'0000; c < (1 << 21); c++) { '0000 ^ ((c & 0b1' 1100'0000' 0000'0000' 0000) >> 18);'0000 ^ ((c & 0b0' 0011'1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0' 0000'0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0' 0000'0000' 0000'0011' 1111));static void TestBadCodePoint() {static void TestNotShortestForm() {// One-byte in two-byte. char oneInTwo[] = {'\0' , '\0' , '\0' };for (char32_t c = '\0' ; c < 0x80; c++) {'0000 ^ ((c & 0b0111' 1100'0000) >> 6); '0000 ^ ((c & 0b0000' 0011'1111)); // One-byte in three-byte. char oneInThree[] = {'\0' , '\0' , '\0' , '\0' };for (char32_t c = '\0' ; c < 0x80; c++) {'0000 ^ ((c & 0b1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0000' 0000'0011' 1111));// One-byte in four-byte. char oneInFour[] = {'\0' , '\0' , '\0' , '\0' , '\0' };for (char32_t c = '\0' ; c < 0x80; c++) {'0000 ^ ((c & 0b1' 1100'0000' 0000'0000' 0000) >> 18);'0000 ^ ((c & 0b0' 0011'1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0' 0000'0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0' 0000'0000' 0000'0011' 1111));// Two-byte in three-byte. char twoInThree[] = {'\0' , '\0' , '\0' , '\0' };for (char32_t c = 0x80; c < 0x800; c++) {'0000 ^ ((c & 0b1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0000' 0000'0011' 1111));// Two-byte in four-byte. char twoInFour[] = {'\0' , '\0' , '\0' , '\0' , '\0' };for (char32_t c = 0x80; c < 0x800; c++) {'0000 ^ ((c & 0b1' 1100'0000' 0000'0000' 0000) >> 18);'0000 ^ ((c & 0b0' 0011'1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0' 0000'0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0' 0000'0000' 0000'0011' 1111));// Three-byte in four-byte. char threeInFour[] = {'\0' , '\0' , '\0' , '\0' , '\0' };for (char32_t c = 0x800; c < 0x1'0000; c++) { '0000 ^ ((c & 0b1' 1100'0000' 0000'0000' 0000) >> 18);'0000 ^ ((c & 0b0' 0011'1111' 0000'0000' 0000) >> 12);'0000 ^ ((c & 0b0' 0000'0000' 1111'1100' 0000) >> 6);'0000 ^ ((c & 0b0' 0000'0000' 0000'0011' 1111));static void TestDecodeOneInvalidUtf8CodePoint() {static void TestDecodeOneUtf8CodePoint() {int main() {return 0;quality 95%
¤ Dauer der Verarbeitung: 0.17 Sekunden
(vorverarbeitet)
¤ *© Formatika GbR, Deutschland
