/* CN */ /* the first few enum constants must keep their values because they correspond to myConverterArray[] */
GB2312_1=1,
ISO_IR_165=2,
CNS_11643=3,
/* is the StateEnum charset value for a DBCS charset? */ #if UCONFIG_ONLY_HTML_CONVERSION #define IS_JP_DBCS(cs) (JISX208==(cs)) #else #define IS_JP_DBCS(cs) (JISX208<=(cs) && (cs)<=KSC5601) #endif
typedefstruct ISO2022State {
int8_t cs[4]; /* charset number for SI (G0)/SO (G1)/SS2 (G2)/SS3 (G3) */
int8_t g; /* 0..3 for G0..G3 (SI/SO/SS2/SS3) */
int8_t prevG; /* g before single shift (SS2 or SS3) */
} ISO2022State;
typedefenum
{
INVALID_2022 = -1, /*Doesn't correspond to a valid iso 2022 escape sequence*/
VALID_NON_TERMINAL_2022 = 0, /*so far corresponds to a valid iso 2022 escape sequence*/
VALID_TERMINAL_2022 = 1, /*corresponds to a valid iso 2022 escape sequence*/
VALID_MAYBE_TERMINAL_2022 = 2/*so far matches one iso 2022 escape sequence, but by adding more characters might match another escape sequence*/
} UCNV_TableStates_2022;
staticinlinevoid
setInitialStateFromUnicodeKR(UConverter* converter,UConverterDataISO2022 *myConverterData){ /* in ISO-2022-KR the designator sequence appears only once *inafilesoweappenditonlyonce
*/ if( converter->charErrorBufferLength==0){
uprv_memset(myConverterData, 0, sizeof(UConverterDataISO2022));
myConverterData->currentType = ASCII1;
cnv->fromUnicodeStatus =false; if(pArgs->locale){
uprv_strncpy(myLocale, pArgs->locale, sizeof(myLocale)-1);
}
version = pArgs->options & UCNV_OPTIONS_VERSION_MASK;
myConverterData->version = version; if(myLocale[0]=='j' && (myLocale[1]=='a'|| myLocale[1]=='p') &&
(myLocale[2]=='_' || myLocale[2]=='\0'))
{ /* open the required converters and cache them */ if(version>MAX_JA_VERSION) { // ICU 55 fails to open a converter for an unsupported version. // Previously, it fell back to version 0, but that would yield // unexpected behavior.
*errorCode = U_MISSING_RESOURCE_ERROR; return;
} if(jpCharsetMasks[version]&CSM(ISO8859_7)) {
myConverterData->myConverterArray[ISO8859_7] =
ucnv_loadSharedData("ISO8859_7", &stackPieces, &stackArgs, errorCode);
}
myConverterData->myConverterArray[JISX208] =
ucnv_loadSharedData("Shift-JIS", &stackPieces, &stackArgs, errorCode); if(jpCharsetMasks[version]&CSM(JISX212)) {
myConverterData->myConverterArray[JISX212] =
ucnv_loadSharedData("jisx-212", &stackPieces, &stackArgs, errorCode);
} if(jpCharsetMasks[version]&CSM(GB2312)) {
myConverterData->myConverterArray[GB2312] =
ucnv_loadSharedData("ibm-5478", &stackPieces, &stackArgs, errorCode); /* gb_2312_80-1 */
} if(jpCharsetMasks[version]&CSM(KSC5601)) {
myConverterData->myConverterArray[KSC5601] =
ucnv_loadSharedData("ksc_5601", &stackPieces, &stackArgs, errorCode);
}
/* set the function pointers to appropriate functions */
cnv->sharedData = const_cast<UConverterSharedData*>(&_ISO2022JPData);
uprv_strcpy(myConverterData->locale,"ja");
(void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ja,version=");
size_t len = uprv_strlen(myConverterData->name);
myConverterData->name[len] = static_cast<char>(myConverterData->version + static_cast<int>('0'));
myConverterData->name[len+1]='\0';
} #if !UCONFIG_ONLY_HTML_CONVERSION elseif(myLocale[0]=='k' && (myLocale[1]=='o'|| myLocale[1]=='r') &&
(myLocale[2]=='_' || myLocale[2]=='\0'))
{ if(version>1) { // ICU 55 fails to open a converter for an unsupported version. // Previously, it fell back to version 0, but that would yield // unexpected behavior.
*errorCode = U_MISSING_RESOURCE_ERROR; return;
} constchar *cnvName; if(version==1) {
cnvName="icu-internal-25546";
} else {
cnvName="ibm-949";
myConverterData->version=version=0;
} if(pArgs->onlyTestIsLoadable) {
ucnv_canCreateConverter(cnvName, errorCode); /* errorCode carries result */
uprv_free(cnv->extraInfo);
cnv->extraInfo=nullptr; return;
} else {
myConverterData->currentConverter=ucnv_open(cnvName, errorCode); if (U_FAILURE(*errorCode)) {
_ISO2022Close(cnv); return;
}
/* initialize the state variables */
setInitialStateToUnicodeKR(cnv, myConverterData);
setInitialStateFromUnicodeKR(cnv, myConverterData);
/* set the function pointers to appropriate functions */
cnv->sharedData = const_cast<UConverterSharedData*>(&_ISO2022KRData);
uprv_strcpy(myConverterData->locale,"ko");
}
} elseif(((myLocale[0]=='z' && myLocale[1]=='h') || (myLocale[0]=='c'&& myLocale[1]=='n'))&&
(myLocale[2]=='_' || myLocale[2]=='\0'))
{ if(version>2) { // ICU 55 fails to open a converter for an unsupported version. // Previously, it fell back to version 0, but that would yield // unexpected behavior.
*errorCode = U_MISSING_RESOURCE_ERROR; return;
}
/* open the required converters and cache them */
myConverterData->myConverterArray[GB2312_1] =
ucnv_loadSharedData("ibm-5478", &stackPieces, &stackArgs, errorCode); if(version>=1) {
myConverterData->myConverterArray[ISO_IR_165] =
ucnv_loadSharedData("iso-ir-165", &stackPieces, &stackArgs, errorCode);
}
myConverterData->myConverterArray[CNS_11643] =
ucnv_loadSharedData("cns-11643-1992", &stackPieces, &stackArgs, errorCode);
/* set the function pointers to appropriate functions */
cnv->sharedData = const_cast<UConverterSharedData*>(&_ISO2022CNData);
uprv_strcpy(myConverterData->locale,"cn");
cnv->sharedData=(UConverterSharedData*)&_ISO2022Data; /* initialize the state variables */
uprv_strcpy(myConverterData->name,"ISO_2022"); #else
*errorCode = U_MISSING_RESOURCE_ERROR; // Was U_UNSUPPORTED_ERROR but changed in ICU 55 to a more standard // data loading error code. return; #endif
}
if (converter->extraInfo != nullptr) { /*close the array of converter pointers and free the memory*/ for (i=0; i<UCNV_2022_MAX_CONVERTERS; i++) { if(array[i]!=nullptr) {
ucnv_unloadSharedDataIfReady(array[i]);
}
}
/*runs through a state machine to determine the escape sequence - codepage correspondence
*/ staticvoid
changeState_2022(UConverter* _this, constchar** source, constchar* sourceLimit,
Variant2022 var,
UErrorCode* err){
UCNV_TableStates_2022 value;
UConverterDataISO2022* myData2022 = static_cast<UConverterDataISO2022*>(_this->extraInfo);
uint32_t key = myData2022->key;
int32_t offset = 0;
int8_t initialToULength = _this->toULength; char c;
value = VALID_NON_TERMINAL_2022; while (*source < sourceLimit) {
c = *(*source)++;
_this->toUBytes[_this->toULength++] = static_cast<uint8_t>(c);
value = getKey_2022(c, reinterpret_cast<int32_t*>(&key), &offset);
switch (value){
case VALID_NON_TERMINAL_2022 : /* continue with the loop */ break;
case VALID_TERMINAL_2022:
key = 0; goto DONE;
case INVALID_2022: goto DONE;
case VALID_MAYBE_TERMINAL_2022: #ifdef U_ENABLE_GENERIC_ISO_2022 /* ESC ( B is ambiguous only for ISO_2022 itself */ if(var == ISO_2022) { /* discard toUBytes[] for ESC ( B because this sequence is correct and complete */
_this->toULength = 0;
/* TODO need to indicate that ESC ( B was seen; if failure, then need to replay from source or from MBCS-style replay */
/* continue with the loop */
value = VALID_NON_TERMINAL_2022; break;
} else #endif
{ /* not ISO_2022 itself, finish here */
value = VALID_TERMINAL_2022;
key = 0; goto DONE;
}
}
}
DONE:
myData2022->key = key;
if (value == VALID_NON_TERMINAL_2022) { /* indicate that the escape sequence is incomplete: key!=0 */ return;
} elseif (value == INVALID_2022 ) {
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
} else/* value == VALID_TERMINAL_2022 */ { switch(var){ #ifdef U_ENABLE_GENERIC_ISO_2022 case ISO_2022:
{ constchar *chosenConverterName = escSeqStateTable_Result_2022[offset]; if(chosenConverterName == nullptr) { /* SS2 or SS3 */
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
_this->toUCallbackReason = UCNV_UNASSIGNED; return;
}
_this->mode = UCNV_SI;
ucnv_close(myData2022->currentConverter);
myData2022->currentConverter = myUConverter = ucnv_open(chosenConverterName, err); if(U_SUCCESS(*err)) {
myUConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP;
_this->mode = UCNV_SO;
} break;
} #endif case ISO_2022_JP:
{
StateEnum tempState = static_cast<StateEnum>(nextStateToUnicodeJP[offset]); switch(tempState) { case INVALID_STATE:
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE; break; case SS2_STATE: if(myData2022->toU2022State.cs[2]!=0) { if(myData2022->toU2022State.g<2) {
myData2022->toU2022State.prevG=myData2022->toU2022State.g;
}
myData2022->toU2022State.g=2;
} else { /* illegal to have SS2 before a matching designator */
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
} break; /* case SS3_STATE: not used in ISO-2022-JP-x */ case ISO8859_1: case ISO8859_7: if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) {
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
} else { /* G2 charset for SS2 */
myData2022->toU2022State.cs[2] = static_cast<int8_t>(tempState);
} break; default: if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) {
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
} else { /* G0 charset */
myData2022->toU2022State.cs[0] = static_cast<int8_t>(tempState);
} break;
}
} break; #if !UCONFIG_ONLY_HTML_CONVERSION case ISO_2022_CN:
{
StateEnum tempState = static_cast<StateEnum>(nextStateToUnicodeCN[offset]); switch(tempState) { case INVALID_STATE:
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE; break; case SS2_STATE: if(myData2022->toU2022State.cs[2]!=0) { if(myData2022->toU2022State.g<2) {
myData2022->toU2022State.prevG=myData2022->toU2022State.g;
}
myData2022->toU2022State.g=2;
} else { /* illegal to have SS2 before a matching designator */
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
} break; case SS3_STATE: if(myData2022->toU2022State.cs[3]!=0) { if(myData2022->toU2022State.g<2) {
myData2022->toU2022State.prevG=myData2022->toU2022State.g;
}
myData2022->toU2022State.g=3;
} else { /* illegal to have SS3 before a matching designator */
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
} break; case ISO_IR_165: if(myData2022->version==0) {
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE; break;
}
U_FALLTHROUGH; case GB2312_1:
U_FALLTHROUGH; case CNS_11643_1:
myData2022->toU2022State.cs[1] = static_cast<int8_t>(tempState); break; case CNS_11643_2:
myData2022->toU2022State.cs[2] = static_cast<int8_t>(tempState); break; default: /* other CNS 11643 planes */ if(myData2022->version==0) {
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
} else {
myData2022->toU2022State.cs[3] = static_cast<int8_t>(tempState);
} break;
}
} break; case ISO_2022_KR: if(offset==0x30){ /* nothing to be done, just accept this one escape sequence */
} else {
*err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
} break; #endif// !UCONFIG_ONLY_HTML_CONVERSION
default:
*err = U_ILLEGAL_ESCAPE_SEQUENCE; break;
}
} if(U_SUCCESS(*err)) {
_this->toULength = 0;
} elseif(*err==U_ILLEGAL_ESCAPE_SEQUENCE) { if(_this->toULength>1) { /* *Ticket5691:consistentillegalsequences: *-Weincludeatleastthefirstbyte(ESC)intheillegalsequence. *-Ifanyofthenon-initialbytescouldbethestartofacharacter, *westoptheillegalsequencebeforethefirstoneofthose. *Inescapesequences,allfollowingbytesare"printable",thatis, *unlesstheyarecompletelyillegal(>7finSBCS,outside21..7einDBCS), *theyarevalidsingle/leadbytes. *Forsimplicity,wealwaysonlyreporttheinitialESCbyteasthe *illegalsequenceandbackoutallotherbyteswelookedat.
*/ /* Back out some bytes. */
int8_t backOutDistance=_this->toULength-1;
int8_t bytesFromThisBuffer=_this->toULength-initialToULength; if(backOutDistance<=bytesFromThisBuffer) { /* same as initialToULength<=1 */
*source-=backOutDistance;
} else { /* Back out bytes from the previous buffer: Need to replay them. */
_this->preToULength = static_cast<int8_t>(bytesFromThisBuffer - backOutDistance); /* same as -(initialToULength-1) */ /* preToULength is negative! */
uprv_memcpy(_this->preToU, _this->toUBytes+1, -_this->preToULength);
*source-=bytesFromThisBuffer;
}
_this->toULength=1;
}
} elseif(*err==U_UNSUPPORTED_ESCAPE_SEQUENCE) {
_this->toUCallbackReason = UCNV_UNASSIGNED;
}
}
#if !UCONFIG_ONLY_HTML_CONVERSION /*Checks the characters of the buffer against valid 2022 escape sequences *ifthematchwereturnapointertotheinitialstartofthesequenceotherwise *wereturnsourceLimit
*/ /*for 2022 looks ahead in the stream *todeterminethelongestpossibleconvertible *datastream
*/ staticinlineconstchar*
getEndOfBuffer_2022(constchar** source, constchar* sourceLimit,
UBool /*flush*/){
constchar* mySource = *source;
#ifdef U_ENABLE_GENERIC_ISO_2022 if (*source >= sourceLimit) return sourceLimit;
do{
if (*mySource == ESC_2022){
int8_t i;
int32_t key = 0;
int32_t offset;
UCNV_TableStates_2022 value = VALID_NON_TERMINAL_2022;
/* Kludge: I could not *figureoutthereasonforvalidatinganescapesequence *twice-oncehereandonceinchangeState_2022(). *isitpossibletohaveanESCcharacterinaISO2022 *bytestreamwhichisvalidinacodepage?Isitlegal?
*/ for (i=0;
(mySource+i < sourceLimit)&&(value == VALID_NON_TERMINAL_2022);
i++) {
value = getKey_2022(*(mySource+i), &key, &offset);
} if (value > 0 || *mySource==ESC_2022) return mySource;
/* This inline function replicates code in _MBCSSingleFromUChar32() function in ucnvmbcs.c *anyfuturechangein_MBCSSingleFromUChar32()functionshouldbereflectedhere. *@paramretvalpointertooutputbyte *@return1roundtripbyte0nomapping-1fallbackbyte
*/ staticinline int32_t
MBCS_SINGLE_FROM_UCHAR32(UConverterSharedData* sharedData,
UChar32 c,
uint32_t* retval,
UBool useFallback)
{ const uint16_t *table;
int32_t value; /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { return0;
} /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */
table=sharedData->mbcs.fromUnicodeTable; /* get the byte for the output */
value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); /* is this code point assigned, or do we use fallbacks? */
*retval = static_cast<uint32_t>(value & 0xff); if(value>=0xf00) { return1; /* roundtrip */
} elseif(useFallback ? value>=0x800 : value>=0xc00) { return -1; /* fallback taken */
} else { return0; /* no mapping */
}
}
/* *Checkthattheresultisa2-bytevaluewitheachbyteintherangeA1..FE *(strictEUCDBCS)beforeacceptingitandsubtracting0x80fromeachbyte *tomoveittotheISO2022range21..7E. *Return0ifoutofrange.
*/ staticinline uint32_t
_2022FromGR94DBCS(uint32_t value) { if (static_cast<uint16_t>(value - 0xa1a1) <= (0xfefe - 0xa1a1) &&
static_cast<uint8_t>(value - 0xa1) <= (0xfe - 0xa1)
) { return value - 0x8080; /* shift down to 21..7e byte range */
} else { return0; /* not valid for ISO 2022 */
}
}
#if0/* 5691: Call sites now check for validity. They can just += 0x8080 after that. */ /* *Thismethoddoesthereverseof_2022FromGR94DBCS().Giventhe2022codepoint,itreturnsthe *2bytevaluethatisintherangeA1..FEforeachbyte.Otherwiseitreturnsthe2022codepoint *unchanged.
*/ staticinline uint32_t
_2022ToGR94DBCS(uint32_t value) {
uint32_t returnValue = value + 0x8080; if( (uint16_t)(returnValue - 0xa1a1) <= (0xfefe - 0xa1a1) &&
(uint8_t)(returnValue - 0xa1) <= (0xfe - 0xa1)) { return returnValue;
} else { return value;
}
} #endif
realSourceLimit = args->sourceLimit; while (args->source < realSourceLimit) { if(myData->key == 0) { /* are we in the middle of an escape sequence? */ /*Find the end of the buffer e.g : Next Escape Seq | end of Buffer*/
mySourceLimit = getEndOfBuffer_2022(&(args->source), realSourceLimit, args->flush);
/* convert to before the ESC or until the end of the buffer */
myData->isFirstBuffer=false;
sourceStart = args->source;
myTargetStart = args->target;
args->converter = myData->currentConverter;
ucnv_toUnicode(args->converter,
&args->target,
args->targetLimit,
&args->source,
mySourceLimit,
args->offsets,
(UBool)(args->flush && mySourceLimit == realSourceLimit),
err);
args->converter = saveThis;
/* set up the state */
converterData = static_cast<UConverterDataISO2022*>(cnv->extraInfo);
pFromU2022State = &converterData->fromU2022State;
choiceCount = 0;
/* check if the last codepoint of previous buffer was a lead surrogate*/ if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) { goto getTrail;
}
sourceChar = *(source++); /*check if the char is a First surrogate*/ if(U16_IS_SURROGATE(sourceChar)) { if(U16_IS_SURROGATE_LEAD(sourceChar)) {
getTrail: /*look ahead to find the trail surrogate*/ if(source < sourceLimit) { /* test the following code unit */
char16_t trail = *source; if(U16_IS_TRAIL(trail)) {
source++;
sourceChar=U16_GET_SUPPLEMENTARY(sourceChar, trail);
cnv->fromUChar32=0x00; /* convert this supplementary code point */ /* exit this condition tree */
} else { /* this is an unmatched lead code unit (1st surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
} else { /* no more input */
cnv->fromUChar32=sourceChar; break;
}
} else { /* this is an unmatched trail code unit (2nd surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
}
/* do not convert SO/SI/ESC */ if(IS_2022_CONTROL(sourceChar)) { /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
if(sourceChar == CR || sourceChar == LF) { /* reset the G2 state at the end of a line (conversion got us into ASCII or JISX201 already) */
pFromU2022State->cs[2] = 0;
choiceCount = 0;
}
switch(mySourceChar) { case UCNV_SI: if(myData->version==3) {
pToU2022State->g=0; continue;
} else { /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
myData->isEmptySegment = false; /* reset this, we have a different error */ break;
}
case UCNV_SO: if(myData->version==3) { /* JIS7: switch to G1 half-width Katakana */
pToU2022State->cs[1] = static_cast<int8_t>(HWKANA_7BIT);
pToU2022State->g=1; continue;
} else { /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
myData->isEmptySegment = false; /* reset this, we have a different error */ break;
}
/* If in ISO-2022-JP only and we successfully completed an escape sequence, but previous segment was empty, create an error */ if(myData->version==0 && myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
args->converter->toUCallbackReason = UCNV_IRREGULAR;
args->converter->toULength = static_cast<int8_t>(toULengthBefore + (mySource - mySourceBefore));
}
}
/* invalid or illegal escape sequence */ if(U_FAILURE(*err)){
args->target = myTarget;
args->source = mySource;
myData->isEmptySegment = false; /* Reset to avoid future spurious errors */ return;
} /* If we successfully completed an escape sequence, we begin a new segment, empty so far */ if(myData->key==0) {
myData->isEmptySegment = true;
} continue;
/* ISO-2022-JP does not use single-byte (C1) SS2 and SS3 */
case CR: case LF: /* automatically reset to single-byte mode */ if (static_cast<StateEnum>(pToU2022State->cs[0]) != ASCII &&
static_cast<StateEnum>(pToU2022State->cs[0]) != JISX201) {
pToU2022State->cs[0] = static_cast<int8_t>(ASCII);
}
pToU2022State->cs[2] = 0;
pToU2022State->g = 0;
U_FALLTHROUGH; default: /* convert one or two bytes */
myData->isEmptySegment = false;
cs = static_cast<StateEnum>(pToU2022State->cs[pToU2022State->g]); if (static_cast<uint8_t>(mySourceChar - 0xa1) <= (0xdf - 0xa1) && myData->version == 4 &&
!IS_JP_DBCS(cs)
) { /* 8-bit halfwidth katakana in any single-byte mode for JIS8 */
targetUniChar = mySourceChar + (HWKANA_START - 0xa1);
/* return from a single-shift state to the previous one */ if(pToU2022State->g >= 2) {
pToU2022State->g=pToU2022State->prevG;
}
} elseswitch(cs) { case ASCII: if(mySourceChar <= 0x7f) {
targetUniChar = mySourceChar;
} break; case ISO8859_1: if(mySourceChar <= 0x7f) {
targetUniChar = mySourceChar + 0x80;
} /* return from a single-shift state to the previous one */
pToU2022State->g=pToU2022State->prevG; break; case ISO8859_7: if(mySourceChar <= 0x7f) { /* convert mySourceChar+0x80 to use a normal 8-bit table */
targetUniChar =
_MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(
myData->myConverterArray[cs],
mySourceChar + 0x80);
} /* return from a single-shift state to the previous one */
pToU2022State->g=pToU2022State->prevG; break; case JISX201: if(mySourceChar <= 0x7f) {
targetUniChar = jisx201ToU(mySourceChar);
} break; case HWKANA_7BIT: if (static_cast<uint8_t>(mySourceChar - 0x21) <= (0x5f - 0x21)) { /* 7-bit halfwidth Katakana */
targetUniChar = mySourceChar + (HWKANA_START - 0x21);
} break; default: /* G0 DBCS */ if(mySource < mySourceLimit) { int leadIsOk, trailIsOk;
uint8_t trailByte;
getTrailByte:
trailByte = static_cast<uint8_t>(*mySource); /* *Ticket5691:consistentillegalsequences: *-Weincludeatleastthefirstbyteintheillegalsequence. *-Ifanyofthenon-initialbytescouldbethestartofacharacter, *westoptheillegalsequencebeforethefirstoneofthose. * *InISO-2022DBCS,ifthesecondbyteisinthe21..7erangeoris *anESC/SO/SI,wereportonlythefirstbyteastheillegalsequence. *Otherwiseweconvertorreportthepairofbytes.
*/
leadIsOk = static_cast<uint8_t>(mySourceChar - 0x21) <= (0x7e - 0x21);
trailIsOk = static_cast<uint8_t>(trailByte - 0x21) <= (0x7e - 0x21); if (leadIsOk && trailIsOk) {
++mySource;
tmpSourceChar = (mySourceChar << 8) | trailByte; if(cs == JISX208) {
_2022ToSJIS(static_cast<uint8_t>(mySourceChar), trailByte, tempBuf);
mySourceChar = tmpSourceChar;
} else { /* Copy before we modify tmpSourceChar so toUnicodeCallback() sees the correct bytes. */
mySourceChar = tmpSourceChar; if (cs == KSC5601) {
tmpSourceChar += 0x8080; /* = _2022ToGR94DBCS(tmpSourceChar) */
}
tempBuf[0] = static_cast<char>(tmpSourceChar >> 8);
tempBuf[1] = static_cast<char>(tmpSourceChar);
}
targetUniChar = ucnv_MBCSSimpleGetNextUChar(myData->myConverterArray[cs], tempBuf, 2, false);
} elseif (!(trailIsOk || IS_2022_CONTROL(trailByte))) { /* report a pair of illegal bytes if the second byte is not a DBCS starter */
++mySource; /* add another bit so that the code below writes 2 bytes in case of error */
mySourceChar = 0x10000 | (mySourceChar << 8) | trailByte;
}
} else {
args->converter->toUBytes[0] = static_cast<uint8_t>(mySourceChar);
args->converter->toULength = 1; goto endloop;
}
} /* End of inner switch */ break;
} /* End of outer switch */ if(targetUniChar < (missingCharMarker-1/*0xfffe*/)){ if(args->offsets){
args->offsets[myTarget - args->target] = static_cast<int32_t>(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
}
*(myTarget++) = static_cast<char16_t>(targetUniChar);
} elseif(targetUniChar > missingCharMarker){ /* disassemble the surrogate pair and write to output*/
targetUniChar-=0x0010000;
*myTarget = static_cast<char16_t>(0xd800 + static_cast<char16_t>(targetUniChar >> 10)); if(args->offsets){
args->offsets[myTarget - args->target] = static_cast<int32_t>(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
}
++myTarget; if(myTarget< args->targetLimit){
*myTarget = static_cast<char16_t>(0xdc00 + static_cast<char16_t>(targetUniChar & 0x3ff)); if(args->offsets){
args->offsets[myTarget - args->target] = static_cast<int32_t>(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
}
++myTarget;
}else{
args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++]=
static_cast<char16_t>(0xdc00 + static_cast<char16_t>(targetUniChar & 0x3ff));
}
} else{ /* Call the callback function*/
toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err); break;
}
} else{ /* goes with "if(myTarget < args->targetLimit)" way up near top of function */
*err =U_BUFFER_OVERFLOW_ERROR; break;
}
}
endloop:
args->target = myTarget;
args->source = mySource;
}
converterData = static_cast<UConverterDataISO2022*>(args->converter->extraInfo); /* if the version is 1 then the user is requesting *conversionwithibm-25546passtheargumentsto *MBCSconverterandreturn
*/ if(converterData->version==1){
UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(args,err); return;
}
} else{ /* oops.. the code point is unassingned *settheerrorandreason
*/
/*check if the char is a First surrogate*/ if(U16_IS_SURROGATE(sourceChar)) { if(U16_IS_SURROGATE_LEAD(sourceChar)) {
getTrail: /*look ahead to find the trail surrogate*/ if(source < sourceLimit) { /* test the following code unit */
char16_t trail = *source; if(U16_IS_TRAIL(trail)) {
source++;
sourceChar=U16_GET_SUPPLEMENTARY(sourceChar, trail);
*err = U_INVALID_CHAR_FOUND; /* convert this surrogate code point */ /* exit this condition tree */
} else { /* this is an unmatched lead code unit (1st surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
}
} else { /* no more input */
*err = U_ZERO_ERROR;
}
} else { /* this is an unmatched trail code unit (2nd surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
}
} else { /* callback(unassigned) for a BMP code point */
*err = U_INVALID_CHAR_FOUND;
}
/* set up the state */
converterData = static_cast<UConverterDataISO2022*>(cnv->extraInfo);
pFromU2022State = &converterData->fromU2022State;
choiceCount = 0;
/* check if the last codepoint of previous buffer was a lead surrogate*/ if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) { goto getTrail;
}
sourceChar = *(source++); /*check if the char is a First surrogate*/ if(U16_IS_SURROGATE(sourceChar)) { if(U16_IS_SURROGATE_LEAD(sourceChar)) {
getTrail: /*look ahead to find the trail surrogate*/ if(source < sourceLimit) { /* test the following code unit */
char16_t trail = *source; if(U16_IS_TRAIL(trail)) {
source++;
sourceChar=U16_GET_SUPPLEMENTARY(sourceChar, trail);
cnv->fromUChar32=0x00; /* convert this supplementary code point */ /* exit this condition tree */
} else { /* this is an unmatched lead code unit (1st surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
} else { /* no more input */
cnv->fromUChar32=sourceChar; break;
}
} else { /* this is an unmatched trail code unit (2nd surrogate) */ /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
}
/* do the conversion */ if(sourceChar <= 0x007f ){ /* do not convert SO/SI/ESC */ if(IS_2022_CONTROL(sourceChar)) { /* callback(illegal) */
*err=U_ILLEGAL_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
/* US-ASCII */ if(pFromU2022State->g == 0) {
buffer[0] = static_cast<char>(sourceChar);
len = 1;
} else {
buffer[0] = UCNV_SI;
buffer[1] = static_cast<char>(sourceChar);
len = 2;
pFromU2022State->g = 0;
choiceCount = 0;
} if(sourceChar == CR || sourceChar == LF) { /* reset the state at the end of a line */
uprv_memset(pFromU2022State, 0, sizeof(ISO2022State));
choiceCount = 0;
}
} else{ /* convert U+0080..U+10ffff */
int32_t i;
int8_t cs, g;
if(choiceCount == 0) { /* try the current SO/G1 converter first */
choices[0] = pFromU2022State->cs[1];
/* default to GB2312_1 if none is designated yet */ if(choices[0] == 0) {
choices[0] = GB2312_1;
}
/* try the other SO/G1 converter; a CNS_11643_1 lookup may result in any plane */ if(choices[0] == GB2312_1) {
choices[1] = static_cast<int8_t>(CNS_11643_1);
} else {
choices[1] = static_cast<int8_t>(GB2312_1);
}
/* write the shift sequence if necessary */ if(g != pFromU2022State->g) { switch(g) { case1:
buffer[len++] = UCNV_SO;
/* set the new state only if it is the locking shift SO/G1, not for SS2 or SS3 */
pFromU2022State->g = 1; break; case2:
buffer[len++] = 0x1b;
buffer[len++] = 0x4e; break; default: /* case 3 */
buffer[len++] = 0x1b;
buffer[len++] = 0x4f; break;
}
}
/* write the two output bytes */
buffer[len++] = static_cast<char>(targetValue >> 8);
buffer[len++] = static_cast<char>(targetValue);
} else { /* if we cannot find the character after checking all codepages *thenthisisanerror
*/
*err = U_INVALID_CHAR_FOUND;
cnv->fromUChar32=sourceChar; break;
}
}
switch(mySourceChar){ case UCNV_SI:
pToU2022State->g=0; if (myData->isEmptySegment) {
myData->isEmptySegment = false; /* we are handling it, reset to avoid future spurious errors */
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
args->converter->toUCallbackReason = UCNV_IRREGULAR;
args->converter->toUBytes[0] = static_cast<uint8_t>(mySourceChar);
args->converter->toULength = 1;
args->target = myTarget;
args->source = mySource; return;
} continue;
case UCNV_SO: if(pToU2022State->cs[1] != 0) {
pToU2022State->g=1;
myData->isEmptySegment = true; /* Begin a new segment, empty so far */ continue;
} else { /* illegal to have SO before a matching designator */
myData->isEmptySegment = false; /* Handling a different error, reset this to avoid future spurious errs */ break;
}
/* After SO there must be at least one character before a designator (designator error handled separately) */ if(myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
*err = U_ILLEGAL_ESCAPE_SEQUENCE;
args->converter->toUCallbackReason = UCNV_IRREGULAR;
args->converter->toULength = static_cast<int8_t>(toULengthBefore + (mySource - mySourceBefore));
}
}
/* set our substitution string into the subconverter */
myConverterData->currentConverter->subChars = reinterpret_cast<uint8_t*>(subchar);
myConverterData->currentConverter->subCharLen = static_cast<int8_t>(length);
/* let the subconverter write the subchar, set/retrieve fromUChar32 state */
args->converter = myConverterData->currentConverter;
myConverterData->currentConverter->fromUChar32 = cnv->fromUChar32;
ucnv_cbFromUWriteSub(args, 0, err);
cnv->fromUChar32 = myConverterData->currentConverter->fromUChar32;
args->converter = cnv;
/* ucnv.c/ucnv_safeClone() copied the main UConverter already */
uprv_memcpy(&localClone->mydata, cnvData, sizeof(UConverterDataISO2022));
localClone->cnv.extraInfo = &localClone->mydata; /* set pointer to extra data */
localClone->cnv.isExtraLocal = true;
if (U_FAILURE(*pErrorCode)) { return;
} #ifdef U_ENABLE_GENERIC_ISO_2022 if (cnv->sharedData == &_ISO2022Data) { /* We use UTF-8 in this case */
sa->addRange(sa->set, 0, 0xd7FF);
sa->addRange(sa->set, 0xE000, 0x10FFFF); return;
} #endif
/* open a set and initialize it with code points that are algorithmically round-tripped */ switch(cnvData->locale[0]){ case'j': /* include JIS X 0201 which is hardcoded */
sa->add(sa->set, 0xa5);
sa->add(sa->set, 0x203e); if(jpCharsetMasks[cnvData->version]&CSM(ISO8859_1)) { /* include Latin-1 for some variants of JP */
sa->addRange(sa->set, 0, 0xff);
} else { /* include ASCII for JP */
sa->addRange(sa->set, 0, 0x7f);
} if(cnvData->version==3 || cnvData->version==4 || which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) { /* *Donottest(jpCharsetMasks[cnvData->version]&CSM(HWKANA_7BIT))!=0 *becausethebitisonforallJPversionsalthoughonlyversions3&4(JIS7&JIS8) *usehalf-widthKatakana. *ThisisbecauseallISO-2022-JPvariantsarelenientinthattheyaccept(intoUnicode) *half-widthKatakanaviatheESC(Isequence. *However,weonlyemit(fromUnicode)half-widthKatakanaaccordingtothe *definitionofeachvariant. * *Whenincludingfallbacks, *weneedtoincludehalf-widthKatakanaUnicodecodepointsforallJPvariantsbecause *JISX0208hashardcodedfallbacksforthem(whichmaptofull-widthKatakana).
*/ /* include half-width Katakana for JP */
sa->addRange(sa->set, HWKANA_START, HWKANA_END);
} break; #if !UCONFIG_ONLY_HTML_CONVERSION case'c': case'z': /* include ASCII for CN */
sa->addRange(sa->set, 0, 0x7f); break; case'k': /* there is only one converter for KR, and it is not in the myConverterArray[] */
cnvData->currentConverter->sharedData->impl->getUnicodeSet(
cnvData->currentConverter, sa, which, pErrorCode); /* the loop over myConverterArray[] will simply not find another converter */ break; #endif default: break;
}
#if0/* Replaced by ucnv_MBCSGetFilteredUnicodeSetForUnicode() until we implement ucnv_getUnicodeSet() with reverse fallbacks. */ if( (cnvData->locale[0]=='c' || cnvData->locale[0]=='z') &&
cnvData->version==0 && i==CNS_11643
) { /* special handling for non-EXT ISO-2022-CN: add only code points for CNS planes 1 and 2 */
ucnv_MBCSGetUnicodeSetForBytes(
cnvData->myConverterArray[i],
sa, UCNV_ROUNDTRIP_SET, 0, 0x81, 0x82,
pErrorCode);
} #endif
¤ Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.0.183Bemerkung:
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.