//

// Copyright (c) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE file in the project root for full license information.

//

/*++

Module Name:

unicode/utf8.c

Abstract:

Functions to encode and decode UTF-8 strings

Revision History:

--*/

#include "pal/utf8.h"

#include "pal/dbgmsg.h"

#include "pal/unicode_data.h"

//

// Constant Declarations.

//

#define ASCII 0x007f

#define UTF8_2_MAX 0x07ff // max UTF8 2-byte sequence (32 * 64 = 2048)

#define UTF8_1ST_OF_2 0xc0 // 110x xxxx

#define UTF8_1ST_OF_3 0xe0 // 1110 xxxx

#define UTF8_1ST_OF_4 0xf0 // 1111 xxxx

#define UTF8_TRAIL 0x80 // 10xx xxxx

#define HIGHER_6_BIT(u) ((u) >> 12)

#define MIDDLE_6_BIT(u) (((u) & 0x0fc0) >> 6)

#define LOWER_6_BIT(u) ((u) & 0x003f)

#define BIT7(a) ((a) & 0x80)

#define BIT6(a) ((a) & 0x40)

#define HIGH_SURROGATE_START 0xd800

#define HIGH_SURROGATE_END 0xdbff

#define LOW_SURROGATE_START 0xdc00

#define LOW_SURROGATE_END 0xdfff

////////////////////////////////////////////////////////////////////////////

//

// UTF8ToUnicode

//

// Maps a UTF-8 character string to its wide character string counterpart.

//

////////////////////////////////////////////////////////////////////////////

int UTF8ToUnicode(

LPCSTR lpSrcStr,

int cchSrc,

LPWSTR lpDestStr,

int cchDest,

DWORD dwFlags

)

{

int nTB = 0; // # trail bytes to follow

int cchWC = 0; // # of Unicode code points generated

CONST BYTE* pUTF8 = (CONST BYTE*)lpSrcStr;

DWORD dwUnicodeChar = 0; // Our character with room for full surrogate char

BOOL bSurrogatePair = FALSE; // Indicate we're collecting a surrogate pair

BOOL bCheckInvalidBytes = (dwFlags & MB_ERR_INVALID_CHARS);

BYTE UTF8;

// Note that we can't test destination buffer length here because we may have to

// iterate through thousands of broken characters which won't be output, even though

// the buffer has no more room.

while (cchSrc--)

{

//

// See if there are any trail bytes.

//

if (BIT7(*pUTF8) == 0)

{

//

// Found ASCII.

//

if (cchDest)

{

// In this function always test buffer size before using it

if (cchWC >= cchDest)

{

// Error: Buffer too small, we didn't process this character

SetLastError(ERROR_INSUFFICIENT_BUFFER);

return (0);

}

lpDestStr[cchWC] = (WCHAR)*pUTF8;

}

nTB = bSurrogatePair = 0;

cchWC++;

}

else if (BIT6(*pUTF8) == 0)

{

//

// Found a trail byte.

// Note : Ignore the trail byte if there was no lead byte.

//

if (nTB != 0)

{

//

// Decrement the trail byte counter.

//

nTB--;

// Add room for trail byte and add the trail byte falue

dwUnicodeChar <<= 6;

dwUnicodeChar |= LOWER_6_BIT(*pUTF8);

// If we're done then we may need to store the data

if (nTB == 0)

{

if (bSurrogatePair)

{

if (cchDest)

{

if ((cchWC + 1) >= cchDest)

{

// Error: Buffer too small, we didn't process this character

SetLastError(ERROR_INSUFFICIENT_BUFFER);

return (0);

}

lpDestStr[cchWC] = (WCHAR)

(((dwUnicodeChar - 0x10000) >> 10) + HIGH_SURROGATE_START);

lpDestStr[cchWC+1] = (WCHAR)

((dwUnicodeChar - 0x10000)%0x400 + LOW_SURROGATE_START);

}

//

// End of sequence. Advance the output counter, turn off surrogateness

//

cchWC += 2;

bSurrogatePair = FALSE;

}

else

{

if (cchDest)

{

if (cchWC >= cchDest)

{

// Error: Buffer too small, we didn't process this character

SetLastError(ERROR_INSUFFICIENT_BUFFER);

return (0);

}

lpDestStr[cchWC] = (WCHAR)dwUnicodeChar;

}

//

// End of sequence. Advance the output counter.

//

cchWC++;

}

}

}

else

{

if (bCheckInvalidBytes)

{

SetLastError(ERROR_NO_UNICODE_TRANSLATION);

return (0);

}

// error - not expecting a trail byte. That is, there is a trailing byte without leading byte.

bSurrogatePair = FALSE;

}

}

else

{

//

// Found a lead byte.

//

if (nTB > 0)

{

// error - A leading byte before the previous sequence is completed.

if (bCheckInvalidBytes)

{

SetLastError(ERROR_NO_UNICODE_TRANSLATION);

return (0);

}

//

// Error - previous sequence not finished.

//

nTB = 0;

bSurrogatePair = FALSE;

// Put this character back so that we can start over another sequence.

cchSrc++;

pUTF8--;

}

else

{

//

// Calculate the number of bytes to follow.

// Look for the first 0 from left to right.

//

UTF8 = *pUTF8;

while (BIT7(UTF8) != 0)

{

UTF8 <<= 1;

nTB++;

}

// Recover the data from the byte

UTF8 >>= nTB;

//

// Check for non-shortest form.

//

switch (nTB)

{

case 1:

nTB = 0;

break;

case 2:

// Make sure that bit 8 ~ bit 11 is not all zero.

// 110XXXXx 10xxxxxx

if ((*pUTF8 & 0x1e) == 0)

{

nTB = 0;

}

break;

case 3:

// Look ahead to check for non-shortest form.

// 1110XXXX 10Xxxxxx 10xxxxxx

if (cchSrc >= 2)

{

if (((*pUTF8 & 0x0f) == 0) && (*(pUTF8 + 1) & 0x20) == 0)

{

nTB = 0;

}

}

break;

case 4:

//

// This is a surrogate unicode pair

//

if (cchSrc >= 3)

{

WORD word = (((WORD)*pUTF8) << 8) | *(pUTF8 + 1);

// Look ahead to check for non-shortest form.

// 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx

// Check if the 5 X bits are all zero.

// 0x0730 == 00000111 00110000

if ( (word & 0x0730) == 0 ||

// If the 21st bit is 1, we have extra work

( (word & 0x0400) == 0x0400 &&

// The 21st bit is 1.

// Make sure that the resulting Unicode is within the valid surrogate range.

// The 4 byte code sequence can hold up to 21 bits, and the maximum valid code point range

// that Unicode (with surrogate) could represent are from U+000000 ~ U+10FFFF.

// Therefore, if the 21 bit (the most significant bit) is 1, we should verify that the 17 ~ 20

// bit are all zero.

// I.e., in 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx,

// XXXXX can only be 10000.

// 0x0330 = 0000 0011 0011 0000

(word & 0x0330) != 0 ) )

{

// Not shortest form

nTB = 0;

}

else

{

// A real surrogate pair

bSurrogatePair = TRUE;

}

}

break;

default:

//

// If the bits is greater than 4, this is an invalid

// UTF8 lead byte.

//

nTB = 0;

break;

}

if (nTB != 0)

{

//

// Store the value from the first byte and decrement

// the number of bytes to follow.

//

dwUnicodeChar = UTF8;

nTB--;

} else

{

if (bCheckInvalidBytes)

{

SetLastError(ERROR_NO_UNICODE_TRANSLATION);

return (0);

}

}

}

}

pUTF8++;

}

if ((bCheckInvalidBytes && nTB != 0) || (cchWC == 0))

{

// About (cchWC == 0):

// Because we now throw away non-shortest form, it is possible that we generate 0 chars.

// In this case, we have to set error to ERROR_NO_UNICODE_TRANSLATION so that we conform

// to the spec of MultiByteToWideChar.

SetLastError(ERROR_NO_UNICODE_TRANSLATION);

return (0);

}

//

// Return the number of Unicode characters written.

//

return (cchWC);

}

////////////////////////////////////////////////////////////////////////////

//

// UnicodeToUTF8

//

// Maps a Unicode character string to its UTF-8 string counterpart.

//

////////////////////////////////////////////////////////////////////////////

int UnicodeToUTF8(

LPCWSTR lpSrcStr,

int cchSrc,

LPSTR lpDestStr,

int cchDest)

{

LPCWSTR lpWC = lpSrcStr;

int cchU8 = 0; // # of UTF8 chars generated

DWORD dwSurrogateChar;

WCHAR wchHighSurrogate = 0;

BOOL bHandled;

while ((cchSrc--) && ((cchDest == 0) || (cchU8 < cchDest)))

{

bHandled = FALSE;

//

// Check if high surrogate is available

//

if ((*lpWC >= HIGH_SURROGATE_START) && (*lpWC <= HIGH_SURROGATE_END))

{

if (cchDest)

{

// Another high surrogate, then treat the 1st as normal

// Unicode character.

if (wchHighSurrogate)

{

if ((cchU8 + 2) < cchDest)

{

lpDestStr[cchU8++] = UTF8_1ST_OF_3 | HIGHER_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | MIDDLE_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | LOWER_6_BIT(wchHighSurrogate);

}

else

{

// not enough buffer

cchSrc++;

break;

}

}

}

else

{

cchU8 += 3;

}

wchHighSurrogate = *lpWC;

bHandled = TRUE;

}

if (!bHandled && wchHighSurrogate)

{

if ((*lpWC >= LOW_SURROGATE_START) && (*lpWC <= LOW_SURROGATE_END))

{

// wheee, valid surrogate pairs

if (cchDest)

{

if ((cchU8 + 3) < cchDest)

{

dwSurrogateChar = (((wchHighSurrogate-0xD800) << 10) + (*lpWC - 0xDC00) + 0x10000);

lpDestStr[cchU8++] = (UTF8_1ST_OF_4 |

(unsigned char)(dwSurrogateChar >> 18)); // 3 bits from 1st byte

lpDestStr[cchU8++] = (UTF8_TRAIL |

(unsigned char)((dwSurrogateChar >> 12) & 0x3f)); // 6 bits from 2nd byte

lpDestStr[cchU8++] = (UTF8_TRAIL |

(unsigned char)((dwSurrogateChar >> 6) & 0x3f)); // 6 bits from 3rd byte

lpDestStr[cchU8++] = (UTF8_TRAIL |

(unsigned char)(0x3f & dwSurrogateChar)); // 6 bits from 4th byte

}

else

{

// not enough buffer

cchSrc++;

break;

}

}

else

{

// we already counted 3 previously (in high surrogate)

cchU8 ++;

}

bHandled = TRUE;

}

else

{

// Bad Surrogate pair : ERROR

// Just process wchHighSurrogate , and the code below will

// process the current code point

if (cchDest)

{

if ((cchU8 + 2) < cchDest)

{

lpDestStr[cchU8++] = UTF8_1ST_OF_3 | HIGHER_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | MIDDLE_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | LOWER_6_BIT(wchHighSurrogate);

}

else

{

// not enough buffer

cchSrc++;

break;

}

}

}

wchHighSurrogate = 0;

}

if (!bHandled)

{

if (*lpWC <= ASCII)

{

//

// Found ASCII.

//

if (cchDest)

{

if (cchU8 < cchDest)

{

lpDestStr[cchU8] = (char)*lpWC;

}

else

{

//

// Error - buffer too small.

//

cchSrc++;

break;

}

}

cchU8++;

}

else if (*lpWC <= UTF8_2_MAX)

{

//

// Found 2 byte sequence if < 0x07ff (11 bits).

//

if (cchDest)

{

if ((cchU8 + 1) < cchDest)

{

//

// Use upper 5 bits in first byte.

// Use lower 6 bits in second byte.

//

lpDestStr[cchU8++] = UTF8_1ST_OF_2 | (*lpWC >> 6);

lpDestStr[cchU8++] = UTF8_TRAIL | LOWER_6_BIT(*lpWC);

}

else

{

//

// Error - buffer too small.

//

cchSrc++;

break;

}

}

else

{

cchU8 += 2;

}

}

else

{

//

// Found 3 byte sequence.

//

if (cchDest)

{

if ((cchU8 + 2) < cchDest)

{

//

// Use upper 4 bits in first byte.

// Use middle 6 bits in second byte.

// Use lower 6 bits in third byte.

//

lpDestStr[cchU8++] = UTF8_1ST_OF_3 | HIGHER_6_BIT(*lpWC);

lpDestStr[cchU8++] = UTF8_TRAIL | MIDDLE_6_BIT(*lpWC);

lpDestStr[cchU8++] = UTF8_TRAIL | LOWER_6_BIT(*lpWC);

}

else

{

//

// Error - buffer too small.

//

cchSrc++;

break;

}

}

else

{

cchU8 += 3;

}

}

}

lpWC++;

}

//

// If the last character was a high surrogate, then handle it as a normal

// unicode character.

//

if ((cchSrc < 0) && (wchHighSurrogate != 0))

{

if (cchDest)

{

if ((cchU8 + 2) < cchDest)

{

lpDestStr[cchU8++] = UTF8_1ST_OF_3 | HIGHER_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | MIDDLE_6_BIT(wchHighSurrogate);

lpDestStr[cchU8++] = UTF8_TRAIL | LOWER_6_BIT(wchHighSurrogate);

}

else

{

cchSrc++;

}

}

}

//

// Make sure the destination buffer was large enough.

//

if (cchDest && (cchSrc >= 0))

{

SetLastError(ERROR_INSUFFICIENT_BUFFER);

return (0);

}

//

// Return the number of UTF-8 characters written.

//

return (cchU8);

}