/*

* Copyright (C) 2020 Apple Inc. All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

*

* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY

* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR

* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#include "config.h"

#include "TextCodecCJK.h"

#include "EncodingTables.h"

#include <mutex>

#include <wtf/text/CodePointIterator.h>

#include <wtf/text/StringBuilder.h>

#include <wtf/unicode/CharacterNames.h>

namespace WebCore {

enum class TextCodecCJK::Encoding : uint8_t {

EUC_JP,

ISO2022JP,

Shift_JIS,

EUC_KR,

Big5,

GBK,

GB18030

};

TextCodecCJK::TextCodecCJK(Encoding encoding)

: m_encoding(encoding)

{

checkEncodingTableInvariants();

}

void TextCodecCJK::registerEncodingNames(EncodingNameRegistrar registrar)

{

// https://encoding.spec.whatwg.org/#names-and-labels

auto registerAliases = [&] (std::initializer_list<const char*> list) {

for (auto* alias : list)

registrar(alias, *list.begin());

};

registerAliases({

"Big5",

"big5-hkscs",

"cn-big5",

"csbig5",

"x-x-big5"

});

registerAliases({

"EUC-JP",

"cseucpkdfmtjapanese",

"x-euc-jp"

});

registerAliases({

"Shift_JIS",

"csshiftjis",

"ms932",

"ms_kanji",

"shift-jis",

"sjis",

"windows-31j",

"x-sjis"

});

registerAliases({

"EUC-KR",

"cseuckr",

"csksc56011987",

"iso-ir-149",

"korean",

"ks_c_5601-1987",

"ks_c_5601-1989",

"ksc5601",

"ksc_5601",

"windows-949",

// These aliases are not in the specification, but WebKit has historically supported them.

"x-windows-949",

"x-uhc",

});

registerAliases({

"ISO-2022-JP",

"csiso2022jp"

});

registerAliases({

"GBK",

"chinese",

"csgb2312",

"csiso58gb231280",

"gb2312",

"gb_2312",

"gb_2312-80",

"iso-ir-58",

"x-gbk",

// These aliases are not in the specification, but WebKit has historically supported them.

"cn-gb",

"csgb231280",

"x-euc-cn",

"euc-cn",

"cp936",

"ms936",

"gb2312-1980",

"windows-936",

"windows-936-2000"

});

registerAliases({

"gb18030",

// These aliases are not in the specification, but WebKit has historically supported them.

"ibm-1392",

"windows-54936"

});

}

void TextCodecCJK::registerCodecs(TextCodecRegistrar registrar)

{

registrar("EUC-JP", [] {

return makeUnique<TextCodecCJK>(Encoding::EUC_JP);

});

registrar("Big5", [] {

return makeUnique<TextCodecCJK>(Encoding::Big5);

});

registrar("Shift_JIS", [] {

return makeUnique<TextCodecCJK>(Encoding::Shift_JIS);

});

registrar("EUC-KR", [] {

return makeUnique<TextCodecCJK>(Encoding::EUC_KR);

});

registrar("ISO-2022-JP", [] {

return makeUnique<TextCodecCJK>(Encoding::ISO2022JP);

});

registrar("GBK", [] {

return makeUnique<TextCodecCJK>(Encoding::GBK);

});

registrar("gb18030", [] {

return makeUnique<TextCodecCJK>(Encoding::GB18030);

});

}

using JIS0208EncodeIndex = std::array<std::pair<UChar, uint16_t>, sizeof(jis0208()) / sizeof(jis0208()[0])>;

static const JIS0208EncodeIndex& jis0208EncodeIndex()

{

// Allocate this at runtime because building it at compile time would make the binary much larger and this is often not used.

static JIS0208EncodeIndex* table;

static std::once_flag once;

std::call_once(once, [&] {

table = new JIS0208EncodeIndex;

auto& index = jis0208();

for (size_t i = 0; i < index.size(); i++)

(*table)[i] = { index[i].second, index[i].first };

stableSortByFirst(*table);

});

return *table;

}

String TextCodecCJK::decodeCommon(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError, const Function<SawError(uint8_t, StringBuilder&)>& byteParser)

{

StringBuilder result;

result.reserveCapacity(length);

if (m_prependedByte && byteParser(*std::exchange(m_prependedByte, std::nullopt), result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

for (size_t i = 0; i < length; i++) {

if (byteParser(bytes[i], result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

if (m_prependedByte && byteParser(*std::exchange(m_prependedByte, std::nullopt), result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

}

if (flush && m_lead) {

m_lead = 0x00;

sawError = true;

result.append(replacementCharacter);

}

return result.toString();

}

static std::optional<UChar> codePointJIS0208(uint16_t pointer)

{

return findFirstInSortedPairs(jis0208(), pointer);

}

static std::optional<UChar> codePointJIS0212(uint16_t pointer)

{

return findFirstInSortedPairs(jis0212(), pointer);

}

// https://encoding.spec.whatwg.org/#euc-jp-decoder

String TextCodecCJK::eucJPDecode(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)

{

return decodeCommon(bytes, length, flush, stopOnError, sawError, [this] (uint8_t byte, StringBuilder& result) {

if (uint8_t lead = std::exchange(m_lead, 0x00)) {

if (lead == 0x8E && byte >= 0xA1 && byte <= 0xDF) {

result.appendCharacter(0xFF61 - 0xA1 + byte);

return SawError::No;

}

if (lead == 0x8F && byte >= 0xA1 && byte <= 0xFE) {

m_jis0212 = true;

m_lead = byte;

return SawError::No;

}

if (lead >= 0xA1 && lead <= 0xFE && byte >= 0xA1 && byte <= 0xFE) {

uint16_t pointer = (lead - 0xA1) * 94 + byte - 0xA1;

if (auto codePoint = std::exchange(m_jis0212, false) ? codePointJIS0212(pointer) : codePointJIS0208(pointer)) {

result.append(*codePoint);

return SawError::No;

}

}

if (isASCII(byte))

m_prependedByte = byte;

return SawError::Yes;

}

if (isASCII(byte)) {

result.append(static_cast<char>(byte));

return SawError::No;

}

if (byte == 0x8E || byte == 0x8F || (byte >= 0xA1 && byte <= 0xFE)) {

m_lead = byte;

return SawError::No;

}

return SawError::Yes;

});

}

// https://encoding.spec.whatwg.org/#euc-jp-encoder

static Vector<uint8_t> eucJPEncode(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler)

{

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator) {

auto codePoint = *iterator;

if (isASCII(codePoint)) {

result.append(codePoint);

continue;

}

if (codePoint == 0x00A5) {

result.append(0x5C);

continue;

}

if (codePoint == 0x203E) {

result.append(0x7E);

continue;

}

if (codePoint >= 0xFF61 && codePoint <= 0xFF9F) {

result.append(0x8E);

result.append(codePoint - 0xFF61 + 0xA1);

continue;

}

if (codePoint == 0x2212)

codePoint = 0xFF0D;

auto pointer = findFirstInSortedPairs(jis0208EncodeIndex(), codePoint);

if (!pointer) {

unencodableHandler(codePoint, result);

continue;

}

result.append(*pointer / 94 + 0xA1);

result.append(*pointer % 94 + 0xA1);

}

return result;

}

// https://encoding.spec.whatwg.org/#iso-2022-jp-decoder

String TextCodecCJK::iso2022JPDecode(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)

{

auto byteParser = [&] (uint8_t byte, StringBuilder& result) {

switch (m_iso2022JPDecoderState) {

case ISO2022JPDecoderState::ASCII:

if (byte == 0x1B) {

m_iso2022JPDecoderState = ISO2022JPDecoderState::EscapeStart;

break;

}

if (byte <= 0x7F && byte != 0x0E && byte != 0x0F && byte != 0x1B) {

m_iso2022JPOutput = false;

result.append(byte);

break;

}

m_iso2022JPOutput = false;

return SawError::Yes;

case ISO2022JPDecoderState::Roman:

if (byte == 0x1B) {

m_iso2022JPDecoderState = ISO2022JPDecoderState::EscapeStart;

break;

}

if (byte == 0x5C) {

m_iso2022JPOutput = false;

result.append(static_cast<UChar>(0x00A5));

break;

}

if (byte == 0x7E) {

m_iso2022JPOutput = false;

result.append(static_cast<UChar>(0x203E));

break;

}

if (byte <= 0x7F && byte != 0x0E && byte != 0x0F && byte != 0x1B && byte != 0x5C && byte != 0x7E) {

m_iso2022JPOutput = false;

result.append(byte);

break;

}

m_iso2022JPOutput = false;

return SawError::Yes;

case ISO2022JPDecoderState::Katakana:

if (byte == 0x1B) {

m_iso2022JPDecoderState = ISO2022JPDecoderState::EscapeStart;

break;

}

if (byte >= 0x21 && byte <= 0x5F) {

m_iso2022JPOutput = false;

result.append(static_cast<UChar>(0xFF61 - 0x21 + byte));

break;

}

m_iso2022JPOutput = false;

return SawError::Yes;

case ISO2022JPDecoderState::LeadByte:

if (byte == 0x1B) {

m_iso2022JPDecoderState = ISO2022JPDecoderState::EscapeStart;

break;

}

if (byte >= 0x21 && byte <= 0x7E) {

m_iso2022JPOutput = false;

m_lead = byte;

m_iso2022JPDecoderState = ISO2022JPDecoderState::TrailByte;

break;

}

m_iso2022JPOutput = false;

return SawError::Yes;

case ISO2022JPDecoderState::TrailByte:

if (byte == 0x1B) {

m_iso2022JPDecoderState = ISO2022JPDecoderState::EscapeStart;

return SawError::Yes;

}

m_iso2022JPDecoderState = ISO2022JPDecoderState::LeadByte;

if (byte >= 0x21 && byte <= 0x7E) {

uint16_t pointer = (m_lead - 0x21) * 94 + byte - 0x21;

if (auto codePoint = codePointJIS0208(pointer)) {

result.append(*codePoint);

break;

}

return SawError::Yes;

}

return SawError::Yes;

case ISO2022JPDecoderState::EscapeStart:

if (byte == 0x24 || byte == 0x28) {

m_lead = byte;

m_iso2022JPDecoderState = ISO2022JPDecoderState::Escape;

break;

}

m_prependedByte = byte;

m_iso2022JPOutput = false;

m_iso2022JPDecoderState = m_iso2022JPDecoderOutputState;

return SawError::Yes;

case ISO2022JPDecoderState::Escape: {

uint8_t lead = std::exchange(m_lead, 0x00);

std::optional<ISO2022JPDecoderState> state;

if (lead == 0x28) {

if (byte == 0x42)

state = ISO2022JPDecoderState::ASCII;

else if (byte == 0x4A)

state = ISO2022JPDecoderState::Roman;

else if (byte == 0x49)

state = ISO2022JPDecoderState::Katakana;

} else if (lead == 0x24 && (byte == 0x40 || byte == 0x42))

state = ISO2022JPDecoderState::LeadByte;

if (state) {

m_iso2022JPDecoderState = *state;

m_iso2022JPDecoderOutputState = *state;

if (std::exchange(m_iso2022JPOutput, true))

return SawError::Yes;

break;

}

m_prependedByte = lead;

m_iso2022JPSecondPrependedByte = byte;

m_iso2022JPOutput = false;

m_iso2022JPDecoderState = m_iso2022JPDecoderOutputState;

return SawError::Yes;

}

}

return SawError::No;

};

StringBuilder result;

result.reserveCapacity(length);

if (m_prependedByte && byteParser(*std::exchange(m_prependedByte, std::nullopt), result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

if (m_iso2022JPSecondPrependedByte && byteParser(*std::exchange(m_iso2022JPSecondPrependedByte, std::nullopt), result) == SawError::Yes && stopOnError) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

for (size_t i = 0; i < length; i++) {

if (byteParser(bytes[i], result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

if (m_prependedByte && byteParser(*std::exchange(m_prependedByte, std::nullopt), result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

if (m_iso2022JPSecondPrependedByte && byteParser(*std::exchange(m_iso2022JPSecondPrependedByte, std::nullopt), result) == SawError::Yes && stopOnError) {

sawError = true;

result.append(replacementCharacter);

if (stopOnError) {

m_lead = 0x00;

return result.toString();

}

}

}

if (flush) {

switch (m_iso2022JPDecoderState) {

case ISO2022JPDecoderState::ASCII:

case ISO2022JPDecoderState::Roman:

case ISO2022JPDecoderState::Katakana:

case ISO2022JPDecoderState::LeadByte:

break;

case ISO2022JPDecoderState::TrailByte:

m_iso2022JPDecoderState = ISO2022JPDecoderState::LeadByte;

FALLTHROUGH;

case ISO2022JPDecoderState::EscapeStart:

sawError = true;

result.append(replacementCharacter);

break;

case ISO2022JPDecoderState::Escape:

sawError = true;

result.append(replacementCharacter);

if (m_lead) {

ASSERT(isASCII(m_lead));

result.append(std::exchange(m_lead, 0x00));

}

break;

}

}

return result.toString();

}

// https://encoding.spec.whatwg.org/#iso-2022-jp-encoder

static Vector<uint8_t> iso2022JPEncode(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler)

{

enum class State : uint8_t { ASCII, Roman, Jis0208 };

State state { State::ASCII };

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto changeStateToASCII = [&] {

state = State::ASCII;

result.append(0x1B);

result.append(0x28);

result.append(0x42);

};

auto statefulUnencodableHandler = [&] (UChar32 codePoint, Vector<uint8_t>& result) {

if (state == State::Jis0208)

changeStateToASCII();

unencodableHandler(codePoint, result);

};

Function<void(UChar32)> parseCodePoint;

parseCodePoint = [&] (UChar32 codePoint) {

if ((state == State::ASCII || state == State::Roman) && (codePoint == 0x000E || codePoint == 0x000F || codePoint == 0x001B)) {

statefulUnencodableHandler(replacementCharacter, result);

return;

}

if (state == State::ASCII && isASCII(codePoint)) {

result.append(codePoint);

return;

}

if (state == State::Roman) {

if (isASCII(codePoint) && codePoint != 0x005C && codePoint !=0x007E) {

result.append(codePoint);

return;

}

if (codePoint == 0x00A5) {

result.append(0x5C);

return;

}

if (codePoint == 0x203E) {

result.append(0x7E);

return;

}

}

if (isASCII(codePoint) && state != State::ASCII) {

if (state != State::ASCII)

changeStateToASCII();

parseCodePoint(codePoint);

return;

}

if ((codePoint == 0x00A5 || codePoint == 0x203E) && state != State::Roman) {

state = State::Roman;

result.append(0x1B);

result.append(0x28);

result.append(0x4A);

parseCodePoint(codePoint);

return;

}

if (codePoint == 0x2212)

codePoint = 0xFF0D;

if (codePoint >= 0xFF61 && codePoint <= 0xFF9F) {

// From https://encoding.spec.whatwg.org/index-iso-2022-jp-katakana.txt

static constexpr std::array<UChar32, 63> iso2022JPKatakana {

0x3002, 0x300C, 0x300D, 0x3001, 0x30FB, 0x30F2, 0x30A1, 0x30A3, 0x30A5, 0x30A7, 0x30A9, 0x30E3, 0x30E5, 0x30E7, 0x30C3, 0x30FC,

0x30A2, 0x30A4, 0x30A6, 0x30A8, 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3, 0x30B5, 0x30B7, 0x30B9, 0x30BB, 0x30BD, 0x30BF,

0x30C1, 0x30C4, 0x30C6, 0x30C8, 0x30CA, 0x30CB, 0x30CC, 0x30CD, 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, 0x30DE, 0x30DF,

0x30E0, 0x30E1, 0x30E2, 0x30E4, 0x30E6, 0x30E8, 0x30E9, 0x30EA, 0x30EB, 0x30EC, 0x30ED, 0x30EF, 0x30F3, 0x309B, 0x309C

};

static_assert(std::size(iso2022JPKatakana) == 0xFF9F - 0xFF61 + 1);

codePoint = iso2022JPKatakana[codePoint - 0xFF61];

}

auto pointer = findFirstInSortedPairs(jis0208EncodeIndex(), codePoint);

if (!pointer) {

statefulUnencodableHandler(codePoint, result);

return;

}

if (state != State::Jis0208) {

state = State::Jis0208;

result.append(0x1B);

result.append(0x24);

result.append(0x42);

parseCodePoint(codePoint);

return;

}

result.append(*pointer / 94 + 0x21);

result.append(*pointer % 94 + 0x21);

};

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator)

parseCodePoint(*iterator);

if (state != State::ASCII)

changeStateToASCII();

return result;

}

// https://encoding.spec.whatwg.org/#shift_jis-decoder

String TextCodecCJK::shiftJISDecode(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)

{

return decodeCommon(bytes, length, flush, stopOnError, sawError, [this] (uint8_t byte, StringBuilder& result) {

if (uint8_t lead = std::exchange(m_lead, 0x00)) {

uint8_t offset = byte < 0x7F ? 0x40 : 0x41;

uint8_t leadOffset = lead < 0xA0 ? 0x81 : 0xC1;

if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFC)) {

uint16_t pointer = (lead - leadOffset) * 188 + byte - offset;

if (pointer >= 8836 && pointer <= 10715) {

result.append(static_cast<UChar>(0xE000 - 8836 + pointer));

return SawError::No;

}

if (auto codePoint = codePointJIS0208(pointer)) {

result.append(*codePoint);

return SawError::No;

}

}

if (isASCII(byte))

m_prependedByte = byte;

return SawError::Yes;

}

if (isASCII(byte) || byte == 0x80) {

result.append(byte);

return SawError::No;

}

if (byte >= 0xA1 && byte <= 0xDF) {

result.append(static_cast<UChar>(0xFF61 - 0xA1 + byte));

return SawError::No;

}

if ((byte >= 0x81 && byte <= 0x9F) || (byte >= 0xE0 && byte <= 0xFC)) {

m_lead = byte;

return SawError::No;

}

return SawError::Yes;

});

}

// https://encoding.spec.whatwg.org/#shift_jis-encoder

static Vector<uint8_t> shiftJISEncode(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler)

{

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator) {

auto codePoint = *iterator;

if (isASCII(codePoint) || codePoint == 0x0080) {

result.append(codePoint);

continue;

}

if (codePoint == 0x00A5) {

result.append(0x5C);

continue;

}

if (codePoint == 0x203E) {

result.append(0x7E);

continue;

}

if (codePoint >= 0xFF61 && codePoint <= 0xFF9F) {

result.append(codePoint - 0xFF61 + 0xA1);

continue;

}

if (codePoint == 0x2212)

codePoint = 0xFF0D;

auto range = findInSortedPairs(jis0208EncodeIndex(), codePoint);

if (range.first == range.second) {

unencodableHandler(codePoint, result);

continue;

}

ASSERT(range.first + 3 >= range.second);

for (auto pair = range.first; pair < range.second; pair++) {

uint16_t pointer = pair->second;

if (pointer >= 8272 && pointer <= 8835)

continue;

uint8_t lead = pointer / 188;

uint8_t leadOffset = lead < 0x1F ? 0x81 : 0xC1;

uint8_t trail = pointer % 188;

uint8_t offset = trail < 0x3F ? 0x40 : 0x41;

result.append(lead + leadOffset);

result.append(trail + offset);

break;

}

}

return result;

}

using EUCKREncodingIndex = std::array<std::pair<UChar, uint16_t>, sizeof(eucKR()) / sizeof(eucKR()[0])>;

static const EUCKREncodingIndex& eucKREncodingIndex()

{

// Allocate this at runtime because building it at compile time would make the binary much larger and this is often not used.

static EUCKREncodingIndex* table;

static std::once_flag once;

std::call_once(once, [&] {

table = new EUCKREncodingIndex;

auto& index = eucKR();

for (size_t i = 0; i < index.size(); i++)

(*table)[i] = { index[i].second, index[i].first };

sortByFirst(*table);

ASSERT(sortedFirstsAreUnique(*table));

});

return *table;

}

// https://encoding.spec.whatwg.org/#euc-kr-encoder

static Vector<uint8_t> eucKREncode(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler)

{

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator) {

auto codePoint = *iterator;

if (isASCII(codePoint)) {

result.append(codePoint);

continue;

}

auto pointer = findFirstInSortedPairs(eucKREncodingIndex(), codePoint);

if (!pointer) {

unencodableHandler(codePoint, result);

continue;

}

result.append(*pointer / 190 + 0x81);

result.append(*pointer % 190 + 0x41);

}

return result;

}

// https://encoding.spec.whatwg.org/#euc-kr-decoder

String TextCodecCJK::eucKRDecode(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)

{

return decodeCommon(bytes, length, flush, stopOnError, sawError, [this] (uint8_t byte, StringBuilder& result) {

if (uint8_t lead = std::exchange(m_lead, 0x00)) {

if (byte >= 0x41 && byte <= 0xFE) {

if (auto codePoint = findFirstInSortedPairs(eucKR(), (lead - 0x81) * 190 + byte - 0x41)) {

result.append(*codePoint);

return SawError::No;

}

}

if (isASCII(byte))

m_prependedByte = byte;

return SawError::Yes;

}

if (isASCII(byte)) {

result.append(byte);

return SawError::No;

}

if (byte >= 0x81 && byte <= 0xFE) {

m_lead = byte;

return SawError::No;

}

return SawError::Yes;

});

}

using Big5EncodeIndex = std::array<std::pair<UChar32, uint16_t>, sizeof(big5()) / sizeof(big5()[0]) - 3904>;

static const Big5EncodeIndex& big5EncodeIndex()

{

// Allocate this at runtime because building it at compile time would make the binary much larger and this is often not used.

static Big5EncodeIndex* table;

static std::once_flag once;

std::call_once(once, [&] {

table = new Big5EncodeIndex;

auto& index = big5();

// Remove the first 3094 elements because of https://encoding.spec.whatwg.org/#index-big5-pointer

ASSERT(index[3903].first == (0xA1 - 0x81) * 157 - 1);

ASSERT(index[3904].first == (0xA1 - 0x81) * 157);

for (size_t i = 3904; i < index.size(); i++)

(*table)[i - 3904] = { index[i].second, index[i].first };

stableSortByFirst(*table);

});

return *table;

}

// https://encoding.spec.whatwg.org/#big5-encoder

static Vector<uint8_t> big5Encode(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler)

{

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator) {

auto codePoint = *iterator;

if (isASCII(codePoint)) {

result.append(codePoint);

continue;

}

auto pointerRange = findInSortedPairs(big5EncodeIndex(), codePoint);

if (pointerRange.first == pointerRange.second) {

unencodableHandler(codePoint, result);

continue;

}

uint16_t pointer = 0;

if (codePoint == 0x2550 || codePoint == 0x255E || codePoint == 0x2561 || codePoint == 0x256A || codePoint == 0x5341 || codePoint == 0x5345)

pointer = (pointerRange.second - 1)->second;

else

pointer = pointerRange.first->second;

if (pointer < 157 * (0xA1 - 0x81)) {

unencodableHandler(codePoint, result);

continue;

}

uint8_t lead = pointer / 157 + 0x81;

uint8_t trail = pointer % 157;

uint8_t offset = trail < 0x3F ? 0x40 : 0x62;

result.append(lead);

result.append(trail + offset);

}

return result;

}

// https://encoding.spec.whatwg.org/index-gb18030-ranges.txt

static const std::array<std::pair<uint32_t, UChar32>, 207>& gb18030Ranges()

{

static std::array<std::pair<uint32_t, UChar32>, 207> ranges {{

{ 0, 0x0080 }, { 36, 0x00A5 }, { 38, 0x00A9 }, { 45, 0x00B2 }, { 50, 0x00B8 }, { 81, 0x00D8 }, { 89, 0x00E2 }, { 95, 0x00EB },

{ 96, 0x00EE }, { 100, 0x00F4 }, { 103, 0x00F8 }, { 104, 0x00FB }, { 105, 0x00FD }, { 109, 0x0102 }, { 126, 0x0114 }, { 133, 0x011C },

{ 148, 0x012C }, { 172, 0x0145 }, { 175, 0x0149 }, { 179, 0x014E }, { 208, 0x016C }, { 306, 0x01CF }, { 307, 0x01D1 }, { 308, 0x01D3 },

{ 309, 0x01D5 }, { 310, 0x01D7 }, { 311, 0x01D9 }, { 312, 0x01DB }, { 313, 0x01DD }, { 341, 0x01FA }, { 428, 0x0252 }, { 443, 0x0262 },

{ 544, 0x02C8 }, { 545, 0x02CC }, { 558, 0x02DA }, { 741, 0x03A2 }, { 742, 0x03AA }, { 749, 0x03C2 }, { 750, 0x03CA }, { 805, 0x0402 },

{ 819, 0x0450 }, { 820, 0x0452 }, { 7922, 0x2011 }, { 7924, 0x2017 }, { 7925, 0x201A }, { 7927, 0x201E }, { 7934, 0x2027 }, { 7943, 0x2031 },

{ 7944, 0x2034 }, { 7945, 0x2036 }, { 7950, 0x203C }, { 8062, 0x20AD }, { 8148, 0x2104 }, { 8149, 0x2106 }, { 8152, 0x210A }, { 8164, 0x2117 },

{ 8174, 0x2122 }, { 8236, 0x216C }, { 8240, 0x217A }, { 8262, 0x2194 }, { 8264, 0x219A }, { 8374, 0x2209 }, { 8380, 0x2210 }, { 8381, 0x2212 },

{ 8384, 0x2216 }, { 8388, 0x221B }, { 8390, 0x2221 }, { 8392, 0x2224 }, { 8393, 0x2226 }, { 8394, 0x222C }, { 8396, 0x222F }, { 8401, 0x2238 },

{ 8406, 0x223E }, { 8416, 0x2249 }, { 8419, 0x224D }, { 8424, 0x2253 }, { 8437, 0x2262 }, { 8439, 0x2268 }, { 8445, 0x2270 }, { 8482, 0x2296 },

{ 8485, 0x229A }, { 8496, 0x22A6 }, { 8521, 0x22C0 }, { 8603, 0x2313 }, { 8936, 0x246A }, { 8946, 0x249C }, { 9046, 0x254C }, { 9050, 0x2574 },

{ 9063, 0x2590 }, { 9066, 0x2596 }, { 9076, 0x25A2 }, { 9092, 0x25B4 }, { 9100, 0x25BE }, { 9108, 0x25C8 }, { 9111, 0x25CC }, { 9113, 0x25D0 },

{ 9131, 0x25E6 }, { 9162, 0x2607 }, { 9164, 0x260A }, { 9218, 0x2641 }, { 9219, 0x2643 }, { 11329, 0x2E82 }, { 11331, 0x2E85 }, { 11334, 0x2E89 },

{ 11336, 0x2E8D }, { 11346, 0x2E98 }, { 11361, 0x2EA8 }, { 11363, 0x2EAB }, { 11366, 0x2EAF }, { 11370, 0x2EB4 }, { 11372, 0x2EB8 }, { 11375, 0x2EBC },

{ 11389, 0x2ECB }, { 11682, 0x2FFC }, { 11686, 0x3004 }, { 11687, 0x3018 }, { 11692, 0x301F }, { 11694, 0x302A }, { 11714, 0x303F }, { 11716, 0x3094 },

{ 11723, 0x309F }, { 11725, 0x30F7 }, { 11730, 0x30FF }, { 11736, 0x312A }, { 11982, 0x322A }, { 11989, 0x3232 }, { 12102, 0x32A4 }, { 12336, 0x3390 },

{ 12348, 0x339F }, { 12350, 0x33A2 }, { 12384, 0x33C5 }, { 12393, 0x33CF }, { 12395, 0x33D3 }, { 12397, 0x33D6 }, { 12510, 0x3448 }, { 12553, 0x3474 },

{ 12851, 0x359F }, { 12962, 0x360F }, { 12973, 0x361B }, { 13738, 0x3919 }, { 13823, 0x396F }, { 13919, 0x39D1 }, { 13933, 0x39E0 }, { 14080, 0x3A74 },

{ 14298, 0x3B4F }, { 14585, 0x3C6F }, { 14698, 0x3CE1 }, { 15583, 0x4057 }, { 15847, 0x4160 }, { 16318, 0x4338 }, { 16434, 0x43AD }, { 16438, 0x43B2 },

{ 16481, 0x43DE }, { 16729, 0x44D7 }, { 17102, 0x464D }, { 17122, 0x4662 }, { 17315, 0x4724 }, { 17320, 0x472A }, { 17402, 0x477D }, { 17418, 0x478E },

{ 17859, 0x4948 }, { 17909, 0x497B }, { 17911, 0x497E }, { 17915, 0x4984 }, { 17916, 0x4987 }, { 17936, 0x499C }, { 17939, 0x49A0 }, { 17961, 0x49B8 },

{ 18664, 0x4C78 }, { 18703, 0x4CA4 }, { 18814, 0x4D1A }, { 18962, 0x4DAF }, { 19043, 0x9FA6 }, { 33469, 0xE76C }, { 33470, 0xE7C8 }, { 33471, 0xE7E7 },

{ 33484, 0xE815 }, { 33485, 0xE819 }, { 33490, 0xE81F }, { 33497, 0xE827 }, { 33501, 0xE82D }, { 33505, 0xE833 }, { 33513, 0xE83C }, { 33520, 0xE844 },

{ 33536, 0xE856 }, { 33550, 0xE865 }, { 37845, 0xF92D }, { 37921, 0xF97A }, { 37948, 0xF996 }, { 38029, 0xF9E8 }, { 38038, 0xF9F2 }, { 38064, 0xFA10 },

{ 38065, 0xFA12 }, { 38066, 0xFA15 }, { 38069, 0xFA19 }, { 38075, 0xFA22 }, { 38076, 0xFA25 }, { 38078, 0xFA2A }, { 39108, 0xFE32 }, { 39109, 0xFE45 },

{ 39113, 0xFE53 }, { 39114, 0xFE58 }, { 39115, 0xFE67 }, { 39116, 0xFE6C }, { 39265, 0xFF5F }, { 39394, 0xFFE6 }, { 189000, 0x10000 }

}};

return ranges;

}

// https://encoding.spec.whatwg.org/#index-gb18030-ranges-code-point

static std::optional<UChar32> gb18030RangesCodePoint(uint32_t pointer)

{

if ((pointer > 39419 && pointer < 189000) || pointer > 1237575)

return std::nullopt;

if (pointer == 7457)

return 0xE7C7;

auto upperBound = std::upper_bound(gb18030Ranges().begin(), gb18030Ranges().end(), makeFirstAdapter(pointer), CompareFirst { });

ASSERT(upperBound != gb18030Ranges().begin());

uint32_t offset = (upperBound - 1)->first;

UChar32 codePointOffset = (upperBound - 1)->second;

return codePointOffset + pointer - offset;

}

// https://encoding.spec.whatwg.org/#index-gb18030-ranges-pointer

static uint32_t gb18030RangesPointer(UChar32 codePoint)

{

if (codePoint == 0xE7C7)

return 7457;

auto upperBound = std::upper_bound(gb18030Ranges().begin(), gb18030Ranges().end(), makeSecondAdapter(codePoint), CompareSecond { });

ASSERT(upperBound != gb18030Ranges().begin());

uint32_t pointerOffset = (upperBound - 1)->first;

UChar32 offset = (upperBound - 1)->second;

return pointerOffset + codePoint - offset;

}

using GB18030EncodeIndex = std::array<std::pair<UChar, uint16_t>, 23940>;

static const GB18030EncodeIndex& gb18030EncodeIndex()

{

// Allocate this at runtime because building it at compile time would make the binary much larger and this is often not used.

static GB18030EncodeIndex* table;

static std::once_flag once;

std::call_once(once, [&] {

table = new GB18030EncodeIndex;

auto& index = gb18030();

for (uint16_t i = 0; i < index.size(); i++)

(*table)[i] = { index[i], i };

stableSortByFirst(*table);

});

return *table;

}

// https://encoding.spec.whatwg.org/#gb18030-decoder

String TextCodecCJK::gb18030Decode(const uint8_t* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)

{

Function<SawError(uint8_t, StringBuilder&)> parseByte;

parseByte = [&] (uint8_t byte, StringBuilder& result) {

if (m_gb18030Third) {

if (byte < 0x30 || byte > 0x39) {

sawError = true;

result.append(replacementCharacter);

m_gb18030First = 0x00;

uint8_t second = std::exchange(m_gb18030Second, 0x00);

uint8_t third = std::exchange(m_gb18030Third, 0x00);

if (parseByte(second, result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

}

if (parseByte(third, result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

}

return parseByte(byte, result);

}

uint8_t first = std::exchange(m_gb18030First, 0x00);

uint8_t second = std::exchange(m_gb18030Second, 0x00);

uint8_t third = std::exchange(m_gb18030Third, 0x00);

if (auto codePoint = gb18030RangesCodePoint(((first - 0x81) * 10 * 126 * 10) + ((second - 0x30) * 10 * 126) + ((third - 0x81) * 10) + byte - 0x30)) {

result.appendCharacter(*codePoint);

return SawError::No;

}

return SawError::Yes;

}

if (m_gb18030Second) {

if (byte >= 0x81 && byte <= 0xFE) {

m_gb18030Third = byte;

return SawError::No;

}

sawError = true;

result.append(replacementCharacter);

m_gb18030First = 0x00;

if (parseByte(std::exchange(m_gb18030Second, 0x00), result) == SawError::Yes) {

sawError = true;

result.append(replacementCharacter);

}

return parseByte(byte, result);

}

if (m_gb18030First) {

if (byte >= 0x30 && byte <= 0x39) {

m_gb18030Second = byte;

return SawError::No;

}

uint8_t lead = std::exchange(m_gb18030First, 0x00);

uint8_t offset = byte < 0x7F ? 0x40 : 0x41;

if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFE)) {

size_t pointer = (lead - 0x81) * 190 + byte - offset;

if (pointer < gb18030().size()) {

result.append(gb18030()[pointer]);

return SawError::No;

}

}

if (isASCII(byte))

m_prependedByte = byte;

return SawError::Yes;

}

if (isASCII(byte)) {

result.append(byte);

return SawError::No;

}

if (byte == 0x80) {

result.appendCharacter(0x20AC);

return SawError::No;

}

if (byte >= 0x81 && byte <= 0xFE) {

m_gb18030First = byte;

return SawError::No;

}

return SawError::Yes;

};

auto result = decodeCommon(bytes, length, flush, stopOnError, sawError, parseByte);

if (flush && (m_gb18030First || m_gb18030Second || m_gb18030Third)) {

m_gb18030First = 0x00;

m_gb18030Second = 0x00;

m_gb18030Third = 0x00;

sawError = true;

result = makeString(result, replacementCharacter);

}

return result;

}

// https://encoding.spec.whatwg.org/#gb18030-encoder

enum class IsGBK : bool { No, Yes };

static Vector<uint8_t> gbEncodeShared(StringView string, Function<void(UChar32, Vector<uint8_t>&)>&& unencodableHandler, IsGBK isGBK)

{

Vector<uint8_t> result;

result.reserveInitialCapacity(string.length());

auto characters = string.upconvertedCharacters();

for (WTF::CodePointIterator<UChar> iterator(characters.get(), characters.get() + string.length()); !iterator.atEnd(); ++iterator) {

auto codePoint = *iterator;

if (isASCII(codePoint)) {

result.append(codePoint);

continue;

}

if (codePoint == 0xE5E5) {

unencodableHandler(codePoint, result);

continue;