/*

* Copyright (c) Facebook, Inc. and its affiliates.

*

* This source code is licensed under the MIT license found in the

* LICENSE file in the root directory of this source tree.

*/

#include "hermes/Parser/JSLexer.h"

#include "dtoa/dtoa.h"

#include "hermes/Support/Conversions.h"

#include "llvh/ADT/StringSwitch.h"

using llvh::Twine;

namespace hermes {

namespace parser {

namespace {

const char *g_tokenStr[] = {

#define TOK(name, str) str,

#include "hermes/Parser/TokenKinds.def"

};

const int UTF8_LINE_TERMINATOR_CHAR0 = 0xe2;

inline bool matchUnicodeLineTerminatorOffset1(const char *curCharPtr_) {

// Line separator \u2028 UTF8 encoded is : e2 80 a8

// Paragraph separator \u2029 UTF8 encoded is: e2 80 a9

return (unsigned char)curCharPtr_[1] == 0x80 &&

((unsigned char)curCharPtr_[2] == 0xa8 ||

(unsigned char)curCharPtr_[2] == 0xa9);

}

} // namespace

const char *tokenKindStr(TokenKind kind) {

assert(kind <= TokenKind::_last_token);

return g_tokenStr[static_cast<unsigned>(kind)];

}

JSLexer::JSLexer(

uint32_t bufId,

SourceErrorManager &sm,

Allocator &allocator,

StringTable *strTab,

bool strictMode,

bool convertSurrogates)

: sm_(sm),

allocator_(allocator),

ownStrTab_(strTab ? nullptr : new StringTable(allocator_)),

strTab_(strTab ? *strTab : *ownStrTab_),

strictMode_(strictMode),

convertSurrogates_(convertSurrogates) {

initializeWithBufferId(bufId);

initializeReservedIdentifiers();

}

JSLexer::JSLexer(

std::unique_ptr<llvh::MemoryBuffer> input,

SourceErrorManager &sm,

Allocator &allocator,

StringTable *strTab,

bool strictMode,

bool convertSurrogates)

: sm_(sm),

allocator_(allocator),

ownStrTab_(strTab ? nullptr : new StringTable(allocator_)),

strTab_(strTab ? *strTab : *ownStrTab_),

strictMode_(strictMode),

convertSurrogates_(convertSurrogates) {

auto bufId = sm_.addNewSourceBuffer(std::move(input));

initializeWithBufferId(bufId);

initializeReservedIdentifiers();

}

void JSLexer::initializeWithBufferId(uint32_t bufId) {

auto *buffer = sm_.getSourceBuffer(bufId);

bufId_ = bufId;

bufferStart_ = buffer->getBufferStart();

bufferEnd_ = buffer->getBufferEnd();

curCharPtr_ = bufferStart_;

assert(*bufferEnd_ == 0 && "buffer must be zero terminated");

}

void JSLexer::initializeReservedIdentifiers() {

// Add all reserved words to the identifier table

#define RESWORD(name) resWordIdent(TokenKind::rw_##name) = getIdentifier(#name);

#include "hermes/Parser/TokenKinds.def"

}

const Token *JSLexer::advance(GrammarContext grammarContext) {

newLineBeforeCurrentToken_ = false;

for (;;) {

assert(curCharPtr_ <= bufferEnd_ && "lexing past end of input");

#define PUNC_L1_1(ch, tok) \

case ch: \

token_.setStart(curCharPtr_); \

token_.setPunctuator(tok); \

++curCharPtr_; \

break

#define PUNC_L2_3(ch1, tok1, ch2a, tok2a, ch2b, tok2b) \

case ch1: \

token_.setStart(curCharPtr_); \

if (curCharPtr_[1] == ch2a) { \

token_.setPunctuator(tok2a); \

curCharPtr_ += 2; \

} else if (curCharPtr_[1] == ch2b) { \

token_.setPunctuator(tok2b); \

curCharPtr_ += 2; \

} else { \

token_.setPunctuator(tok1); \

curCharPtr_ += 1; \

} \

break

#define PUNC_L2_2(ch1, tok1, ch2, tok2) \

case ch1: \

token_.setStart(curCharPtr_); \

if (curCharPtr_[1] == (ch2)) { \

token_.setPunctuator(tok2); \

curCharPtr_ += 2; \

} else { \

token_.setPunctuator(tok1); \

curCharPtr_ += 1; \

} \

break

#define PUNC_L3_3(ch1, tok1, ch2, tok2, ch3, tok3) \

case ch1: \

token_.setStart(curCharPtr_); \

if (curCharPtr_[1] != (ch2)) { \

token_.setPunctuator(tok1); \

curCharPtr_ += 1; \

} else if (curCharPtr_[2] == (ch3)) { \

token_.setPunctuator(tok3); \

curCharPtr_ += 3; \

} else { \

token_.setPunctuator(tok2); \

curCharPtr_ += 2; \

} \

break

switch ((unsigned char)*curCharPtr_) {

case 0:

token_.setStart(curCharPtr_);

if (curCharPtr_ == bufferEnd_) {

token_.setEof();

} else {

if (!error(

token_.getStartLoc(),

"unrecognized Unicode character \\u0000")) {

token_.setEof();

} else {

++curCharPtr_;

continue;

}

}

break;

// clang-format off

PUNC_L1_1('}', TokenKind::r_brace);

PUNC_L1_1('(', TokenKind::l_paren);

PUNC_L1_1(')', TokenKind::r_paren);

PUNC_L1_1('[', TokenKind::l_square);

PUNC_L1_1(']', TokenKind::r_square);

PUNC_L1_1(';', TokenKind::semi);

PUNC_L1_1(',', TokenKind::comma);

PUNC_L1_1('~', TokenKind::tilde);

PUNC_L1_1(':', TokenKind::colon);

// { {|

case '{':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == GrammarContext::Flow) &&

curCharPtr_[1] == '|') {

token_.setPunctuator(TokenKind::l_bracepipe);

curCharPtr_ += 2;

} else {

token_.setPunctuator(TokenKind::l_brace);

curCharPtr_ += 1;

}

break;

// = => == ===

case '=':

token_.setStart(curCharPtr_);

if (curCharPtr_[1] == '>') {

token_.setPunctuator(TokenKind::equalgreater);

curCharPtr_ += 2;

} else if (curCharPtr_[1] != '=') {

token_.setPunctuator(TokenKind::equal);

curCharPtr_ += 1;

} else if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::equalequalequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::equalequal);

curCharPtr_ += 2;

}

break;

// ! != !==

PUNC_L3_3('!', TokenKind::exclaim, '=', TokenKind::exclaimequal, '=', TokenKind::exclaimequalequal);

// + ++ +=

// - -- -=

// & && &=

// | || |=

PUNC_L2_3('+', TokenKind::plus, '+', TokenKind::plusplus, '=', TokenKind::plusequal);

PUNC_L2_3('-', TokenKind::minus, '-', TokenKind::minusminus, '=', TokenKind::minusequal);

case '&':

token_.setStart(curCharPtr_);

if (curCharPtr_[1] == '&') {

if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::ampampequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::ampamp);

curCharPtr_ += 2;

}

} else if (curCharPtr_[1] == '=') {

token_.setPunctuator(TokenKind::ampequal);

curCharPtr_ += 2;

} else {

token_.setPunctuator(TokenKind::amp);

curCharPtr_ += 1;

}

break;

case '|':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == GrammarContext::Flow) &&

curCharPtr_[1] == '}') {

token_.setPunctuator(TokenKind::piper_brace);

curCharPtr_ += 2;

} else {

if (curCharPtr_[1] == '|') {

if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::pipepipeequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::pipepipe);

curCharPtr_ += 2;

}

} else if (curCharPtr_[1] == '=') {

token_.setPunctuator(TokenKind::pipeequal);

curCharPtr_ += 2;

} else {

token_.setPunctuator(TokenKind::pipe);

curCharPtr_ += 1;

}

}

break;

// ? ?? ?.

case '?':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == GrammarContext::Flow)) {

token_.setPunctuator(TokenKind::question);

curCharPtr_ += 1;

} else if (curCharPtr_[1] == '.' && !isdigit(curCharPtr_[2])) {

// OptionalChainingPunctuator ::

// ?. [lookahead does not contain DecimalDigit]

// This is done to prevent `x?.3:y` from being recognized

// as `x ?. 3 : y` instead of `x ? .3 : y`.

token_.setPunctuator(TokenKind::questiondot);

curCharPtr_ += 2;

} else if (curCharPtr_[1] == '?') {

if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::questionquestionequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::questionquestion);

curCharPtr_ += 2;

}

} else {

token_.setPunctuator(TokenKind::question);

curCharPtr_ += 1;

}

break;

// * *= ** **=

case '*':

token_.setStart(curCharPtr_);

if (curCharPtr_[1] == '=') {

token_.setPunctuator(TokenKind::starequal);

curCharPtr_ += 2;

} else if (curCharPtr_[1] != '*') {

token_.setPunctuator(TokenKind::star);

curCharPtr_ += 1;

} else if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::starstarequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::starstar);

curCharPtr_ += 2;

}

break;

// * *=

// ^ ^=

// / /=

PUNC_L2_2('^', TokenKind::caret, '=', TokenKind::caretequal);

// % %=

case '%':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == GrammarContext::Flow) &&

curCharPtr_ + 7 <= bufferEnd_ &&

llvh::StringRef(curCharPtr_, 7) == "%checks") {

token_.setIdentifier(getStringLiteral("%checks"));

curCharPtr_ += 7;

} else if (curCharPtr_[1] == ('=')) {

token_.setPunctuator(TokenKind::percentequal);

curCharPtr_ += 2;

} else {

token_.setPunctuator(TokenKind::percent);

curCharPtr_ += 1;

}

break;

// clang-format on

case '\r':

case '\n':

++curCharPtr_;

newLineBeforeCurrentToken_ = true;

continue;

// Line separator \u2028 UTF8 encoded is : e2 80 a8

// Paragraph separator \u2029 UTF8 encoded is : e2 80 a9

case UTF8_LINE_TERMINATOR_CHAR0:

if (matchUnicodeLineTerminatorOffset1(curCharPtr_)) {

curCharPtr_ += 3;

newLineBeforeCurrentToken_ = true;

continue;

} else {

goto default_label;

}

case '\v':

case '\f':

++curCharPtr_;

continue;

case '\t':

case ' ':

// Spaces frequently come in groups, so use a tight inner loop to skip.

do

++curCharPtr_;

while (*curCharPtr_ == '\t' || *curCharPtr_ == ' ');

continue;

// No-break space \u00A0 is UTF8 encoded as: c2 a0

case 0xc2:

if ((unsigned char)curCharPtr_[1] == 0xa0) {

curCharPtr_ += 2;

continue;

} else {

goto default_label;

}

// Byte-order mark \uFEFF is encoded as: ef bb bf

case 0xef:

if ((unsigned char)curCharPtr_[1] == 0xbb &&

(unsigned char)curCharPtr_[2] == 0xbf) {

curCharPtr_ += 3;

continue;

} else {

goto default_label;

}

case '/':

if (curCharPtr_[1] == '/') { // Line comment?

if (LLVM_UNLIKELY(curCharPtr_[2] == '#')) {

if (auto sourceMappingUrl =

tryReadMagicComment("sourceMappingURL", curCharPtr_)) {

sm_.setSourceMappingUrl(bufId_, sourceMappingUrl.getValue());

} else if (

auto sourceUrl =

tryReadMagicComment("sourceURL", curCharPtr_)) {

sm_.setSourceUrl(bufId_, sourceUrl.getValue());

}

}

curCharPtr_ = skipLineComment(curCharPtr_);

continue;

} else if (curCharPtr_[1] == '*') { // Block comment?

curCharPtr_ = skipBlockComment(curCharPtr_);

continue;

} else {

token_.setStart(curCharPtr_);

if (grammarContext == AllowRegExp) {

scanRegExp();

} else if (curCharPtr_[1] == '=') {

token_.setPunctuator(TokenKind::slashequal);

curCharPtr_ += 2;

} else {

token_.setPunctuator(TokenKind::slash);

curCharPtr_ += 1;

}

}

break;

// #! (hashbang) at the very start of the buffer.

case '#':

if (LLVM_UNLIKELY(

curCharPtr_ == bufferStart_ && curCharPtr_[1] == '!')) {

curCharPtr_ = skipLineComment(curCharPtr_);

continue;

} else {

goto default_label;

}

// < <= << <<=

case '<':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == JSLexer::GrammarContext::Flow)) {

token_.setPunctuator(TokenKind::less);

curCharPtr_ += 1;

} else if (curCharPtr_[1] == '=') {

token_.setPunctuator(TokenKind::lessequal);

curCharPtr_ += 2;

} else if (curCharPtr_[1] == '<') {

if (curCharPtr_[2] == '=') {

token_.setPunctuator(TokenKind::lesslessequal);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::lessless);

curCharPtr_ += 2;

}

} else {

token_.setPunctuator(TokenKind::less);

curCharPtr_ += 1;

}

break;

// > >= >> >>> >>= >>>=

case '>':

token_.setStart(curCharPtr_);

if ((HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == JSLexer::GrammarContext::Flow)) ||

(HERMES_PARSE_JSX &&

LLVM_UNLIKELY(

grammarContext ==

JSLexer::GrammarContext::AllowJSXIdentifier))) {

token_.setPunctuator(TokenKind::greater);

curCharPtr_ += 1;

} else if (curCharPtr_[1] == '=') { // >=

token_.setPunctuator(TokenKind::greaterequal);

curCharPtr_ += 2;

} else if (curCharPtr_[1] == '>') { // >>

if (curCharPtr_[2] == '=') { // >>=

token_.setPunctuator(TokenKind::greatergreaterequal);

curCharPtr_ += 3;

} else if (curCharPtr_[2] == '>') { // >>>

if (curCharPtr_[3] == '=') { // >>>=

token_.setPunctuator(TokenKind::greatergreatergreaterequal);

curCharPtr_ += 4;

} else {

token_.setPunctuator(TokenKind::greatergreatergreater);

curCharPtr_ += 3;

}

} else {

token_.setPunctuator(TokenKind::greatergreater);

curCharPtr_ += 2;

}

} else {

token_.setPunctuator(TokenKind::greater);

curCharPtr_ += 1;

}

break;

case '.':

token_.setStart(curCharPtr_);

if (curCharPtr_[1] >= '0' && curCharPtr_[1] <= '9') {

scanNumber();

} else if (curCharPtr_[1] == '.' && curCharPtr_[2] == '.') {

token_.setPunctuator(TokenKind::dotdotdot);

curCharPtr_ += 3;

} else {

token_.setPunctuator(TokenKind::period);

++curCharPtr_;

}

break;

// clang-format off

case '0': case '1': case '2': case '3': case '4':

case '5': case '6': case '7': case '8': case '9':

// clang-format on

token_.setStart(curCharPtr_);

scanNumber();

break;

// clang-format off

case '_': case '$':

case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':

case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':

case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':

case 'v': case 'w': case 'x': case 'y': case 'z':

case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':

case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N':

case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':

case 'V': case 'W': case 'X': case 'Y': case 'Z':

// clang-format on

token_.setStart(curCharPtr_);

scanIdentifierFastPathInContext(curCharPtr_, grammarContext);

break;

case '@':

token_.setStart(curCharPtr_);

if (HERMES_PARSE_FLOW &&

LLVM_UNLIKELY(grammarContext == GrammarContext::Flow)) {

scanIdentifierFastPathInContext(curCharPtr_, grammarContext);

} else {

curCharPtr_ += 1;

errorRange(token_.getStartLoc(), "unrecognized character '@'");

continue;

}

break;

case '\\': {

token_.setStart(curCharPtr_);

tmpStorage_.clear();

uint32_t cp = consumeUnicodeEscape();

if (!isUnicodeIdentifierStart(cp)) {

errorRange(

token_.getStartLoc(),

"Unicode escape \\u" + Twine::utohexstr(cp) +

" is not a valid identifier start");

continue;

} else {

appendUnicodeToStorage(cp);

}

scanIdentifierPartsInContext(grammarContext);

break;

}

case '\'':

case '"':

token_.setStart(curCharPtr_);

scanStringInContext(grammarContext);

break;

case '`':

token_.setStart(curCharPtr_);

scanTemplateLiteral();

break;

default_label:

default: {

token_.setStart(curCharPtr_);

uint32_t ch = decodeUTF8();

if (isUnicodeOnlyLetter(ch)) {

tmpStorage_.clear();

appendUnicodeToStorage(ch);

scanIdentifierPartsInContext(grammarContext);

} else if (isUnicodeOnlySpace(ch)) {

continue;

} else {

if (ch > 31 && ch < 127)

errorRange(

token_.getStartLoc(),

"unrecognized character '" + Twine((char)ch) + "'");

else

errorRange(

token_.getStartLoc(),

"unrecognized Unicode character \\u" + Twine::utohexstr(ch));

continue;

}

break;

}

}

// Always terminate the loop unless "continue" was used.

break;

} // for(;;)

token_.setEnd(curCharPtr_);

return &token_;

}

const Token *JSLexer::advanceInJSXChild() {

token_.setStart(curCharPtr_);

for (;;) {

assert(curCharPtr_ <= bufferEnd_ && "lexing past end of input");

switch (*curCharPtr_) {

PUNC_L1_1('{', TokenKind::l_brace);

PUNC_L1_1('<', TokenKind::less);

case 0:

if (curCharPtr_ == bufferEnd_) {

token_.setEof();

break;

}

// Fall-through to start scanning text.

LLVM_FALLTHROUGH;

default: {

token_.setStart(curCharPtr_);

// FIXME: Cook rawStorage_ into a value using XHTML entities.

rawStorage_.clear();

for (;;) {

char c = *curCharPtr_;

if ((c == 0 && curCharPtr_ == bufferEnd_) || c == '{' || c == '<') {

token_.setJSXText(

getStringLiteral(rawStorage_.str()),

getStringLiteral(rawStorage_.str()));

break;

}

rawStorage_.push_back(c);

++curCharPtr_;

}

break;

}

}

// Always terminate the loop unless "continue" was used.

break;

}

token_.setEnd(curCharPtr_);

return &token_;

}

bool JSLexer::isCurrentTokenADirective() {

// The current token must be a string literal without escapes.

if (token_.getKind() != TokenKind::string_literal ||

token_.getStringLiteralContainsEscapes()) {

return false;

}

const char *ptr = curCharPtr_;

// A directive is a string literal (the current token, directly behind

// curCharPtr_), followed by a semicolon, new line, or eof that we will now

// try to find. There can also be comments. So, we loop, consuming whitespace

// until we encounter:

// - EOF. Don't consume it and succeed.

// - Semicolon. Don't consume it and succeed.

// - Right brace. Don't consume it and succeed.

// - A new line. Don't consume it and succeed.

// - A line comment. It implies a new line. Don't consume it and succeed.

// - A block comment. Consume it and continue.

// - Anything else. We consume nothing and fail.

for (;;) {

assert(ptr <= bufferEnd_ && "lexing past end of input");

switch (*((const unsigned char *)ptr)) {

case 0:

// EOF?

if (ptr == bufferEnd_)

return true;

// We encountered a stray 0 character.

return false;

case ';':

case '}':

return true;

case '\r':

case '\n':

return true;

// Line separator \u2028 UTF8 encoded is : e2 80 a8

// Paragraph separator \u2029 UTF8 encoded is : e2 80 a9

case UTF8_LINE_TERMINATOR_CHAR0:

if (matchUnicodeLineTerminatorOffset1(ptr))

return true;

return false;

case '\v':

case '\f':

// Skip whitespace.

++ptr;

continue;

case '\t':

case ' ':

// Spaces frequently come in groups, so use a tight inner loop to skip.

do

++ptr;

while (*ptr == '\t' || *ptr == ' ');

continue;

// No-break space \u00A0 is UTF8 encoded as: c2 a0

case 0xc2:

if ((unsigned char)ptr[1] == 0xa0) {

ptr += 2;

continue;

} else {

goto default_label;

}

// Byte-order mark \uFEFF is encoded as: ef bb bf

case 0xef:

if ((unsigned char)ptr[1] == 0xbb && (unsigned char)ptr[2] == 0xbf) {

ptr += 3;

continue;

} else {

goto default_label;

}

case '/':

if (ptr[1] == '/') { // Line comment?

// It implies a new line, so we are good.

return true;

} else if (ptr[1] == '*') { // Block comment?

SourceErrorManager::SaveAndSuppressMessages suppress(&sm_);

ptr = skipBlockComment(ptr);

continue;

} else {

return false;

}

// Handle all other characters: if it is a unicode space, skip it.

// Otherwise we have failed.

default_label:

default: {

if (hermes::isUTF8Start(*ptr)) {

auto peeked = _peekUTF8(ptr);

if (isUnicodeOnlySpace(peeked.first)) {

ptr = peeked.second;

continue;

}

}

return false;

}

}

}

// We arrive here if we matched a directive. 'ptr' is the final character.

return true;

}

const Token *JSLexer::rescanRBraceInTemplateLiteral() {

assert(token_.getKind() == TokenKind::r_brace && "need } to rescan");

--curCharPtr_;

assert(*curCharPtr_ == '}' && "non-} was scanned as r_brace");

token_.setStart(curCharPtr_);

scanTemplateLiteral();

token_.setEnd(curCharPtr_);

return &token_;

}

OptValue<TokenKind> JSLexer::lookahead1(OptValue<TokenKind> expectedToken) {

assert(

(token_.getKind() == TokenKind::identifier || token_.isResWord()) &&

"unsupported current token");

UniqueString *savedIdent = token_.getResWordOrIdentifier();

TokenKind savedKind = token_.getKind();

SMLoc start = token_.getStartLoc();

SMLoc end = token_.getEndLoc();

const char *cur = curCharPtr_;

SourceErrorManager::SaveAndSuppressMessages suppress(&sm_);

advance();

OptValue<TokenKind> kind = token_.getKind();

if (isNewLineBeforeCurrentToken()) {

// Disregard anything after LineTerminator.

kind = llvh::None;

} else if (expectedToken == kind) {

// Do not move the cursor back.

return kind;

}

token_.setStart(start.getPointer());

token_.setEnd(end.getPointer());

if (savedKind == TokenKind::identifier) {

token_.setIdentifier(savedIdent);

} else {

token_.setResWord(savedKind, savedIdent);

}

seek(SMLoc::getFromPointer(cur));

return kind;

}

uint32_t JSLexer::consumeUnicodeEscape() {

assert(*curCharPtr_ == '\\');

++curCharPtr_;

if (*curCharPtr_ != 'u') {

error(

{SMLoc::getFromPointer(curCharPtr_ - 1),

SMLoc::getFromPointer(curCharPtr_ + 1)},

"invalid Unicode escape");

return UNICODE_REPLACEMENT_CHARACTER;

}

++curCharPtr_;

if (*curCharPtr_ == '{') {

auto cp = consumeBracedCodePoint();

if (!cp.hasValue()) {

// consumeBracedCodePoint has reported an error.

return UNICODE_REPLACEMENT_CHARACTER;

}

return *cp;

}

auto cp = consumeHex(4);

if (!cp)

return UNICODE_REPLACEMENT_CHARACTER;

// We don't need t check for valid UTF-16. JavaScript allows invalid surrogate

// pairs, so we just encode every UTF-16 code into a UTF-8 sequence, even

// though theoretically it is not a valid UTF-8. (UTF-8 would be "valid" if we

// collected the surrogate pair, decoded it into UTF-32 and encoded that into

// UTF-16).

return cp.getValue();

}

llvh::Optional<uint32_t> JSLexer::consumeUnicodeEscapeOptional() {

const char *start = curCharPtr_;

assert(*curCharPtr_ == '\\');

++curCharPtr_;

if (*curCharPtr_ != 'u') {

curCharPtr_ = start;

return llvh::None;

}

++curCharPtr_;

if (*curCharPtr_ == '{') {

// Avoid reporting an error because we are consuming the escape optionally.

auto cp = consumeBracedCodePoint(false);

if (!cp) {

curCharPtr_ = start;

return llvh::None;

}

return *cp;

}

auto cp = consumeHex(4, false);

if (!cp) {

curCharPtr_ = start;

return llvh::None;

}

// We don't need t check for valid UTF-16. JavaScript allows invalid surrogate

// pairs, so we just encode every UTF-16 code into a UTF-8 sequence, even

// though theoretically it is not a valid UTF-8. (UTF-8 would be "valid" if we

// collected the surrogate pair, decoded it into UTF-32 and encoded that into

// UTF-16).

return cp.getValue();

}

bool JSLexer::consumeIdentifierStart() {

if (*curCharPtr_ == '_' || *curCharPtr_ == '$' ||

((*curCharPtr_ | 32) >= 'a' && (*curCharPtr_ | 32) <= 'z')) {

tmpStorage_.clear();

tmpStorage_.push_back(*curCharPtr_++);

return true;

}

if (*curCharPtr_ == '\\') {

SMLoc startLoc = SMLoc::getFromPointer(curCharPtr_);

tmpStorage_.clear();

uint32_t cp = consumeUnicodeEscape();

if (!isUnicodeIdentifierStart(cp)) {

errorRange(

startLoc,

"Unicode escape \\u" + Twine::utohexstr(cp) +

"is not a valid identifier start");

} else {

appendUnicodeToStorage(cp);

}

return true;

}

if (LLVM_LIKELY(!isUTF8Start(*curCharPtr_)))

return false;

auto decoded = _peekUTF8();

if (isUnicodeIdentifierStart(decoded.first)) {

tmpStorage_.clear();

appendUnicodeToStorage(decoded.first);

curCharPtr_ = decoded.second;

return true;

}

return false;

}

template <JSLexer::IdentifierMode Mode>

bool JSLexer::consumeOneIdentifierPartNoEscape() {

char ch = *curCharPtr_;

if (ch == '_' || ch == '$' || ((ch | 32) >= 'a' && (ch | 32) <= 'z') ||

(ch >= '0' && ch <= '9') || (Mode == IdentifierMode::JSX && ch == '-') ||

(Mode == IdentifierMode::Flow && ch == '@')) {

tmpStorage_.push_back(*curCharPtr_++);

return true;

} else if (LLVM_UNLIKELY(isUTF8Start(ch))) {

// If we have encountered a Unicode character, we try to decode it. If it

// can be a part of the identifier, we consume it, otherwise we leave it

// alone.

auto decoded = _peekUTF8();

if (isUnicodeIdentifierPart(decoded.first)) {

appendUnicodeToStorage(decoded.first);

curCharPtr_ = decoded.second;

return true;

}

}

return false;

}

template <JSLexer::IdentifierMode Mode>

void JSLexer::consumeIdentifierParts() {

for (;;) {

// Try consuming an non-escaped identifier part. Failing that, check for an

// escape.

if (consumeOneIdentifierPartNoEscape<Mode>())

continue;

else if (*curCharPtr_ == '\\') {

// Decode the escape.

SMLoc startLoc = SMLoc::getFromPointer(curCharPtr_);

uint32_t cp = consumeUnicodeEscape();

if (!isUnicodeIdentifierPart(cp)) {

errorRange(

startLoc,

"Unicode escape \\u" + Twine::utohexstr(cp) +

"is not a valid identifier codepoint");

} else {

appendUnicodeToStorage(cp);

}

} else

break;

}

}

unsigned char JSLexer::consumeOctal(unsigned maxLen) {

assert(*curCharPtr_ >= '0' && *curCharPtr_ <= '7');

if (strictMode_) {

if (!error(

SMLoc::getFromPointer(curCharPtr_ - 1),

"octals not allowed in strict mode")) {

return 0;

}

}

auto res = (unsigned char)(*curCharPtr_++ - '0');

while (--maxLen && *curCharPtr_ >= '0' && *curCharPtr_ <= '7')

res = (res << 3) + *curCharPtr_++ - '0';

return res;

}

llvh::Optional<uint32_t> JSLexer::consumeHex(

unsigned requiredLen,

bool errorOnFail) {

uint32_t cp = 0;

for (unsigned i = 0; i != requiredLen; ++i) {

unsigned ch = *curCharPtr_;

if (ch >= '0' && ch <= '9') {

ch -= '0';

} else {

// Now that we know it is not a digit, it is safe to lowercase.

ch |= 32;

if (ch >= 'a' && ch <= 'f') {

ch -= 'a' - 10;

} else {

if (errorOnFail) {

error(SMLoc::getFromPointer(curCharPtr_), "invalid hex number");

}

return llvh::None;

}

}

cp = (cp << 4) + ch;

++curCharPtr_;

}

return cp;

}

llvh::Optional<uint32_t> JSLexer::consumeBracedCodePoint(bool errorOnFail) {

assert(*curCharPtr_ == '{' && "braced codepoint must begin with {");

++curCharPtr_;

const char *start = curCharPtr_;

// Set to true if we failed to get a code point that is in bounds or saw

// an invalid character.

bool failed = false;

// Loop until we hit the } or eof, max out the value, or see an invalid char.

uint32_t cp = 0;

for (; *curCharPtr_ != '}'; ++curCharPtr_) {

int ch = *curCharPtr_;

if (ch >= '0' && ch <= '9') {