//-------------------------------------------------------------------------------------------------------

// Copyright (C) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.

//-------------------------------------------------------------------------------------------------------

#include "ParserPch.h"

/*****************************************************************************

*

* The following table speeds various tests of characters, such as whether

* a given character can be part of an identifier, and so on.

*/

int CountNewlines(LPCOLESTR psz, int cch)

{

int cln = 0;

while (0 != *psz && 0 != cch--)

{

switch (*psz++)

{

case _u('\xD'):

if (*psz == _u('\xA'))

{

++psz;

if (0 == cch--)

break;

}

// fall-through

case _u('\xA'):

cln++;

break;

}

}

return cln;

}

template< typename CharT >

struct AorW

{

};

// Specialization for UTF8Char

template<>

struct AorW< UTF8Char >

{

// Expressing the args as "arrays of size N" ensures that the both args

// are the same length. If not, we get a compile time error.

template< size_t N >

static const UTF8Char* Choose( const char (&a)[N], const char16 (&w)[N] )

{

// The reinterpret_cast is necessary to go from signed to unsigned char

return reinterpret_cast< const UTF8Char* >(a);

}

template< size_t N >

static const bool Test(const char (&a)[N], const char16 (&w)[N], LPCUTF8 value)

{

return 0 == memcmp(a, value, (N - 1) * sizeof(utf8char_t));

}

template< size_t N >

static const bool Test(const char (&a)[N], const char16 (&w)[N], LPCUTF8 start, LPCUTF8 end)

{

return (end - start == N - 1) && (0 == memcmp(a, start, (N - 1) * sizeof(utf8char_t)));

}

};

// Specialization for OLECHAR

template<>

struct AorW< OLECHAR >

{

template< size_t N >

static const char16* Choose( const char (&a)[N], const char16 (&w)[N] )

{

return w;

}

template < size_t N >

static bool Test(const char (&a)[N], const char16 (&w)[N], const char16 *value)

{

return 0 == memcmp(w, value, (N - 1) * sizeof(char16));

}

template < size_t N >

static bool Test(const char (&a)[N], const char16 (&w)[N], const char16 *start, const char16 *end)

{

return (end - start == N - 1) && (0 == memcmp(w, start, (N - 1) * sizeof(char16)));

}

};

BOOL Token::IsKeyword() const

{

// keywords (but not future reserved words)

return (tk <= tkYIELD);

}

tokens Token::SetRegex(UnifiedRegex::RegexPattern *const pattern, Parser *const parser)

{

Assert(parser);

if(pattern)

parser->RegisterRegexPattern(pattern);

this->u.pattern = pattern;

return tk = tkRegExp;

}

IdentPtr Token::CreateIdentifier(HashTbl * hashTbl)

{

Assert(this->u.pid == nullptr);

if (this->u.pchMin)

{

Assert(IsIdentifier());

IdentPtr pid = hashTbl->PidHashNameLen(this->u.pchMin, this->u.length);

this->u.pid = pid;

return pid;

}

Assert(IsReservedWord());

IdentPtr pid = hashTbl->PidFromTk(tk);

this->u.pid = pid;

return pid;

}

template <typename EncodingPolicy>

Scanner<EncodingPolicy>::Scanner(Parser* parser, HashTbl *phtbl, Token *ptoken, ErrHandler *perr, Js::ScriptContext* scriptContext)

{

AssertMem(phtbl);

AssertMem(ptoken);

AssertMem(perr);

m_parser = parser;

m_phtbl = phtbl;

m_ptoken = ptoken;

m_cMinLineMultiUnits = 0;

m_perr = perr;

m_fHadEol = FALSE;

m_doubleQuoteOnLastTkStrCon = FALSE;

m_OctOrLeadingZeroOnLastTKNumber = false;

m_fStringTemplateDepth = 0;

m_scanState = ScanStateNormal;

m_scriptContext = scriptContext;

m_line = 0;

m_startLine = 0;

m_pchStartLine = NULL;

m_ichMinError = 0;

m_ichLimError = 0;

m_tempChBuf.m_pscanner = this;

m_tempChBufSecondary.m_pscanner = this;

m_iecpLimTokPrevious = (size_t)-1;

this->charClassifier = scriptContext->GetCharClassifier();

this->es6UnicodeMode = scriptContext->GetConfig()->IsES6UnicodeExtensionsEnabled();

m_fYieldIsKeyword = false;

m_fAwaitIsKeyword = false;

m_typeAnnotationsOn = CONFIG_FLAG(TypeAnnotations);

}

template <typename EncodingPolicy>

Scanner<EncodingPolicy>::~Scanner(void)

{

}

/*****************************************************************************

*

* Initializes the scanner to prepare to scan the given source text.

*/

template <typename EncodingPolicy>

void Scanner<EncodingPolicy>::SetText(EncodedCharPtr pszSrc, size_t offset, size_t length, charcount_t charOffset, ULONG grfscr, ULONG lineNumber)

{

// Save the start of the script and add the offset to get the point where we should start scanning.

m_pchBase = pszSrc;

m_pchLast = m_pchBase + offset + length;

m_pchPrevLine = m_currentCharacter = m_pchMinLine = m_pchMinTok = pszSrc + offset;

this->RestoreMultiUnits(offset - charOffset);

// Absorb any byte order mark at the start

if(offset == 0)

{

switch( this->PeekFull(m_currentCharacter, m_pchLast) )

{

case 0xFFEE: // "Opposite" endian BOM

// We do not support big-endian encodings

// fall-through

case 0xFEFF: // "Correct" BOM

this->template ReadFull<true>(m_currentCharacter, m_pchLast);

break;

}

}

m_line = lineNumber;

m_startLine = lineNumber;

m_pchStartLine = m_currentCharacter;

m_ptoken->tk = tkNone;

m_fIsModuleCode = (grfscr & fscrIsModuleCode) != 0;

m_fHadEol = FALSE;

m_fSyntaxColor = (grfscr & fscrSyntaxColor) != 0;

m_DeferredParseFlags = ScanFlagNone;

}

template <typename EncodingPolicy>

void Scanner<EncodingPolicy>::PrepareForBackgroundParse(Js::ScriptContext *scriptContext)

{

scriptContext->GetThreadContext()->GetStandardChars((EncodedChar*)0);

scriptContext->GetThreadContext()->GetStandardChars((char16*)0);

}

//-----------------------------------------------------------------------------

// Number of code points from 'first' up to, but not including the next

// newline character, embedded NUL, or 'last', depending on which comes first.

//

// This is used to determine a length of BSTR, which can't contain a NUL character.

//-----------------------------------------------------------------------------

template <typename EncodingPolicy>

charcount_t Scanner<EncodingPolicy>::LineLength(EncodedCharPtr first, EncodedCharPtr last)

{

charcount_t result = 0;

EncodedCharPtr p = first;

for (;;)

{

switch( this->template ReadFull<false>(p, last) )

{

case kchNWL: // _C_NWL

case kchRET:

case kchLS:

case kchPS:

case kchNUL: // _C_NUL

return result;

}

result++;

}

}

template <typename EncodingPolicy>

charcount_t Scanner<EncodingPolicy>::UpdateLine(int32 &line, EncodedCharPtr start, EncodedCharPtr last, charcount_t ichStart, charcount_t ichEnd)

{

EncodedCharPtr p = start;

charcount_t ich = ichStart;

int32 current = line;

charcount_t lastStart = ichStart;

while (ich < ichEnd)

{

ich++;

switch (this->template ReadFull<false>(p, last))

{

case kchRET:

if (this->PeekFull(p, last) == kchNWL)

{

ich++;

this->template ReadFull<false>(p, last);

}

// fall-through

case kchNWL:

case kchLS:

case kchPS:

current++;

lastStart = ich;

break;

case kchNUL:

goto done;

}

}

done:

line = current;

return lastStart;

}

template <typename EncodingPolicy>

bool Scanner<EncodingPolicy>::TryReadEscape(EncodedCharPtr& startingLocation, EncodedCharPtr endOfSource, codepoint_t *outChar)

{

Assert(outChar != nullptr);

Assert(startingLocation <= endOfSource);

EncodedCharPtr currentLocation = startingLocation;

codepoint_t charToOutput = 0x0;

// '\' is Assumed as there is only one caller

// Read 'u' characters

if (currentLocation >= endOfSource || this->ReadFirst(currentLocation, endOfSource) != 'u')

{

return false;

}

bool expectCurly = false;

if (currentLocation < endOfSource && this->PeekFirst(currentLocation, endOfSource) == '{' && es6UnicodeMode)

{

expectCurly = true;

// Move past the character

this->ReadFirst(currentLocation, endOfSource);

}

uint i = 0;

OLECHAR ch = 0;

int hexValue = 0;

uint maxHexDigits = (expectCurly ? MAXUINT32 : 4u);

for(; i < maxHexDigits && currentLocation < endOfSource; i++)

{

if (!Js::NumberUtilities::FHexDigit(ch = this->ReadFirst(currentLocation, endOfSource), &hexValue))

{

break;

}

charToOutput = charToOutput * 0x10 + hexValue;

if (charToOutput > 0x10FFFF)

{

return false;

}

}

//At least 4 characters have to be read

if (i == 0 || (i != 4 && !expectCurly))

{

return false;

}

Assert(expectCurly ? es6UnicodeMode : true);

if (expectCurly && ch != '}')

{

return false;

}

*outChar = charToOutput;

startingLocation = currentLocation;

return true;

}

template <typename EncodingPolicy>

template <bool bScan>

bool Scanner<EncodingPolicy>::TryReadCodePointRest(codepoint_t lower, EncodedCharPtr& startingLocation, EncodedCharPtr endOfSource, codepoint_t *outChar, bool *outContainsMultiUnitChar)

{

Assert(outChar != nullptr);

Assert(outContainsMultiUnitChar != nullptr);

Assert(es6UnicodeMode);

Assert(Js::NumberUtilities::IsSurrogateLowerPart(lower));

EncodedCharPtr currentLocation = startingLocation;

*outChar = lower;

if (currentLocation < endOfSource)

{

size_t restorePoint = this->m_cMultiUnits;

codepoint_t upper = this->template ReadFull<bScan>(currentLocation, endOfSource);

if (Js::NumberUtilities::IsSurrogateUpperPart(upper))

{

*outChar = Js::NumberUtilities::SurrogatePairAsCodePoint(lower, upper);

if (this->IsMultiUnitChar(static_cast<OLECHAR>(upper)))

{

*outContainsMultiUnitChar = true;

}

startingLocation = currentLocation;

}

else

{

this->RestoreMultiUnits(restorePoint);

}

}

return true;

}

template <typename EncodingPolicy>

template <bool bScan>

inline bool Scanner<EncodingPolicy>::TryReadCodePoint(EncodedCharPtr &startingLocation, EncodedCharPtr endOfSource, codepoint_t *outChar, bool *hasEscape, bool *outContainsMultiUnitChar)

{

Assert(outChar != nullptr);

Assert(outContainsMultiUnitChar != nullptr);

if (startingLocation >= endOfSource)

{

return false;

}

codepoint_t ch = this->template ReadFull<bScan>(startingLocation, endOfSource);

if (FBigChar(ch))

{

if (this->IsMultiUnitChar(static_cast<OLECHAR>(ch)))

{

*outContainsMultiUnitChar = true;

}

if (es6UnicodeMode && Js::NumberUtilities::IsSurrogateLowerPart(ch))

{

return TryReadCodePointRest<bScan>(ch, startingLocation, endOfSource, outChar, outContainsMultiUnitChar);

}

}

else if (ch == '\\' && TryReadEscape(startingLocation, endOfSource, &ch))

{

*hasEscape = true;

}

*outChar = ch;

return true;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::ScanIdentifier(bool identifyKwds, EncodedCharPtr *pp)

{

EncodedCharPtr p = *pp;

EncodedCharPtr pchMin = p;

// JS6 allows unicode characters in the form of \uxxxx escape sequences

// to be part of the identifier.

bool fHasEscape = false;

bool fHasMultiChar = false;

codepoint_t codePoint = INVALID_CODEPOINT;

size_t multiUnitsBeforeLast = this->m_cMultiUnits;

// Check if we started the id

if (!TryReadCodePoint<true>(p, m_pchLast, &codePoint, &fHasEscape, &fHasMultiChar))

{

// If no chars. could be scanned as part of the identifier, return error.

return tkScanError;

}

Assert(codePoint < 0x110000u);

if (!charClassifier->IsIdStart(codePoint))

{

// Put back the last character

this->RestoreMultiUnits(multiUnitsBeforeLast);

// If no chars. could be scanned as part of the identifier, return error.

return tkScanError;

}

return ScanIdentifierContinue(identifyKwds, fHasEscape, fHasMultiChar, pchMin, p, pp);

}

template <typename EncodingPolicy>

BOOL Scanner<EncodingPolicy>::FastIdentifierContinue(EncodedCharPtr&p, EncodedCharPtr last)

{

if (EncodingPolicy::MultiUnitEncoding)

{

while (p < last)

{

EncodedChar currentChar = *p;

if (this->IsMultiUnitChar(currentChar))

{

// multi unit character, we may not have reach the end yet

return FALSE;

}

Assert(currentChar != '\\' || !charClassifier->IsIdContinueFast<false>(currentChar));

if (!charClassifier->IsIdContinueFast<false>(currentChar))

{

// only reach the end of the identifier if it is not the start of an escape sequence

return currentChar != '\\';

}

p++;

}

// We have reach the end of the identifier.

return TRUE;

}

// Not fast path for non multi unit encoding

return false;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::ScanIdentifierContinue(bool identifyKwds, bool fHasEscape, bool fHasMultiChar,

EncodedCharPtr pchMin, EncodedCharPtr p, EncodedCharPtr *pp)

{

EncodedCharPtr last = m_pchLast;

while (true)

{

// Fast path for utf8, non-multi unit char and not escape

if (FastIdentifierContinue(p, last))

{

break;

}

// Slow path that has to deal with multi unit encoding

codepoint_t codePoint = INVALID_CODEPOINT;

EncodedCharPtr pchBeforeLast = p;

size_t multiUnitsBeforeLast = this->m_cMultiUnits;

if (TryReadCodePoint<true>(p, last, &codePoint, &fHasEscape, &fHasMultiChar))

{

Assert(codePoint < 0x110000u);

if (charClassifier->IsIdContinue(codePoint))

{

continue;

}

}

// Put back the last character

p = pchBeforeLast;

this->RestoreMultiUnits(multiUnitsBeforeLast);

break;

}

Assert(p - pchMin > 0 && p - pchMin <= LONG_MAX);

*pp = p;

if (!identifyKwds)

{

return tkID;

}

// During syntax coloring, scanner doesn't need to convert the escape sequence to get actual characters, it just needs the classification information

// So call up hashtables custom method to check if the string scanned is identifier or keyword.

// Do the same for deferred parsing, but use a custom method that only tokenizes JS keywords.

if ((m_DeferredParseFlags & ScanFlagSuppressIdPid) != 0)

{

m_ptoken->SetIdentifier(NULL);

if (!fHasEscape)

{

// If there are no escape, that the main scan loop would have found the keyword already

// So we can just assume it is an ID

DebugOnly(int32 cch = UnescapeToTempBuf(pchMin, p));

DebugOnly(tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode()));

Assert(tk == tkID || (tk == tkYIELD && !m_fYieldIsKeyword) || (tk == tkAWAIT && !m_fAwaitIsKeyword));

return tkID;

}

int32 cch = UnescapeToTempBuf(pchMin, p);

tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode());

return (!m_fYieldIsKeyword && tk == tkYIELD) || (!m_fAwaitIsKeyword && tk == tkAWAIT) ? tkID : tk;

}

else if (m_fSyntaxColor)

{

m_ptoken->SetIdentifier(NULL);

// We always need to check TkFromNameLenColor because

// the main Scan switch doesn't detect all non-keyword that needs coloring

// (e.g. int)

int32 cch = UnescapeToTempBuf(pchMin, p);

return m_phtbl->TkFromNameLenColor(m_tempChBuf.m_prgch, cch);

}

// UTF16 Scanner are only for syntax coloring, so it shouldn't come here.

if (EncodingPolicy::MultiUnitEncoding && !fHasMultiChar && !fHasEscape)

{

Assert(sizeof(EncodedChar) == 1);

// If there are no escape, that the main scan loop would have found the keyword already

// So we can just assume it is an ID

DebugOnly(int32 cch = UnescapeToTempBuf(pchMin, p));

DebugOnly(tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode()));

Assert(tk == tkID || (tk == tkYIELD && !m_fYieldIsKeyword) || (tk == tkAWAIT && !m_fAwaitIsKeyword));

m_ptoken->SetIdentifier(reinterpret_cast<const char *>(pchMin), (int32)(p - pchMin));

return tkID;

}

IdentPtr pid = PidOfIdentiferAt(pchMin, p, fHasEscape, fHasMultiChar);

m_ptoken->SetIdentifier(pid);

if (!fHasEscape)

{

// If it doesn't have escape, then Scan() should have taken care of keywords (except

// yield if m_fYieldIsKeyword is false, in which case yield is treated as an identifier, and except

// await if m_fAwaitIsKeyword is false, in which case await is treated as an identifier).

// We don't have to check if the name is reserved word and return it as an Identifier

Assert(pid->Tk(IsStrictMode()) == tkID

|| (pid->Tk(IsStrictMode()) == tkYIELD && !m_fYieldIsKeyword)

|| (pid->Tk(IsStrictMode()) == tkAWAIT && !m_fAwaitIsKeyword));

return tkID;

}

tokens tk = pid->Tk(IsStrictMode());

return tk == tkID || (tk == tkYIELD && !m_fYieldIsKeyword) || (tk == tkAWAIT && !m_fAwaitIsKeyword) ? tkID : tkNone;

}

template <typename EncodingPolicy>

IdentPtr Scanner<EncodingPolicy>::PidAt(size_t iecpMin, size_t iecpLim)

{

Assert(iecpMin < AdjustedLength() && iecpLim <= AdjustedLength() && iecpLim > iecpMin);

return PidOfIdentiferAt(m_pchBase + iecpMin, m_pchBase + iecpLim);

}

template <typename EncodingPolicy>

uint32 Scanner<EncodingPolicy>::UnescapeToTempBuf(EncodedCharPtr p, EncodedCharPtr last)

{

m_tempChBuf.Init();

while( p < last )

{

codepoint_t codePoint;

bool hasEscape, isMultiChar;

bool gotCodePoint = TryReadCodePoint<false>(p, last, &codePoint, &hasEscape, &isMultiChar);

Assert(gotCodePoint);

Assert(codePoint < 0x110000);

if (codePoint < 0x10000)

{

m_tempChBuf.AppendCh((OLECHAR)codePoint);

}

else

{

char16 lower, upper;

Js::NumberUtilities::CodePointAsSurrogatePair(codePoint, &lower, &upper);

m_tempChBuf.AppendCh(lower);

m_tempChBuf.AppendCh(upper);

}

}

return m_tempChBuf.m_ichCur;

}

template <typename EncodingPolicy>

IdentPtr Scanner<EncodingPolicy>::PidOfIdentiferAt(EncodedCharPtr p, EncodedCharPtr last)

{

int32 cch = UnescapeToTempBuf(p, last);

return m_phtbl->PidHashNameLen(m_tempChBuf.m_prgch, cch);

}

template <typename EncodingPolicy>

IdentPtr Scanner<EncodingPolicy>::PidOfIdentiferAt(EncodedCharPtr p, EncodedCharPtr last, bool fHadEscape, bool fHasMultiChar)

{

// If there is an escape sequence in the JS6 identifier or it is a UTF8

// source then we have to convert it to the equivalent char so we use a

// buffer for translation.

if ((EncodingPolicy::MultiUnitEncoding && fHasMultiChar) || fHadEscape)

{

return PidOfIdentiferAt(p, last);

}

else if (EncodingPolicy::MultiUnitEncoding)

{

Assert(sizeof(EncodedChar) == 1);

return m_phtbl->PidHashNameLen(reinterpret_cast<const char *>(p), (int32)(last - p));

}

else

{

Assert(sizeof(EncodedChar) == 2);

return m_phtbl->PidHashNameLen(reinterpret_cast< const char16 * >(p), (int32)(last - p));

}

}

template <typename EncodingPolicy>

typename Scanner<EncodingPolicy>::EncodedCharPtr Scanner<EncodingPolicy>::FScanNumber(EncodedCharPtr p, double *pdbl, bool& likelyInt)

{

EncodedCharPtr last = m_pchLast;

EncodedCharPtr pchT;

likelyInt = true;

// Reset

m_OctOrLeadingZeroOnLastTKNumber = false;

if ('0' == this->PeekFirst(p, last))

{

switch(this->PeekFirst(p + 1, last))

{

case '.':

case 'e':

case 'E':

likelyInt = false;

// Floating point

goto LFloat;

case 'x':

case 'X':

// Hex

*pdbl = Js::NumberUtilities::DblFromHex(p + 2, &pchT);

if (pchT == p + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'x'/'X'

*pdbl = 0;

return p + 1;

}

else

return pchT;

case 'o':

case 'O':

// Octal

*pdbl = Js::NumberUtilities::DblFromOctal(p + 2, &pchT);

if (pchT == p + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'o'/'O'

*pdbl = 0;

return p + 1;

}

return pchT;

case 'b':

case 'B':

// Binary

*pdbl = Js::NumberUtilities::DblFromBinary(p + 2, &pchT);

if (pchT == p + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'b'/'B'

*pdbl = 0;

return p + 1;

}

return pchT;

default:

// Octal

*pdbl = Js::NumberUtilities::DblFromOctal(p, &pchT);

Assert(pchT > p);

#if !SOURCERELEASE

// If an octal literal is malformed then it is in fact a decimal literal.

#endif // !SOURCERELEASE

if(*pdbl != 0 || pchT > p + 1)

m_OctOrLeadingZeroOnLastTKNumber = true; //report as an octal or hex for JSON when leading 0. Just '0' is ok

switch (*pchT)

{

case '8':

case '9':

// case 'e':

// case 'E':

// case '.':

m_OctOrLeadingZeroOnLastTKNumber = false; //08... or 09....

goto LFloat;

}

return pchT;

}

}

else

{

LFloat:

*pdbl = Js::NumberUtilities::StrToDbl(p, &pchT, likelyInt);

Assert(pchT == p || !Js::NumberUtilities::IsNan(*pdbl));

return pchT;

}

}

template <typename EncodingPolicy>

BOOL Scanner<EncodingPolicy>::oFScanNumber(double *pdbl, bool& likelyInt)

{

EncodedCharPtr pchT;

m_OctOrLeadingZeroOnLastTKNumber = false;

likelyInt = true;

if ('0' == *m_currentCharacter)

{

switch (m_currentCharacter[1])

{

case '.':

case 'e':

case 'E':

likelyInt = false;

// Floating point.

goto LFloat;

case 'x':

case 'X':

// Hex.

*pdbl = Js::NumberUtilities::DblFromHex<EncodedChar>(m_currentCharacter + 2, &pchT);

if (pchT == m_currentCharacter + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'x'/'X'

*pdbl = 0;

m_currentCharacter++;

}

else

m_currentCharacter = pchT;

break;

case 'o':

case 'O':

*pdbl = Js::NumberUtilities::DblFromOctal(m_currentCharacter + 2, &pchT);

if (pchT == m_currentCharacter + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'o'/'O'

*pdbl = 0;

m_currentCharacter++;

}

else

m_currentCharacter = pchT;

break;

case 'b':

case 'B':

*pdbl = Js::NumberUtilities::DblFromBinary(m_currentCharacter + 2, &pchT);

if (pchT == m_currentCharacter + 2)

{

// "Octal zero token "0" followed by an identifier token beginning with character 'b'/'B'

*pdbl = 0;

m_currentCharacter++;

}

else

m_currentCharacter = pchT;

break;

default:

// Octal.

*pdbl = Js::NumberUtilities::DblFromOctal(m_currentCharacter, &pchT);

Assert(pchT > m_currentCharacter);

#if !SOURCERELEASE

// If an octal literal is malformed then it is in fact a decimal literal.

#endif // !SOURCERELEASE

if(*pdbl != 0 || pchT > m_currentCharacter + 1)

m_OctOrLeadingZeroOnLastTKNumber = true; //report as an octal or hex for JSON when leading 0. Just '0' is ok

switch (*pchT)

{

case '8':

case '9':

// case 'e':

// case 'E':

// case '.':

m_OctOrLeadingZeroOnLastTKNumber = false; //08... or 09....

goto LFloat;

}

m_currentCharacter = pchT;

break;

}

}

else

{

LFloat:

// Let StrToDbl do all the work.

*pdbl = Js::NumberUtilities::StrToDbl(m_currentCharacter, &pchT, likelyInt);

if (pchT == m_currentCharacter)

return FALSE;

m_currentCharacter = pchT;

Assert(!Js::NumberUtilities::IsNan(*pdbl));

}

return TRUE;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::TryRescanRegExp()

{

EncodedCharPtr current = m_currentCharacter;

tokens result = RescanRegExp();

if (result == tkScanError)

m_currentCharacter = current;

return result;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::RescanRegExp()

{

#if DEBUG

switch (m_ptoken->tk)

{

case tkDiv:

Assert(m_currentCharacter == m_pchMinTok + 1);

break;

case tkAsgDiv:

Assert(m_currentCharacter == m_pchMinTok + 2);

break;

default:

AssertMsg(FALSE, "Who is calling RescanRegExp?");

break;

}

#endif //DEBUG

m_currentCharacter = m_pchMinTok;

if (*m_currentCharacter != '/')

Error(ERRnoSlash);

m_currentCharacter++;

tokens tk = tkNone;

{

ArenaAllocator alloc(_u("RescanRegExp"), m_parser->GetAllocator()->GetPageAllocator(), m_parser->GetAllocator()->outOfMemoryFunc);

tk = ScanRegExpConstant(&alloc);

}

return tk;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::RescanRegExpNoAST()

{

#if DEBUG

switch (m_ptoken->tk)

{

case tkDiv:

Assert(m_currentCharacter == m_pchMinTok + 1);

break;

case tkAsgDiv:

Assert(m_currentCharacter == m_pchMinTok + 2);

break;

default:

AssertMsg(FALSE, "Who is calling RescanRegExpNoParseTree?");

break;

}

#endif //DEBUG

m_currentCharacter = m_pchMinTok;

if (*m_currentCharacter != '/')

Error(ERRnoSlash);

m_currentCharacter++;

tokens tk = tkNone;

{

ArenaAllocator alloc(_u("RescanRegExp"), m_parser->GetAllocator()->GetPageAllocator(), m_parser->GetAllocator()->outOfMemoryFunc);

{

tk = ScanRegExpConstantNoAST(&alloc);

}

}

return tk;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::RescanRegExpTokenizer()

{

#if DEBUG

switch (m_ptoken->tk)

{

case tkDiv:

Assert(m_currentCharacter == m_pchMinTok + 1);

break;

case tkAsgDiv:

Assert(m_currentCharacter == m_pchMinTok + 2);

break;

default:

AssertMsg(FALSE, "Who is calling RescanRegExpNoParseTree?");

break;

}

#endif //DEBUG

m_currentCharacter = m_pchMinTok;

if (*m_currentCharacter != '/')

Error(ERRnoSlash);

m_currentCharacter++;

tokens tk = tkNone;

ThreadContext *threadContext = ThreadContext::GetContextForCurrentThread();

threadContext->EnsureRecycler();

Js::TempArenaAllocatorObject *alloc = threadContext->GetTemporaryAllocator(_u("RescanRegExp"));

TryFinally(

[&]() /* try block */

{

tk = this->ScanRegExpConstantNoAST(alloc->GetAllocator());

},

[&](bool /* hasException */) /* finally block */

{

threadContext->ReleaseTemporaryAllocator(alloc);

});

return tk;

}

template <typename EncodingPolicy>

tokens Scanner<EncodingPolicy>::ScanRegExpConstant(ArenaAllocator* alloc)

{

if (m_parser && m_parser->IsBackgroundParser())

{

PROBE_STACK_NO_DISPOSE(m_scriptContext, Js::Constants::MinStackRegex);

}

else

{

PROBE_STACK(m_scriptContext, Js::Constants::MinStackRegex);

}

// SEE ALSO: RegexHelper::PrimCompileDynamic()

#ifdef PROFILE_EXEC

m_scriptContext->ProfileBegin(Js::RegexCompilePhase);

#endif

ArenaAllocator* ctAllocator = alloc;

UnifiedRegex::StandardChars<EncodedChar>* standardEncodedChars = m_scriptContext->GetThreadContext()->GetStandardChars((EncodedChar*)0);

UnifiedRegex::StandardChars<char16>* standardChars = m_scriptContext->GetThreadContext()->GetStandardChars((char16*)0);

#if ENABLE_REGEX_CONFIG_OPTIONS

UnifiedRegex::DebugWriter *w = 0;

if (REGEX_CONFIG_FLAG(RegexDebug))

w = m_scriptContext->GetRegexDebugWriter();

if (REGEX_CONFIG_FLAG(RegexProfile))

m_scriptContext->GetRegexStatsDatabase()->BeginProfile();

#endif

UnifiedRegex::Node* root = 0;

charcount_t totalLen = 0, bodyChars = 0, totalChars = 0, bodyLen = 0;

UnifiedRegex::RegexFlags flags = UnifiedRegex::NoRegexFlags;

UnifiedRegex::Parser<EncodingPolicy, true> parser

( m_scriptContext

, ctAllocator

, standardEncodedChars

, standardChars

, this->IsFromExternalSource()

#if ENABLE_REGEX_CONFIG_OPTIONS

, w

#endif

);

try

{

root = parser.ParseLiteral(m_currentCharacter, m_pchLast, bodyLen, totalLen, bodyChars, totalChars, flags);

}

catch (UnifiedRegex::ParseError e)

{

#ifdef PROFILE_EXEC

m_scriptContext->ProfileEnd(Js::RegexCompilePhase);