/**

* A TypeScript tokenizer modified for AssemblyScript.

*

* Skips over trivia and provides a general mark/reset mechanism for the parser to utilize on

* ambiguous tokens.

*

* @module tokenizer

*//***/

import {

DiagnosticCode,

DiagnosticMessage,

DiagnosticEmitter

} from "./diagnostics";

import {

Source,

CommentKind

} from "./ast";

import {

CharCode,

isLineBreak,

isWhiteSpace,

isIdentifierStart,

isIdentifierPart,

isDecimalDigit,

isOctalDigit,

isKeywordCharacter

} from "./util";

/** Named token types. */

export enum Token {

// keywords

// discarded: ANY, BOOLEAN, NEVER, NUMBER, STRING, SYMBOL, UNDEFINED, LESSTHAN_SLASH

ABSTRACT,

AS,

ASYNC,

AWAIT, // ES2017

BREAK, // ES2017

CASE, // ES2017

CATCH, // ES2017

CLASS, // ES2017

CONST, // ES2017

CONTINUE, // ES2017

CONSTRUCTOR,

DEBUGGER, // ES2017

DECLARE,

DEFAULT, // ES2017

DELETE, // ES2017

DO, // ES2017

ELSE, // ES2017

ENUM, // ES2017 future

EXPORT, // ES2017

EXTENDS, // ES2017

FALSE, // ES

FINALLY, // ES2017

FOR, // ES2017

FROM, // AS possible identifier

FUNCTION, // ES2017

GET,

IF, // ES2017

IMPLEMENTS, // ES2017 non-lexical

IMPORT, // ES2017

IN, // ES2017

INSTANCEOF, // ES2017

INTERFACE, // ES2017 non-lexical

IS,

KEYOF,

LET, // ES2017 non-lexical

MODULE, // AS possible identifier

NAMESPACE, // AS possible identifier

NEW, // ES2017

NULL, // ES

OF,

PACKAGE, // ES2017 non-lexical

PRIVATE, // ES2017 non-lexical

PROTECTED, // ES2017 non-lexical

PUBLIC, // ES2017 non-lexical

READONLY,

RETURN, // ES2017

SET,

STATIC, // ES2017 non-lexical

SUPER, // ES2017

SWITCH, // ES2017

THIS, // ES2017

THROW, // ES2017

TRUE, // ES

TRY, // ES2017

TYPE, // AS possible identifier

TYPEOF, // ES2017

VAR, // ES2017

VOID, // ES2017

WHILE, // ES2017

WITH, // ES2017

YIELD, // ES2017

// punctuation

OPENBRACE,

CLOSEBRACE,

OPENPAREN,

CLOSEPAREN,

OPENBRACKET,

CLOSEBRACKET,

DOT,

DOT_DOT_DOT,

SEMICOLON,

COMMA,

LESSTHAN,

GREATERTHAN,

LESSTHAN_EQUALS,

GREATERTHAN_EQUALS,

EQUALS_EQUALS,

EXCLAMATION_EQUALS,

EQUALS_EQUALS_EQUALS,

EXCLAMATION_EQUALS_EQUALS,

EQUALS_GREATERTHAN,

PLUS,

MINUS,

ASTERISK_ASTERISK,

ASTERISK,

SLASH,

PERCENT,

PLUS_PLUS,

MINUS_MINUS,

LESSTHAN_LESSTHAN,

GREATERTHAN_GREATERTHAN,

GREATERTHAN_GREATERTHAN_GREATERTHAN,

AMPERSAND,

BAR,

CARET,

EXCLAMATION,

TILDE,

AMPERSAND_AMPERSAND,

BAR_BAR,

QUESTION,

QUESTION_QUESTION,

QUESTION_DOT,

COLON,

EQUALS,

PLUS_EQUALS,

MINUS_EQUALS,

ASTERISK_EQUALS,

ASTERISK_ASTERISK_EQUALS,

SLASH_EQUALS,

PERCENT_EQUALS,

LESSTHAN_LESSTHAN_EQUALS,

GREATERTHAN_GREATERTHAN_EQUALS,

GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS,

AMPERSAND_EQUALS,

BAR_EQUALS,

CARET_EQUALS,

AT,

// literals

IDENTIFIER,

STRINGLITERAL,

INTEGERLITERAL,

FLOATLITERAL,

// meta

INVALID,

ENDOFFILE

}

export enum IdentifierHandling {

DEFAULT,

PREFER,

ALWAYS

}

export function tokenFromKeyword(text: string): Token {

assert(text.length);

switch (text.charCodeAt(0)) {

case CharCode.a: {

switch (text) {

case "abstract": return Token.ABSTRACT;

case "as": return Token.AS;

case "async": return Token.ASYNC;

case "await": return Token.AWAIT;

}

break;

}

case CharCode.b: {

switch (text) {

case "break": return Token.BREAK;

}

break;

}

case CharCode.c: {

switch (text) {

case "case": return Token.CASE;

case "catch": return Token.CATCH;

case "class": return Token.CLASS;

case "continue": return Token.CONTINUE;

case "const": return Token.CONST;

case "constructor": return Token.CONSTRUCTOR;

}

break;

}

case CharCode.d: {

switch (text) {

case "debugger": return Token.DEBUGGER;

case "declare": return Token.DECLARE;

case "default": return Token.DEFAULT;

case "delete": return Token.DELETE;

case "do": return Token.DO;

}

break;

}

case CharCode.e: {

switch (text) {

case "else": return Token.ELSE;

case "enum": return Token.ENUM;

case "export": return Token.EXPORT;

case "extends": return Token.EXTENDS;

}

break;

}

case CharCode.f: {

switch (text) {

case "false": return Token.FALSE;

case "finally": return Token.FINALLY;

case "for": return Token.FOR;

case "from": return Token.FROM;

case "function": return Token.FUNCTION;

}

break;

}

case CharCode.g: {

switch (text) {

case "get": return Token.GET;

}

break;

}

case CharCode.i: {

switch (text) {

case "if": return Token.IF;

case "implements": return Token.IMPLEMENTS;

case "import": return Token.IMPORT;

case "in": return Token.IN;

case "instanceof": return Token.INSTANCEOF;

case "interface": return Token.INTERFACE;

case "is": return Token.IS;

}

break;

}

case CharCode.k: {

switch (text) {

case "keyof": return Token.KEYOF;

}

break;

}

case CharCode.l: {

switch (text) {

case "let": return Token.LET;

}

break;

}

case CharCode.m: {

switch (text) {

case "module": return Token.MODULE;

}

break;

}

case CharCode.n: {

switch (text) {

case "namespace": return Token.NAMESPACE;

case "new": return Token.NEW;

case "null": return Token.NULL;

}

break;

}

case CharCode.o: {

switch (text) {

case "of": return Token.OF;

}

break;

}

case CharCode.p: {

switch (text) {

case "package": return Token.PACKAGE;

case "private": return Token.PRIVATE;

case "protected": return Token.PROTECTED;

case "public": return Token.PUBLIC;

}

break;

}

case CharCode.r: {

switch (text) {

case "readonly": return Token.READONLY;

case "return": return Token.RETURN;

}

break;

}

case CharCode.s: {

switch (text) {

case "set": return Token.SET;

case "static": return Token.STATIC;

case "super": return Token.SUPER;

case "switch": return Token.SWITCH;

}

break;

}

case CharCode.t: {

switch (text) {

case "this": return Token.THIS;

case "throw": return Token.THROW;

case "true": return Token.TRUE;

case "try": return Token.TRY;

case "type": return Token.TYPE;

case "typeof": return Token.TYPEOF;

}

break;

}

case CharCode.v: {

switch (text) {

case "var": return Token.VAR;

case "void": return Token.VOID;

}

break;

}

case CharCode.w: {

switch (text) {

case "while": return Token.WHILE;

case "with": return Token.WITH;

}

break;

}

case CharCode.y: {

switch (text) {

case "yield": return Token.YIELD;

}

break;

}

}

return Token.INVALID;

}

export function tokenIsAlsoIdentifier(token: Token): bool {

switch (token) {

case Token.ABSTRACT:

case Token.AS:

case Token.CONSTRUCTOR:

case Token.DECLARE:

case Token.DELETE:

case Token.FROM:

case Token.FOR:

case Token.GET:

case Token.INSTANCEOF:

case Token.IS:

case Token.KEYOF:

case Token.MODULE:

case Token.NAMESPACE:

case Token.NULL:

case Token.READONLY:

case Token.SET:

case Token.TYPE:

case Token.VOID: return true;

default: return false;

}

}

export function isIllegalVariableIdentifier(name: string): bool {

assert(name.length);

switch (name.charCodeAt(0)) {

case CharCode.d: return name == "delete";

case CharCode.f: return name == "for";

case CharCode.i: return name == "instanceof";

case CharCode.n: return name == "null";

case CharCode.v: return name == "void";

}

return false;

}

export function operatorTokenToString(token: Token): string {

switch (token) {

case Token.DELETE: return "delete";

case Token.IN: return "in";

case Token.INSTANCEOF: return "instanceof";

case Token.NEW: return "new";

case Token.TYPEOF: return "typeof";

case Token.VOID: return "void";

case Token.YIELD: return "yield";

case Token.DOT_DOT_DOT: return "...";

case Token.COMMA: return ",";

case Token.LESSTHAN: return "<";

case Token.GREATERTHAN: return ">";

case Token.LESSTHAN_EQUALS: return "<=";

case Token.GREATERTHAN_EQUALS: return ">=";

case Token.EQUALS_EQUALS: return "==";

case Token.EXCLAMATION_EQUALS: return "!=";

case Token.EQUALS_EQUALS_EQUALS: return "===";

case Token.EXCLAMATION_EQUALS_EQUALS: return "!==";

case Token.PLUS: return "+";

case Token.MINUS: return "-";

case Token.ASTERISK_ASTERISK: return "**";

case Token.ASTERISK: return "*";

case Token.SLASH: return "/";

case Token.PERCENT: return "%";

case Token.PLUS_PLUS: return "++";

case Token.MINUS_MINUS: return "--";

case Token.LESSTHAN_LESSTHAN: return "<<";

case Token.GREATERTHAN_GREATERTHAN: return ">>";

case Token.GREATERTHAN_GREATERTHAN_GREATERTHAN: return ">>>";

case Token.AMPERSAND: return "&";

case Token.BAR: return "|";

case Token.CARET: return "^";

case Token.EXCLAMATION: return "!";

case Token.TILDE: return "~";

case Token.AMPERSAND_AMPERSAND: return "&&";

case Token.BAR_BAR: return "||";

case Token.QUESTION_QUESTION: return "??";

case Token.EQUALS: return "=";

case Token.PLUS_EQUALS: return "+=";

case Token.MINUS_EQUALS: return "-=";

case Token.ASTERISK_EQUALS: return "*=";

case Token.ASTERISK_ASTERISK_EQUALS: return "**=";

case Token.SLASH_EQUALS: return "/=";

case Token.PERCENT_EQUALS: return "%=";

case Token.LESSTHAN_LESSTHAN_EQUALS: return "<<=";

case Token.GREATERTHAN_GREATERTHAN_EQUALS: return ">>=";

case Token.GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS: return ">>>=";

case Token.AMPERSAND_EQUALS: return "&=";

case Token.BAR_EQUALS: return "|=";

case Token.CARET_EQUALS: return "^=";

default: {

assert(false);

return "";

}

}

}

export class Range {

source: Source;

start: i32;

end: i32;

// TODO: set these while tokenizing

// line: i32;

// column: i32;

constructor(source: Source, start: i32, end: i32) {

this.source = source;

this.start = start;

this.end = end;

}

static join(a: Range, b: Range): Range {

if (a.source != b.source) throw new Error("source mismatch");

return new Range(a.source,

a.start < b.start ? a.start : b.start,

a.end > b.end ? a.end : b.end

);

}

get atStart(): Range {

return new Range(this.source, this.start, this.start);

}

get atEnd(): Range {

return new Range(this.source, this.end, this.end);

}

get line(): i32 {

var text = this.source.text;

var line = 1;

for (let pos = this.start; pos >= 0; --pos) {

if (text.charCodeAt(pos) == CharCode.LINEFEED) line++;

}

return line;

}

get column(): i32 {

var text = this.source.text;

var column = 0;

for (let pos = this.start - 1; pos >= 0; --pos) {

if (text.charCodeAt(pos) == CharCode.LINEFEED) break;

++column;

}

return column;

}

toString(): string {

return this.source.text.substring(this.start, this.end);

}

debugInfoRef: usize = 0;

}

declare function parseFloat(str: string): f64;

/** Handler for intercepting comments while tokenizing. */

export type CommentHandler = (kind: CommentKind, text: string, range: Range) => void;

/** Tokenizes a source to individual {@link Token}s. */

export class Tokenizer extends DiagnosticEmitter {

source: Source;

end: i32 = 0;

pos: i32 = 0;

token: Token = -1;

tokenPos: i32 = 0;

nextToken: Token = -1;

nextTokenPos: i32 = 0;

nextTokenOnNewLine: bool = false;

onComment: CommentHandler | null = null;

/** Constructs a new tokenizer. */

constructor(source: Source, diagnostics: DiagnosticMessage[] | null = null) {

super(diagnostics);

this.source = source;

this.pos = 0;

this.end = source.text.length;

this.diagnostics = diagnostics || new Array();

var end = this.end;

var text = source.text;

// skip bom

if (

this.pos < end &&

text.charCodeAt(this.pos) == CharCode.BYTEORDERMARK

) {

++this.pos;

}

// skip shebang

if (

this.pos + 1 < end &&

text.charCodeAt(this.pos) == CharCode.HASH &&

text.charCodeAt(this.pos + 1) == CharCode.EXCLAMATION

) {

this.pos += 2;

while (

this.pos < end &&

text.charCodeAt(this.pos) != CharCode.LINEFEED

) {

++this.pos;

}

// 'next' now starts at lf or eof

}

}

next(identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT): Token {

this.nextToken = -1;

return this.token = this.unsafeNext(identifierHandling);

}

private unsafeNext(

identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT,

maxTokenLength: i32 = i32.MAX_VALUE

): Token {

var end = this.end;

var text = this.source.text;

while (this.pos < end) {

this.tokenPos = this.pos;

let c = text.charCodeAt(this.pos);

switch (c) {

case CharCode.CARRIAGERETURN: {

if (!(

++this.pos < end &&

text.charCodeAt(this.pos) == CharCode.LINEFEED

)) break;

// otherwise fall-through

}

case CharCode.LINEFEED:

case CharCode.TAB:

case CharCode.VERTICALTAB:

case CharCode.FORMFEED:

case CharCode.SPACE: {

++this.pos;

break;

}

case CharCode.EXCLAMATION: {

++this.pos;

if (

maxTokenLength > 1 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

if (

maxTokenLength > 2 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.EXCLAMATION_EQUALS_EQUALS;

}

return Token.EXCLAMATION_EQUALS;

}

return Token.EXCLAMATION;

}

case CharCode.DOUBLEQUOTE:

case CharCode.SINGLEQUOTE:

case CharCode.BACKTICK: { // TODO

return Token.STRINGLITERAL; // expects a call to readString

}

case CharCode.PERCENT: {

++this.pos;

if (

maxTokenLength > 1 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.PERCENT_EQUALS;

}

return Token.PERCENT;

}

case CharCode.AMPERSAND: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.AMPERSAND) {

++this.pos;

return Token.AMPERSAND_AMPERSAND;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.AMPERSAND_EQUALS;

}

}

return Token.AMPERSAND;

}

case CharCode.OPENPAREN: {

++this.pos;

return Token.OPENPAREN;

}

case CharCode.CLOSEPAREN: {

++this.pos;

return Token.CLOSEPAREN;

}

case CharCode.ASTERISK: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.ASTERISK_EQUALS;

}

if (chr == CharCode.ASTERISK) {

++this.pos;

if (

maxTokenLength > 2 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.ASTERISK_ASTERISK_EQUALS;

}

return Token.ASTERISK_ASTERISK;

}

}

return Token.ASTERISK;

}

case CharCode.PLUS: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.PLUS) {

++this.pos;

return Token.PLUS_PLUS;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.PLUS_EQUALS;

}

}

return Token.PLUS;

}

case CharCode.COMMA: {

++this.pos;

return Token.COMMA;

}

case CharCode.MINUS: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.MINUS) {

++this.pos;

return Token.MINUS_MINUS;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.MINUS_EQUALS;

}

}

return Token.MINUS;

}

case CharCode.DOT: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (isDecimalDigit(chr)) {

--this.pos;

return Token.FLOATLITERAL; // expects a call to readFloat

}

if (

maxTokenLength > 2 && this.pos + 1 < end &&

chr == CharCode.DOT &&

text.charCodeAt(this.pos + 1) == CharCode.DOT

) {

this.pos += 2;

return Token.DOT_DOT_DOT;

}

}

return Token.DOT;

}

case CharCode.SLASH: {

let commentStartPos = this.pos;

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.SLASH) { // single-line

let commentKind = CommentKind.LINE;

if (

this.pos + 1 < end &&

text.charCodeAt(this.pos + 1) == CharCode.SLASH

) {

++this.pos;

commentKind = CommentKind.TRIPLE;

}

while (++this.pos < end) {

if (text.charCodeAt(this.pos) == CharCode.LINEFEED) {

++this.pos;

break;

}

}

if (this.onComment) {

this.onComment(

commentKind,

text.substring(commentStartPos, this.pos),

this.range(commentStartPos, this.pos)

);

}

break;

}

if (chr == CharCode.ASTERISK) { // multi-line

let closed = false;

while (++this.pos < end) {

c = text.charCodeAt(this.pos);

if (

c == CharCode.ASTERISK &&

this.pos + 1 < end &&

text.charCodeAt(this.pos + 1) == CharCode.SLASH

) {

this.pos += 2;

closed = true;

break;

}

}

if (!closed) {

this.error(

DiagnosticCode._0_expected,

this.range(this.pos), "*/"

);

} else if (this.onComment) {

this.onComment(

CommentKind.BLOCK,

text.substring(commentStartPos, this.pos),

this.range(commentStartPos, this.pos)

);

}

break;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.SLASH_EQUALS;

}

}

return Token.SLASH;

}

case CharCode._0:

case CharCode._1:

case CharCode._2:

case CharCode._3:

case CharCode._4:

case CharCode._5:

case CharCode._6:

case CharCode._7:

case CharCode._8:

case CharCode._9: {

return this.testInteger()

? Token.INTEGERLITERAL // expects a call to readInteger

: Token.FLOATLITERAL; // expects a call to readFloat

}

case CharCode.COLON: {

++this.pos;

return Token.COLON;

}

case CharCode.SEMICOLON: {

++this.pos;

return Token.SEMICOLON;

}

case CharCode.LESSTHAN: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.LESSTHAN) {

++this.pos;

if (

maxTokenLength > 2 &&

this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.LESSTHAN_LESSTHAN_EQUALS;

}

return Token.LESSTHAN_LESSTHAN;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.LESSTHAN_EQUALS;

}

}

return Token.LESSTHAN;

}

case CharCode.EQUALS: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.EQUALS) {

++this.pos;

if (

maxTokenLength > 2 &&

this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.EQUALS_EQUALS_EQUALS;

}

return Token.EQUALS_EQUALS;

}

if (chr == CharCode.GREATERTHAN) {

++this.pos;

return Token.EQUALS_GREATERTHAN;

}

}

return Token.EQUALS;

}

case CharCode.GREATERTHAN: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.GREATERTHAN) {

++this.pos;

if (maxTokenLength > 2 && this.pos < end) {

chr = text.charCodeAt(this.pos);

if (chr == CharCode.GREATERTHAN) {

++this.pos;

if (

maxTokenLength > 3 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS;

}

return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.GREATERTHAN_GREATERTHAN_EQUALS;

}

}

return Token.GREATERTHAN_GREATERTHAN;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.GREATERTHAN_EQUALS;

}

}

return Token.GREATERTHAN;

}

case CharCode.QUESTION: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.QUESTION) {

++this.pos;

return Token.QUESTION_QUESTION;

} else if (

chr == CharCode.DOT && (

this.pos + 1 == end ||

!isDecimalDigit(text.charCodeAt(this.pos + 1))

)

) {

++this.pos;

return Token.QUESTION_DOT;

}

}

return Token.QUESTION;

}

case CharCode.OPENBRACKET: {

++this.pos;

return Token.OPENBRACKET;

}

case CharCode.CLOSEBRACKET: {

++this.pos;

return Token.CLOSEBRACKET;

}

case CharCode.CARET: {

++this.pos;

if (

maxTokenLength > 1 && this.pos < end &&

text.charCodeAt(this.pos) == CharCode.EQUALS

) {

++this.pos;

return Token.CARET_EQUALS;

}

return Token.CARET;

}

case CharCode.OPENBRACE: {

++this.pos;

return Token.OPENBRACE;

}

case CharCode.BAR: {

++this.pos;

if (maxTokenLength > 1 && this.pos < end) {

let chr = text.charCodeAt(this.pos);

if (chr == CharCode.BAR) {

++this.pos;

return Token.BAR_BAR;

}

if (chr == CharCode.EQUALS) {

++this.pos;

return Token.BAR_EQUALS;

}

}

return Token.BAR;

}

case CharCode.CLOSEBRACE: {

++this.pos;

return Token.CLOSEBRACE;

}

case CharCode.TILDE: {

++this.pos;

return Token.TILDE;

}

case CharCode.AT: {

++this.pos;

return Token.AT;

}

default: {

if (isIdentifierStart(c)) {

if (isKeywordCharacter(c)) {

let posBefore = this.pos;

while (

++this.pos < end &&

isIdentifierPart(c = text.charCodeAt(this.pos))

) {

if (!isKeywordCharacter(c)) {

this.pos = posBefore;

return Token.IDENTIFIER;

}

}

let keywordText = text.substring(posBefore, this.pos);

let keywordToken = tokenFromKeyword(keywordText);

if (

keywordToken !== Token.INVALID &&

identifierHandling !== IdentifierHandling.ALWAYS &&

!(

identifierHandling === IdentifierHandling.PREFER &&

tokenIsAlsoIdentifier(keywordToken)

)

) {

return keywordToken;

}

this.pos = posBefore;

}

return Token.IDENTIFIER; // expects a call to readIdentifier

} else if (isWhiteSpace(c)) {

++this.pos;

break;

}

this.error(

DiagnosticCode.Invalid_character,

this.range(this.pos, this.pos + 1)

);

++this.pos;

return Token.INVALID;

}

}

}

return Token.ENDOFFILE;

}

peek(

checkOnNewLine: bool = false,

identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT,

maxCompoundLength: i32 = i32.MAX_VALUE

): Token {

var text = this.source.text;