Remove error on invalid jsdoc tokens (microsoft#32769)

sandersn · web-flow · commit 85b8d27ea381 · 2019-08-09T12:53:15.000-07:00
* Remove error on invalid jsdoc tokens In JSDoc: 1. In the scanner, don't issue an error, even for invalid identifiers. 2. In the binder, don't issue an error for reserved (but otherwise valid) identifiers. /** * Example of 1: "\" * Example of 2: @Private */ * Update baselines * Add invalid unicode escape test for JSDoc * Add quotes around invalid unicode escape * Add another unicode escape JSDoc test
diff --git a/src/compiler/binder.ts b/src/compiler/binder.ts
@@ -1842,7 +1842,8 @@ namespace ts {
                 node.originalKeywordKind! >= SyntaxKind.FirstFutureReservedWord &&
                 node.originalKeywordKind! <= SyntaxKind.LastFutureReservedWord &&
                 !isIdentifierName(node) &&
-                !(node.flags & NodeFlags.Ambient)) {
+                !(node.flags & NodeFlags.Ambient) &&
+                !(node.flags & NodeFlags.JSDoc)) {
 
                 // Report error only if there are no parse errors in file
                 if (!file.parseDiagnostics.length) {
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -6434,7 +6434,7 @@ namespace ts {
             export function parseIsolatedJSDocComment(content: string, start: number | undefined, length: number | undefined): { jsDoc: JSDoc, diagnostics: Diagnostic[] } | undefined {
                 initializeState(content, ScriptTarget.Latest, /*_syntaxCursor:*/ undefined, ScriptKind.JS);
                 sourceFile = <SourceFile>{ languageVariant: LanguageVariant.Standard, text: content }; // tslint:disable-line no-object-literal-type-assertion
-                const jsDoc = parseJSDocCommentWorker(start, length);
+                const jsDoc = doInsideOfContext(NodeFlags.JSDoc, () => parseJSDocCommentWorker(start, length));
                 const diagnostics = parseDiagnostics;
                 clearState();
 
@@ -6446,7 +6446,7 @@ namespace ts {
                 const saveParseDiagnosticsLength = parseDiagnostics.length;
                 const saveParseErrorBeforeNextFinishedNode = parseErrorBeforeNextFinishedNode;
 
-                const comment = parseJSDocCommentWorker(start, length);
+                const comment = doInsideOfContext(NodeFlags.JSDoc, () => parseJSDocCommentWorker(start, length));
                 if (comment) {
                     comment.parent = parent;
                 }
@@ -6477,7 +6477,7 @@ namespace ts {
                 CallbackParameter = 1 << 2,
             }
 
-            export function parseJSDocCommentWorker(start = 0, length: number | undefined): JSDoc | undefined {
+            function parseJSDocCommentWorker(start = 0, length: number | undefined): JSDoc | undefined {
                 const content = sourceText;
                 const end = length === undefined ? content.length : start + length;
                 length = end - start;
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -2159,7 +2159,6 @@ namespace ts {
                         tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts();
                         return token = getIdentifierToken();
                     }
-                    error(Diagnostics.Invalid_character);
                     pos++;
                     return token = SyntaxKind.Unknown;
             }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.@link tags.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.@link tags.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 127,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 66,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 44,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 49,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.asteriskAfterPreamble.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.asteriskAfterPreamble.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 23,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "comment": "* @type {number}"
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.authorTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.authorTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 112,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.emptyComment.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.emptyComment.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 5,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.leadingAsterisk.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.leadingAsterisk.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 27,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 61,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noLeadingAsterisk.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noLeadingAsterisk.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 27,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 20,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 34,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 59,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName1.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 61,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName2.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 66,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType1.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 34,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType2.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 46,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramWithoutType.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramWithoutType.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 23,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag1.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 29,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag2.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 54,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnsTag1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnsTag1.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 30,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 24,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag2.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 26,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag3.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag3.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 27,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag4.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag4.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 27,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag5.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag5.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 28,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag6.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag6.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 60,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.threeAsterisks.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.threeAsterisks.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 7,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "comment": "*"
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTag2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTag2.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 60,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTagOnSameLine.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTagOnSameLine.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 56,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typeTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typeTag.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 27,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typedefTagWithChildrenTags.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typedefTagWithChildrenTags.json
@@ -2,6 +2,7 @@
     "kind": "JSDocComment",
     "pos": 0,
     "end": 102,
+    "flags": "JSDoc",
     "modifierFlagsCache": 0,
     "transformFlags": 0,
     "tags": {
diff --git a/tests/baselines/reference/jsdocInvalidTokens.symbols b/tests/baselines/reference/jsdocInvalidTokens.symbols
@@ -0,0 +1,12 @@
+=== tests/cases/conformance/scanner/jsdocInvalidTokens.js ===
+/**
+ *
+ * unicode-escape = "\u{abcdefghi}" -- should not have error for invalid unicode escape
+ * unicode-escape2 = "qq\u{abcdefghi}" -- no error here either
+ * quoted-pair = "\" -- should not have error for invalid quote sequence
+ * or for the tag below:
+ * @private
+ */
+var hi = 1
+>hi : Symbol(hi, Decl(jsdocInvalidTokens.js, 8, 3))
+
diff --git a/tests/baselines/reference/jsdocInvalidTokens.types b/tests/baselines/reference/jsdocInvalidTokens.types
@@ -0,0 +1,13 @@
+=== tests/cases/conformance/scanner/jsdocInvalidTokens.js ===
+/**
+ *
+ * unicode-escape = "\u{abcdefghi}" -- should not have error for invalid unicode escape
+ * unicode-escape2 = "qq\u{abcdefghi}" -- no error here either
+ * quoted-pair = "\" -- should not have error for invalid quote sequence
+ * or for the tag below:
+ * @private
+ */
+var hi = 1
+>hi : number
+>1 : 1
+
diff --git a/tests/cases/conformance/scanner/jsdocInvalidTokens.ts b/tests/cases/conformance/scanner/jsdocInvalidTokens.ts
@@ -0,0 +1,15 @@
+// @Filename: jsdocInvalidTokens.js
+// @strict: true
+// @allowJs: true
+// @checkJs: true
+// @noEmit: true
+
+/**
+ *
+ * unicode-escape = "\u{abcdefghi}" -- should not have error for invalid unicode escape
+ * unicode-escape2 = "qq\u{abcdefghi}" -- no error here either
+ * quoted-pair = "\" -- should not have error for invalid quote sequence
+ * or for the tag below:
+ * @private
+ */
+var hi = 1

Original file line number	Diff line number	Diff line change
`@@ -2159,7 +2159,6 @@ namespace ts {`
`2159`	`2159`	`tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts();`
`2160`	`2160`	`return token = getIdentifierToken();`
`2161`	`2161`	`}`
`2162`		`- error(Diagnostics.Invalid_character);`
`2163`	`2162`	`pos++;`
`2164`	`2163`	`return token = SyntaxKind.Unknown;`
`2165`	`2164`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@`
`2`	`2`	`"kind": "JSDocComment",`
`3`	`3`	`"pos": 0,`
`4`	`4`	`"end": 5,`
	`5`	`+ "flags": "JSDoc",`
`5`	`6`	`"modifierFlagsCache": 0,`
`6`	`7`	`"transformFlags": 0`
`7`	`8`	`}`