package processing.mode.cpp;

import processing.mode.cpp.FunctionPointerType;

import processing.mode.cpp.FunctionSignatureType;

import processing.mode.cpp.NamedType;

import processing.mode.cpp.Param;

import processing.mode.cpp.TypeRef;

import java.util.ArrayList;

import java.util.List;

/**

* Recursive-descent parser for CppMode's Java-flavored-C++ grammar subset.

*

* Design notes:

* - Comments are consumed by skipComments() and attached as leadingComments

* on the next real node produced, per the Lexer's comment-as-first-class-

* token design (see Lexer notes). This is the direct replacement for the

* old blankCommentsAndLiterals regex pass.

* - Fail-fast: the first parse error throws ParseException immediately.

* No error recovery is attempted (see ParseException's notes for why).

* - Multi-declarator statements ("int rectX, rectY;") are desugared into

* multiple single-name nodes at this layer -- see parseVariableDeclTail.

*/

public final class Parser {

private final List<CppLexerToken> tokens;

private int pos = 0;

public Parser(List<CppLexerToken> tokens) {

this.tokens = tokens;

}

public static CompilationUnit parse(String source) {

List<CppLexerToken> tokens = new CppLexer(source).tokenize();

return new Parser(tokens).parseCompilationUnit();

}

/** Test-only entry point: parses a single TypeRef from source text in isolation. */

static TypeRef parseTypeRefFromString(String source) {

List<CppLexerToken> tokens = new CppLexer(source).tokenize();

return new Parser(tokens).parseTypeRef();

}

// ----- Core cursor helpers -----------------------------------------

//

// peek()/peek(ahead)/advance() transparently skip over comment tokens.

// This is deliberate and fixes a real bug class found via real-corpus

// testing: a line comment can legally appear in the MIDDLE of a

// multi-line expression (e.g. a function call whose arguments are each

// commented, confirmed real by Move_Eye.pde's

// "camera(x, y, z, // eyeX, eyeY, eyeZ \n ...")). The original design

// only collected leading comments at statement/declaration boundaries

// via consumeLeadingComments(), which left every OTHER parse point

// (inside parseArgList, parsePostfix, anywhere mid-expression) just as

// comment-blind as the regex passes this parser exists to replace.

// Routing comment-skipping through the cursor primitives themselves

// fixes every call site at once instead of needing a fix at each one.

//

// consumeLeadingComments() still exists and is still called explicitly

// at statement/declaration start -- it now serves only to COLLECT the

// comment tokens for attachment to the next node's leadingComments

// field, not to make parsing work correctly (that's now peek/advance's

// job). It works correctly alongside this since skipCommentsAt() is

// idempotent -- calling it twice in a row from the same position is a

// no-op the second time.

private void skipCommentsAt() {

while (pos < tokens.size()

&& (tokens.get(pos).type() == CppLexerTokenType.LINE_COMMENT

|| tokens.get(pos).type() == CppLexerTokenType.BLOCK_COMMENT)) {

pos++;

}

}

private CppLexerToken peek() {

int p = pos;

while (p < tokens.size() && (tokens.get(p).type() == CppLexerTokenType.LINE_COMMENT || tokens.get(p).type() == CppLexerTokenType.BLOCK_COMMENT)) p++;

return p < tokens.size() ? tokens.get(p) : tokens.get(tokens.size() - 1);

}

private CppLexerToken peek(int ahead) {

// Walk forward from pos, skipping comment tokens, to find the

// "ahead"-th real (non-comment) token after the current position.

skipCommentsAt();

int p = pos;

int remaining = ahead;

while (remaining > 0 && p < tokens.size() - 1) {

p++;

while (p < tokens.size() && (tokens.get(p).type() == CppLexerTokenType.LINE_COMMENT

|| tokens.get(p).type() == CppLexerTokenType.BLOCK_COMMENT)) {

p++;

}

remaining--;

}

return p < tokens.size() ? tokens.get(p) : tokens.get(tokens.size() - 1);

}

private CppLexerToken advance() {

skipCommentsAt();

CppLexerToken t = tokens.get(pos);

if (t.type() != CppLexerTokenType.EOF) pos++;

return t;

}

private CppLexerToken prev() {

// Walk backward from pos-1 skipping comments, return the last

// consumed real token. Used by expectPunct(";") to point the

// error caret at the end of the broken statement, not the start

// of the next token.

int p = pos - 1;

while (p > 0 && (tokens.get(p).type() == CppLexerTokenType.LINE_COMMENT

|| tokens.get(p).type() == CppLexerTokenType.BLOCK_COMMENT)) p--;

return p >= 0 ? tokens.get(p) : tokens.get(0);

}

private boolean isAtEnd() {

return peek().type() == CppLexerTokenType.EOF;

}

private boolean check(CppLexerTokenType type) {

return peek().type() == type;

}

private boolean checkKeyword(String kw) {

return peek().isKeyword(kw);

}

private boolean checkPunct(String p) {

return peek().isPunct(p);

}

private boolean checkOp(String op) {

return peek().isOp(op);

}

/** True if the current token is the integer literal "0" specifically

* (not any other literal or expression) -- used to recognize the

* pure-virtual specifier "= 0" precisely, without accidentally

* matching some other "= <expr>" shape. */

private boolean checkLiteralZero() {

CppLexerToken t = peek();

return t.type() == CppLexerTokenType.INT_LITERAL && t.text().equals("0");

}

private boolean matchKeyword(String kw) {

if (checkKeyword(kw)) { advance(); return true; }

return false;

}

private boolean matchPunct(String p) {

if (checkPunct(p)) { advance(); return true; }

return false;

}

private boolean matchOp(String op) {

if (checkOp(op)) { advance(); return true; }

return false;

}

private CppLexerToken expectPunct(String p) {

if (checkPunct(p)) return advance();

if (p.equals(";")) {

// Point the caret at the end of the previous token (where the

// semicolon should have been), not the start of the next token.

CppLexerToken last = prev();

throw new ParseException(

"expected ';' but found '" + peek().text() + "'",

last.line(), last.col() + last.text().length()

);

}

throw error("expected '" + p + "' but found '" + peek().text() + "'");

}

private CppLexerToken expectOp(String op) {

if (checkOp(op)) return advance();

throw error("expected '" + op + "' but found '" + peek().text() + "'");

}

private CppLexerToken expectKeyword(String kw) {

if (checkKeyword(kw)) return advance();

throw error("expected '" + kw + "' but found '" + peek().text() + "'");

}

/**

* Set of KEYWORD-classified tokens that are NOT actually reserved

* words in real Processing/Java/C++ -- they're lexed as keywords

* purely so they can be recognized as TYPE names in a type-name

* position (see Lexer's KEYWORDS set and its own comment on this),

* but that classification incorrectly also blocked them from ever

* being used as a declarator/variable NAME, which real Processing

* code is free to do.

*

* Found via a real sketch (RayTracer.pde's "Vec3 color;" struct

* field -- "color" the Processing pseudo-type, used as an ordinary

* field name, which is completely legal Processing/C++ and not

* something the language itself reserves) hitting

* "expected identifier but found 'color'" outright.

*

* Deliberately a narrow, explicit allow-list -- NOT a general

* relaxation letting any KEYWORD stand in for an identifier, which

* would incorrectly let real reserved words ("int", "if", "class",

* etc.) be used as names too. Only pseudo-type keywords that this

* project itself introduced for type-name recognition belong here.

*/

private static final java.util.Set<String> PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES = java.util.Set.of(

"color"

);

private CppLexerToken expectIdentifier() {

if (check(CppLexerTokenType.IDENTIFIER)) return advance();

if (check(CppLexerTokenType.KEYWORD) && PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text())) {

return advance();

}

throw error("expected identifier but found '" + peek().text() + "'");

}

private ParseException error(String message) {

CppLexerToken t = peek();

return new ParseException(message, t.line(), t.col());

}

/**

* Consumes and discards any whitespace-adjacent comment tokens immediately

* preceding the current position, returning them so the caller can attach

* them as leadingComments on the node it's about to build. Must be called

* at the start of every "parse one item/statement/declaration" entry point

* so comments are never silently dropped nor mistaken for code.

*/

/**

* Consumes and collects any comment tokens sitting at the current raw

* position, for attachment to the next node's leadingComments field.

*

* Deliberately reads tokens.get(pos) directly rather than calling

* peek()/check() -- those now transparently skip comment tokens (see

* the cursor-helpers notes above, added after the Move_Eye.pde bug),

* so they can never see a comment to report back here. This method is

* the one place that still needs to look at raw, unfiltered token-

* stream position.

*/

private List<CppLexerToken> consumeLeadingComments() {

List<CppLexerToken> comments = new ArrayList<>();

while (pos < tokens.size()

&& (tokens.get(pos).type() == CppLexerTokenType.LINE_COMMENT

|| tokens.get(pos).type() == CppLexerTokenType.BLOCK_COMMENT)) {

comments.add(tokens.get(pos));

pos++;

}

return comments;

}

// ----- Entry point ---------------------------------------------------

public CompilationUnit parseCompilationUnit() {

List<TopLevelItem> items = new ArrayList<>();

consumeLeadingComments(); // leading file-level comments currently dropped

// at EOF if nothing follows; fine for now

while (!isAtEnd()) {

List<CppLexerToken> comments = consumeLeadingComments();

if (isAtEnd()) break;

items.addAll(parseTopLevelItem(comments));

}

return new CompilationUnit(items);

}

/**

* Dispatches on lookahead to the appropriate top-level (or class-member,

* since both positions share this same dispatch per TopLevelItem's design

* notes) sub-parser. Returns a List since a single source statement can

* desugar into multiple VariableDecl nodes (multi-declarator decls,

* confirmed at both top-level and class-member scope by the Button and

* Handles fixtures respectively) -- every other branch returns a

* single-element list.

*/

private List<TopLevelItem> parseTopLevelItem(List<CppLexerToken> leadingComments) {

consumeAttributes();

if (check(CppLexerTokenType.PREPROCESSOR_DIRECTIVE)) {

CppLexerToken t = advance();

return List.of(new PreprocessorLine(t.text(), t.line(), t.col(), leadingComments));

}

// C++20 concept definition: "concept Name = constraint-expr;"

// "concept" is lexed as IDENTIFIER (not in the keyword set), so

// use text comparison rather than checkKeyword.

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("concept")) {

int startPos = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); }

// Reconstruct the raw text and emit as-is (concept defs are valid at

// namespace scope and must be visible before use in template params).

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

List<String> templateParams = List.of();

// extern "C" / extern "C++" linkage specification -- consume verbatim

// "extern template class Foo<T>;" -- explicit instantiation declaration

// Suppresses implicit instantiation in this TU. Consume verbatim to ";".

if (checkKeyword("extern") && pos + 1 < tokens.size()

&& tokens.get(pos + 1).isKeyword("template")) {

int startPos = pos;

while (!isAtEnd() && !checkPunct(";")) advance();

matchPunct(";");

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(),

tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

if (checkKeyword("extern") && pos + 1 < tokens.size()

&& tokens.get(pos + 1).type() == CppLexerTokenType.STRING_LITERAL) {

advance(); advance(); // consume extern "C++"

if (checkPunct("{")) {

// extern "C++" { ... } block -- parse contents as top-level items

advance(); // consume {

List<TopLevelItem> result = new ArrayList<>();

while (!isAtEnd() && !checkPunct("}")) {

List<CppLexerToken> innerComments = consumeLeadingComments();

if (checkPunct("}")) break;

result.addAll(parseTopLevelItem(innerComments));

}

matchPunct("}");

return result;

} else {

// extern "C++" single-declaration

return parseTopLevelItem(leadingComments);

}

}

if (matchKeyword("template")) {

// Explicit template instantiation: "template class Foo<int>;" (no < after template)

if (!checkOp("<")) {

int startPos = pos - 1;

while (!isAtEnd() && !checkPunct(";")) advance();

matchPunct(";");

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); }

raw.append(";");

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

templateParams = parseTemplateParamList();

consumeLeadingComments();

// C++20 concept definition: "template<typename T> concept Foo = ..."

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("concept")) {

int startPos = pos - templateParams.size() - 3; // approximate; use simpler reconstruction

// Build the full concept declaration text

StringBuilder raw = new StringBuilder("template<");

for (int i = 0; i < templateParams.size(); i++) {

if (i > 0) raw.append(", ");

raw.append(templateParams.get(i));

}

raw.append("> ");

// Capture from "concept" through ";"

int conceptStart = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";");

for (int i = conceptStart; i < pos - 1; i++) {

if (i > conceptStart) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(conceptStart).line(), tokens.get(conceptStart).col(), leadingComments));

}

// C++20 "requires" clause after template params.

// "requires" is lexed as IDENTIFIER (not in the keyword set).

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) {

advance(); // 'requires'

int depth = 0;

while (!isAtEnd()) {

if (checkPunct("(") || checkPunct("[")) { depth++; advance(); continue; }

if (checkPunct(")") || checkPunct("]")) { depth--; advance(); continue; }

if (checkOp("<")) { depth++; advance(); continue; }

if (checkOp(">")) { depth--; advance(); continue; }

if (checkOp(">>")) { depth -= 2; advance(); continue; }

if (checkPunct(";")) break;

if (checkPunct("{")) {

// Braced block in requires: consume it entirely

int _bd = 0;

while (!isAtEnd()) {

if (checkPunct("{")) _bd++;

else if (checkPunct("}")) { _bd--; if (_bd == 0) { advance(); break; } }

advance();

}

continue;

}

if (depth == 0) {

if (checkKeyword("void") || checkKeyword("auto") ||

checkKeyword("int") || checkKeyword("float") ||

checkKeyword("bool") || checkKeyword("char") ||

checkKeyword("double")|| checkKeyword("long") ||

checkKeyword("class") || checkKeyword("struct") ||

checkKeyword("const") || checkKeyword("static") ||

checkKeyword("inline")|| checkKeyword("virtual") ||

checkKeyword("explicit")|| checkKeyword("friend")) {

break;

}

if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size()) {

CppLexerToken next = tokens.get(pos + 1);

if (next.type() == CppLexerTokenType.IDENTIFIER) break;

}

}

advance();

}

}

// C++20 concept used as a constraint directly in the template head:

// "template<Concept T>" -- Concept is an identifier, not typename/class.

// This is handled in parseTemplateParamName already.

}

if (checkKeyword("class") || checkKeyword("struct") || checkKeyword("union")) {

// Partial specialization: "template<typename T> struct Foo<T*> { ... }"

// After parsing the class/struct, check for a specialization arg list.

// BUT: "struct TypeName funcName(...)" is a function with elaborated return type

if (pos + 2 < tokens.size()

&& tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER

&& tokens.get(pos + 2).type() == CppLexerTokenType.IDENTIFIER

&& !tokens.get(pos + 1).isKeyword("class") && !tokens.get(pos + 1).isKeyword("struct")) {

return parseFunctionOrVariable(leadingComments, templateParams, true);

}

return parseTypeDef(leadingComments, templateParams);

}

if (checkKeyword("enum")) {

return List.of(parseEnumDecl(leadingComments));

}

if (checkKeyword("namespace") || (checkKeyword("inline") && pos + 1 < tokens.size() && tokens.get(pos + 1).isKeyword("namespace"))) {

return List.of(parseNamespaceDecl(leadingComments));

}

// typedef -- emit verbatim as-is (covers function-pointer typedefs, etc.)

if (checkKeyword("typedef")) {

int startPos = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); }

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

if (checkKeyword("typedef")) {

int startPos = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";");

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

if (checkKeyword("typedef")) {

int startPos = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";");

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

// static_assert at top level: emit verbatim as PreprocessorLine

if (checkKeyword("static_assert")) {

int startPos = pos; advance();

expectPunct("("); int _d=1;

while (!isAtEnd() && _d > 0) {

if (checkPunct("(")) _d++; else if (checkPunct(")")) _d--;

advance();

}

matchPunct(";");

StringBuilder raw = new StringBuilder();

for (int i = startPos; i < pos; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); }

if (!raw.toString().endsWith(";")) raw.append(";");

CppLexerToken t0 = tokens.get(startPos);

return List.of(new TopLevelStatement(

new ExprStatement(new Identifier(raw.toString(), t0.line(), t0.col(), List.of()),

t0.line(), t0.col(), List.of()),

t0.line(), t0.col(), leadingComments));

}

if (checkKeyword("using") && tokens.get(pos + 1).isKeyword("namespace")) {

return List.of(parseUsingNamespaceDecl(leadingComments));

}

// using T = Type; -- type alias. Consume to ';' and emit as-is.

// template<typename T> using Alias = ...; -- template alias, prepend template params.

if (checkKeyword("using")) {

int startPos = pos;

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); }

StringBuilder raw = new StringBuilder();

// Prepend template params if this is a template alias

if (!templateParams.isEmpty()) {

raw.append("template<");

for (int ti = 0; ti < templateParams.size(); ti++) {

if (ti > 0) raw.append(", ");

raw.append(templateParams.get(ti));

}

raw.append("> ");

}

for (int i = startPos; i < pos - 1; i++) {

if (i > startPos) raw.append(' ');

raw.append(tokens.get(i).text());

}

raw.append(';');

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

// Destructor dispatch: "~Name()", "virtual ~Name()", "constexpr virtual ~Name()", "constexpr ~Name()"

if (checkOp("~")

|| (checkKeyword("virtual") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("~"))

|| (checkKeyword("constexpr") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("~"))

|| (checkKeyword("constexpr") && pos + 1 < tokens.size() && tokens.get(pos + 1).isKeyword("virtual")

&& pos + 2 < tokens.size() && tokens.get(pos + 2).isOp("~"))) {

return List.of(parseFunctionOrConstructorOrDestructor(leadingComments, templateParams));

}

// Global structured binding: "auto [a, b] = expr;"

if (isStructuredBindingStart()) {

Statement sb2 = parseStructuredBinding(leadingComments);

return List.of(new TopLevelStatement(sb2, sb2.line(), sb2.col(), leadingComments));

}

// Deduction guide: "Wrapper(const char*) -> Wrapper<std::string>;"

if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) {

int scan = pos + 2; int depth = 1;

while (scan < tokens.size() && depth > 0) {

if (tokens.get(scan).isPunct("(")) depth++;

else if (tokens.get(scan).isPunct(")")) depth--;

scan++;

}

if (scan < tokens.size() && tokens.get(scan).isPunct("->")) {

int startPos = pos;

while (!isAtEnd() && !checkPunct(";")) advance();

matchPunct(";");

// Reconstruct: include template params prefix if present

StringBuilder raw = new StringBuilder();

if (!templateParams.isEmpty()) {

raw.append("template<");

for (int i = 0; i < templateParams.size(); i++) {

if (i > 0) raw.append(", ");

raw.append(templateParams.get(i));

}

raw.append("> ");

}

for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); }

raw.append(";");

return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments));

}

}

// Bare statement at top level: control-flow keywords can only be statements,

// never declarations. Route directly to statement parsing (static-mode sketches).

if (checkKeyword("for") || checkKeyword("while") || checkKeyword("do") ||

checkKeyword("if") || checkKeyword("switch") || checkKeyword("return") ||

checkKeyword("break") || checkKeyword("continue") ||

checkKeyword("try") || checkKeyword("throw")) {

CppLexerToken start = peek();

Statement stmt = parseStatement(leadingComments);

return List.of(new TopLevelStatement(stmt, start.line(), start.col(), leadingComments));

}

return parseFunctionOrVariable(leadingComments, templateParams, true);

}

/** "template<typename T, typename U, ...>" -- returns just the param names. */

private List<String> parseTemplateParamList() {

// "template" already consumed by the caller; just consume "<...>"

if (checkKeyword("template")) advance(); // in case called with template still present

expectOp("<");

List<String> params = new ArrayList<>();

// template<> is a valid explicit full specialization marker -- emit as empty list

if (!checkOp(">")) {

params.add(parseTemplateParamText());

while (matchPunct(",")) {

params.add(parseTemplateParamText());

}

}

if (checkOp(">>")) {

splitTrailingShiftIntoTwoCloseAngles();

}

expectOp(">");

return params;

}

/**

* Parses one template parameter and returns its FULL TEXT as a String,

* preserving everything needed for CodeGen to emit it verbatim:

* "typename T"

* "typename... Args"

* "class T"

* "template<typename> class Container"

* "auto V"

* "int N = sizeof(T)"

* "typename std::enable_if<std::is_integral<T>::value, int>::type = 0"

* "Numeric T" (concept-constrained)

*

* The AST stores List<String> for template params. Previously it stored only

* the param name, which lost all type/variadic/default information. Now it

* stores the full text so CodeGen can round-trip it as "template<FULL_TEXT>".

*/

private String parseTemplateParamText() {

int startPos = pos;

// We'll capture tokens until we hit ',' or '>' or '>>' at depth 0.

// Complex cases (template-template, SFINAE) are handled by parseTemplateParamName

// for semantic purposes, but here we just need the text.

parseTemplateParamName(); // advance pos past the parameter

int endPos = pos;

// Reconstruct the text from the tokens we consumed

StringBuilder sb = new StringBuilder();

for (int i = startPos; i < endPos; i++) {

if (i > startPos) {

CppLexerToken prev = tokens.get(i - 1);

CppLexerToken cur = tokens.get(i);

// Add spacing between tokens where needed

if (needsSpace(prev, cur)) sb.append(' ');

}

sb.append(tokens.get(i).text());

}

return sb.toString();

}

/** Whether two adjacent tokens need a space between them in output. */

private boolean needsSpace(CppLexerToken prev, CppLexerToken cur) {

String p = prev.text(), c = cur.text();

// No space needed before/after punctuation pairs

if (p.equals("<") || p.equals(">") || p.equals(">>") || p.equals("*")

|| p.equals("&") || p.equals("&&") || p.equals("::")) return false;

if (c.equals(">") || c.equals(">>") || c.equals("<") || c.equals("::")

|| c.equals("(") || c.equals(")") || c.equals(",")) return false;

// Space between two word tokens (keywords, identifiers)

if ((prev.type() == CppLexerTokenType.KEYWORD || prev.type() == CppLexerTokenType.IDENTIFIER)

&& (cur.type() == CppLexerTokenType.KEYWORD || cur.type() == CppLexerTokenType.IDENTIFIER)) return true;

// Space before/after ellipsis in most contexts

if (p.equals("...") || c.equals("...")) return true;

// Space after "=" for defaults

if (p.equals("=") || c.equals("=")) return true;

return false;

}

/**

* Parses one template parameter in any of the C++ forms:

*

* typename T -- type param

* class T -- type param (alternate spelling)

* typename... Args -- variadic type param pack

* class... Args -- variadic type param pack

* template<typename> class C -- template-template param

* template<typename> class... C -- variadic template-template param

* int N -- non-type param

* auto V -- non-type param (C++17 auto NTTP)

* typename T = Default -- type param with default

* int N = 42 -- non-type param with default

* typename std::enable_if<...>::type = 0 -- SFINAE sink param

* Concept T -- constrained type param (C++20)

*

* Returns just the name string (or a descriptive placeholder for complex

* params the AST doesn't fully model -- consistent with existing practice

* where template params are stored as strings, not as typed AST nodes).

*/

private String parseTemplateParamName() {

// --- template-template parameter: "template<...> class Name" ---

if (checkKeyword("template")) {

parseTemplateParamList(); // consume the nested <...> (recursive)

matchKeyword("class");

matchKeyword("typename");

// variadic template-template: "template<typename> class... C"

if (checkPunct("...")) advance();

if (check(CppLexerTokenType.IDENTIFIER)) return advance().text();

return "<template-template>";

}

// --- concept-constrained type parameter: "Numeric T" or "Numeric typename" ---

// Also handles concept-constrained template-template params:

// "template<Concept typename> class C" -- the inner param has

// concept name then "typename" keyword (not an identifier).

if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size()

&& (tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER

|| tokens.get(pos + 1).isKeyword("typename")

|| tokens.get(pos + 1).isKeyword("class"))

&& pos + 2 < tokens.size()

&& (tokens.get(pos + 2).isOp(">") || tokens.get(pos + 2).isOp(">>")

|| tokens.get(pos + 2).isPunct(",")

|| tokens.get(pos + 2).isPunct("...")

|| tokens.get(pos + 2).isOp("="))) {

advance(); // concept name

if (checkPunct("...")) advance();

// param name may be a keyword (typename/class) or identifier or absent

String name;

if (check(CppLexerTokenType.IDENTIFIER)) name = advance().text();

else if (checkKeyword("typename") || checkKeyword("class")) { advance(); name = "<anon>"; }

else name = "<anon>";

if (matchOp("=")) parseTemplateDefaultValue();

return name;

}

// --- typename / class type parameter ---

if (checkKeyword("typename") || checkKeyword("class")) {

advance(); // consume typename/class

// variadic pack: "typename... Args"

if (checkPunct("...")) advance();

// SFINAE sink: "typename std::enable_if<cond, T>::type = 0"

// After typename, if what follows is a qualified name (has :: or

// template args), it's a non-type SFINAE param, not a plain

// type-param name. Parse it as a full type and grab any default.

if (check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD)) {

boolean isSfinaeType = checkPunct("::")

|| (pos + 1 < tokens.size() && (

tokens.get(pos + 1).isPunct("::") ||

tokens.get(pos + 1).isOp("<")));

if (isSfinaeType) {

try { parseTypeRef(); } catch (ParseException ignored) {}

if (matchOp("=")) parseTemplateDefaultValue();

return "<sfinae>";

}

String name = advance().text();

if (matchOp("=")) parseTemplateDefaultValue();

return name;

}

// Anonymous typename with default: "typename = std::enable_if<true>"

if (matchOp("=")) parseTemplateDefaultValue();

return "<anon>";

}

// --- auto NTTP: "auto V" or "auto... Vs" ---

if (checkKeyword("auto")) {

advance();

if (checkPunct("...")) advance();

String name = check(CppLexerTokenType.IDENTIFIER) ? advance().text() : "<anon>";

if (matchOp("=")) parseTemplateDefaultValue();

return name;

}

// --- Non-type parameter: "int N", "std::size_t N", "const char* S",

// or member-pointer NTTP: "int T::* Field"

TypeRef type = parseTypeRef();

if (checkPunct("...")) advance();

String name;

if (check(CppLexerTokenType.IDENTIFIER)) {

name = advance().text();

// Member pointer NTTP: "int T::* Field" -- consume ::* and actual name

if (checkPunct("::") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("*")) {

advance(); advance(); // :: and *

if (check(CppLexerTokenType.IDENTIFIER)) name = advance().text();

}

} else if (checkPunct("(")) {

// Member function pointer NTTP: "int (T::* Method)(int) const"

// or pointer-to-array NTTP: "int (*Arr)[N]"

advance(); // consume (

// consume optional Class:: qualifier

if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size()

&& tokens.get(pos + 1).isPunct("::")) {

advance(); advance(); // Class ::

}

matchOp("*"); matchOp("&"); // * or &

name = check(CppLexerTokenType.IDENTIFIER) ? advance().text() : "<nttp-ptr>";

expectPunct(")");

// consume trailing (params) and optional const

if (checkPunct("(")) {

advance(); int d = 1;

while (!isAtEnd() && d > 0) {

if (checkPunct("(")) d++; else if (checkPunct(")")) d--;

advance();

}

matchKeyword("const");

}

// consume trailing [dims] for pointer-to-array

while (checkPunct("[")) {

advance();

if (!checkPunct("]")) parseExpr();

expectPunct("]");

}

} else {

name = type instanceof NamedType nt ? nt.baseName() : "<nttp>";

}

// optional default value: "int N = 42" or "::type = 0"

if (matchOp("=")) parseTemplateDefaultValue();

return name;

}

/**

* Parses a class/struct declaration: "class Name [: public Base1, ...] { members };".

* "public"/"private"/"protected" access specifiers within the body are

* consumed and discarded (not modeled in the AST -- not yet confirmed

* necessary by any semantic pass; if member visibility ever matters,

* this is the place to start tracking it).

*/

private List<TopLevelItem> parseTypeDef(List<CppLexerToken> leadingComments, List<String> templateParams) {

CppLexerToken start = peek();

String kind = checkKeyword("class") ? "class" : checkKeyword("union") ? "union" : "struct";

advance();

// alignas specifier: "struct alignas(64) CacheLine"

if (checkKeyword("alignas") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) {

advance(); advance(); int _ad=1;

while (!isAtEnd() && _ad > 0) { if (checkPunct("(")) _ad++; else if (checkPunct(")")) _ad--; advance(); }

}

// Also consume [[attributes]] before the name

consumeAttributes();

// GCC __attribute__((...)): "struct __attribute__((packed)) Name"

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("__attribute__")) {

advance();

if (checkPunct("(")) { advance(); int _ad=1; while(!isAtEnd()&&_ad>0){if(checkPunct("("))_ad++;else if(checkPunct(")"))_ad--;advance();} }

}

// MSVC __declspec(...): "struct __declspec(align(16)) Name"

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("__declspec")) {

advance();

if (checkPunct("(")) { advance(); int _ad=1; while(!isAtEnd()&&_ad>0){if(checkPunct("("))_ad++;else if(checkPunct(")"))_ad--;advance();} }

}

// Anonymous struct/class: "struct { ... } var;" -- generate synthetic name

String name;

if (checkPunct("{")) {

name = "_Anon_" + start.line() + "_" + start.col();

} else {

name = expectIdentifier().text();

}

// Partial or explicit specialization: "template<typename T> struct Foo<T*>"

// or "template<> struct TypeName<int>" -- capture the specialization arg

// text and fold it into the name so CodeGen emits "TypeName<int>" correctly.

if (checkOp("<")) {

// Specialization args: "Foo<T, N, false>" -- scan to matching > manually

int argStart = pos;

advance(); // consume <

int depth = 1;

while (!isAtEnd() && depth > 0) {

if (checkOp("<")) { depth++; advance(); }

else if (checkOp(">")) { depth--; advance(); }

else if (checkOp(">>")) {

depth -= 2;

splitTrailingShiftIntoTwoCloseAngles();

advance(); // consume first >

if (depth <= 0) { advance(); break; } // consume second > and exit

advance(); // consume second > when still inside

}

else advance();

}

int argEnd = pos;

// Reconstruct the specialization suffix verbatim from tokens

StringBuilder spec = new StringBuilder();

for (int i = argStart; i < argEnd; i++) spec.append(tokens.get(i).text());

name = name + spec.toString();

}

// Forward declaration: "struct Node;" or "class Container;" with no body.

// Also handles "template<typename T = int> class Container;" with default args.

if (checkPunct(";")) {

advance();

return List.of(new TypeDef(kind, name, templateParams, List.of(), List.of(),

start.line(), start.col(), leadingComments));

}

List<String> baseClasses = new ArrayList<>();

if (matchPunct(":")) {

baseClasses.add(parseBaseClassEntry());

while (matchPunct(",")) {

baseClasses.add(parseBaseClassEntry());

}

}

expectPunct("{");

List<TopLevelItem> members = new ArrayList<>();

while (!checkPunct("}")) {

if (isAtEnd()) throw error("unexpected end of input inside class/struct body for '" + name + "'");

List<CppLexerToken> comments = consumeLeadingComments();

if (checkPunct("}")) break;

if (matchKeyword("public") || matchKeyword("private") || matchKeyword("protected")) {

expectPunct(":");

continue;

}

members.addAll(parseClassMember(comments, name));

}

expectPunct("}");

// The trailing ';' after a class/struct body is standard C++, but

// confirmed OPTIONAL in real .pde input -- Scrollbar.pde's

// "class HScrollbar { ... }" has no trailing ';' at all. Processing's

// own preprocessing tolerates this; this parser does too rather than

// hard-requiring strict C++ grammar here.

TypeDef typeDef = new TypeDef(kind, name, templateParams, baseClasses, members, start.line(), start.col(), leadingComments);

// Trailing declarators: "class Foo { ... } *ptr;" or "} a, b, *c;"

// Only enter if there is NO trailing semicolon yet.

boolean nextIsDeclarator = checkOp("*") || checkPunct("*")

|| peek().type() == CppLexerTokenType.IDENTIFIER;

if (!matchPunct(";") && !isAtEnd() && !checkPunct("}") && nextIsDeclarator) {

List<TopLevelItem> result = new ArrayList<>();

result.add(typeDef);

do {

int ptrDepth = 0;

while (matchOp("*") || matchPunct("*")) ptrDepth++;

if (peek().type() != CppLexerTokenType.IDENTIFIER) break;

CppLexerToken varTok = advance();

Expr init = null;

if (matchPunct("=")) init = parseExpr();

TypeRef varType = new NamedType(name, List.of(), ptrDepth, false, false, false);

result.add(new VariableDecl(varType, varTok.text(), List.of(), init,

false, false, varTok.line(), varTok.col(), List.of()));

} while (matchPunct(","));

expectPunct(";");

return result;

}

return List.of(typeDef);

}

/** "public BaseName" / "private BaseName" / "protected BaseName" / bare "BaseName". */

private String parseBaseClassEntry() {

boolean isVirtualBase = matchKeyword("virtual");

matchKeyword("public");

if (!checkKeyword("public")) {

matchKeyword("private");

matchKeyword("protected");

}

if (!isVirtualBase) isVirtualBase = matchKeyword("virtual");

String name = parseQualifiedTypeName();

// Consume template args on base class name: "Base<Derived>" (CRTP),

// "std::enable_shared_from_this<T>", etc.

if (checkOp("<") && looksLikeTemplateArgList()) {

StringBuilder sb = new StringBuilder(name);

sb.append('<');

advance(); // '<'

int depth = 1;

while (!isAtEnd() && depth > 0) {

if (checkOp("<")) depth++;

else if (checkOp(">")) { depth--; if (depth == 0) break; }

else if (checkOp(">>")) { depth -= 2; if (depth <= 0) { splitTrailingShiftIntoTwoCloseAngles(); break; } }

sb.append(peek().text());

advance();

}

expectOp(">");

sb.append('>');

name = sb.toString();

}

boolean isPackExpansion = matchPunct("..."); // pack expansion in base class list: "Bases..."

String suffix = isPackExpansion ? "..." : "";

return isVirtualBase ? "virtual " + name + suffix : name + suffix;

}

/**

* A class member is almost always parseTopLevelItem's domain (field decl,

* method, nested type) -- but constructors/destructors named after the

* enclosing class need special handling parseTopLevelItem's generic

* function/variable dispatch can't do alone (no return type at all,

* possible initializer list), so this wraps that in enclosing-class

* context.

*/

private List<TopLevelItem> parseClassMember(List<CppLexerToken> leadingComments, String enclosingClassName) {

if (check(CppLexerTokenType.PREPROCESSOR_DIRECTIVE)) {

CppLexerToken t = advance();

return List.of(new PreprocessorLine(t.text(), t.line(), t.col(), leadingComments));

}

consumeAttributes();

// friend declarations: "friend class Foo;" or "friend void func(...);"

// Consume entirely -- friend declarations don't produce AST members we need.

if (checkKeyword("friend")) {

advance(); // consume 'friend'

// Friend function with a body: "friend T operator*(...) { ... }"

// Must be parsed as a real function, not consumed to ';'.

// Detect by scanning ahead: if we find '{' before ';', it has a body.

int scan = pos;

int depth = 0;

boolean hasBody = false;

while (scan < tokens.size()) {

String txt = tokens.get(scan).text();

if (txt.equals("{")) { if (depth++ == 0) { hasBody = true; break; } }

else if (txt.equals("}")) { if (--depth == 0) break; }

else if (txt.equals(";") && depth == 0) break;

scan++;

}

if (hasBody) {

// Friend function WITH a body is a free function defined in-class

// but semantically at namespace scope. Return it as a TopLevelItem

// to be hoisted out of the class body.

// We must NOT emit it as a class member since free functions with 2+

// parameters can't be member functions.

List<TopLevelItem> items = parseFunctionOrVariable(leadingComments, List.of(), false);

return items; // will be added to class members and hoisted by ClassHoister

}

// Friend declaration (no body) -- consume to ';'

{ int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); }

return List.of();

}

List<String> templateParams = List.of();

if (checkKeyword("template")) {

templateParams = parseTemplateParamList();

consumeLeadingComments();

// Trailing requires clause on template member: "template<T>\n requires Concept<T>"

if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) {

advance(); // consume requires

if (checkPunct("(")) { advance(); int _d=1; while(!isAtEnd()&&_d>0){if(checkPunct("("))_d++;else if(checkPunct(")"))_d--;advance();} }

else { int _d=0; while(!isAtEnd()){if(checkOp("<"))_d++;else if(checkOp(">")&&_d>0)_d--;else if(_d==0&&(checkPunct("{;")||checkPunct(";")|| checkKeyword("auto")||checkKeyword("void")||checkKeyword("bool")||checkKeyword("int")||checkKeyword("float")||checkKeyword("const")||checkKeyword("inline")||checkKeyword("static")||checkKeyword("virtual")||checkKeyword("constexpr")||checkKeyword("explicit")||checkKeyword("operator")))break;advance();} }

consumeLeadingComments();

}

}

if (checkKeyword("class") || checkKeyword("struct") || checkKeyword("union")) {

// Anonymous struct: "struct { float x, y; } position;"

if (pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("{")) {

int anonStart = pos;

advance(); // consume struct/class

int bd = 0;

while (!isAtEnd()) {

if (checkPunct("{")) { bd++; advance(); }