/*

* Cppcheck - A tool for static C/C++ code analysis

* Copyright (C) 2007-2026 Cppcheck team.

*

* This program is free software: you can redistribute it and/or modify

* it under the terms of the GNU General Public License as published by

* the Free Software Foundation, either version 3 of the License, or

* (at your option) any later version.

*

* This program is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

* GNU General Public License for more details.

*

* You should have received a copy of the GNU General Public License

* along with this program. If not, see <http://www.gnu.org/licenses/>.

*/

#include "errortypes.h"

#include "fixture.h"

#include "helpers.h"

#include "path.h"

#include "platform.h"

#include "preprocessor.h" // usually tests here should not use preprocessor...

#include "settings.h"

#include "standards.h"

#include "token.h"

#include "tokenize.h"

#include "tokenlist.h"

#include <cstdint>

#include <cstring>

#include <list>

#include <set>

#include <sstream>

#include <string>

#include <utility>

#include <vector>

#include <simplecpp.h>

class ErrorLogger;

class TestTokenizer : public TestFixture {

public:

TestTokenizer() : TestFixture("TestTokenizer") {}

private:

const Settings settings0 = settingsBuilder().library("qt.cfg").build();

const Settings settings1 = settingsBuilder().library("qt.cfg").library("std.cfg").debugwarnings().build();

const Settings settings2 = settingsBuilder(settings1).cpp(Standards::CPP11).c(Standards::C11).build();

const Settings settings3 = settingsBuilder(settings0).c(Standards::C89).cpp(Standards::CPP03).build();

const Settings settings_windows = settingsBuilder().library("windows.cfg").debugwarnings().cpp(Standards::CPP11).build();

void run() override {

mNewTemplate = true;

TEST_CASE(tokenize1);

TEST_CASE(tokenize2);

TEST_CASE(tokenize4);

TEST_CASE(tokenize5);

TEST_CASE(tokenize7);

TEST_CASE(tokenize8);

TEST_CASE(tokenize9);

TEST_CASE(tokenize11);

TEST_CASE(tokenize13); // bailout if the code contains "@" - that is not handled well.

TEST_CASE(tokenize14); // tokenize "0X10" => 16

TEST_CASE(tokenizeHexWithSuffix); // tokenize 0xFFFFFFul

TEST_CASE(tokenize15); // tokenize ".123"

TEST_CASE(tokenize17); // #2759

TEST_CASE(tokenize18); // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

TEST_CASE(tokenize19); // #3006 (segmentation fault)

TEST_CASE(tokenize21); // tokenize 0x0E-7

TEST_CASE(tokenize22); // special marker $ from preprocessor

TEST_CASE(tokenize25); // #4239 (segmentation fault)

TEST_CASE(tokenize26); // #4245 (segmentation fault)

TEST_CASE(tokenize27); // #4525 (segmentation fault)

TEST_CASE(tokenize31); // #3503 (Wrong handling of member function taking function pointer as argument)

TEST_CASE(tokenize32); // #5884 (fsanitize=undefined: left shift of negative value -10000 in lib/templatesimplifier.cpp:852:46)

TEST_CASE(tokenize33); // #5780 Various crashes on valid template code

TEST_CASE(tokenize34); // #8031

TEST_CASE(tokenize35); // #8361

TEST_CASE(tokenize36); // #8436

TEST_CASE(tokenize37); // #8550

TEST_CASE(tokenize38); // #9569

TEST_CASE(tokenize39); // #9771

TEST_CASE(tokenize40); // #13181

TEST_CASE(tokenize41); // #13847

TEST_CASE(tokenize42); // #13861

TEST_CASE(validate);

TEST_CASE(objectiveC); // Syntax error should be written for objective C/C++ code.

TEST_CASE(syntax_case_default);

TEST_CASE(removePragma);

TEST_CASE(foreach); // #3690

TEST_CASE(ifconstexpr);

TEST_CASE(combineOperators);

TEST_CASE(concatenateNegativeNumber);

TEST_CASE(longtok);

TEST_CASE(simplifyHeadersAndUnusedTemplates1);

TEST_CASE(simplifyHeadersAndUnusedTemplates2);

TEST_CASE(simplifyAt);

TEST_CASE(inlineasm);

TEST_CASE(simplifyAsm2); // #4725 (writing asm() around "^{}")

TEST_CASE(ifAddBraces1);

TEST_CASE(ifAddBraces2);

TEST_CASE(ifAddBraces3);

TEST_CASE(ifAddBraces4);

TEST_CASE(ifAddBraces5);

TEST_CASE(ifAddBraces7);

TEST_CASE(ifAddBraces9);

TEST_CASE(ifAddBraces11);

TEST_CASE(ifAddBraces12);

TEST_CASE(ifAddBraces13);

TEST_CASE(ifAddBraces15); // #2616 - unknown macro before if

TEST_CASE(ifAddBraces16);

TEST_CASE(ifAddBraces17); // '} else' should be in the same line

TEST_CASE(ifAddBraces18); // #3424 - if if { } else else

TEST_CASE(ifAddBraces19); // #3928 - if for if else

TEST_CASE(ifAddBraces20); // #5012 - syntax error 'else }'

TEST_CASE(ifAddBracesLabels); // #5332 - if (x) label: {} ..

TEST_CASE(switchAddBracesLabels);

TEST_CASE(whileAddBraces);

TEST_CASE(whileAddBracesLabels);

TEST_CASE(doWhileAddBraces);

TEST_CASE(doWhileAddBracesLabels);

TEST_CASE(forAddBraces1);

TEST_CASE(forAddBraces2); // #5088

TEST_CASE(forAddBracesLabels);

TEST_CASE(simplifyExternC);

TEST_CASE(simplifyKeyword); // #5842 - remove C99 static keyword between []

TEST_CASE(isOneNumber);

TEST_CASE(simplifyFunctionParameters);

TEST_CASE(simplifyFunctionParameters1); // #3721

TEST_CASE(simplifyFunctionParameters2); // #4430

TEST_CASE(simplifyFunctionParameters3); // #4436

TEST_CASE(simplifyFunctionParameters4); // #9421

TEST_CASE(simplifyFunctionParametersMultiTemplate);

TEST_CASE(simplifyFunctionParametersErrors);

TEST_CASE(simplifyFunctionTryCatch);

TEST_CASE(removeParentheses1); // Ticket #61

TEST_CASE(removeParentheses3);

TEST_CASE(removeParentheses4); // Ticket #390

TEST_CASE(removeParentheses5); // Ticket #392

TEST_CASE(removeParentheses6);

TEST_CASE(removeParentheses7);

TEST_CASE(removeParentheses8); // Ticket #1865

TEST_CASE(removeParentheses9); // Ticket #1962

TEST_CASE(removeParentheses10); // Ticket #2320

TEST_CASE(removeParentheses11); // Ticket #2505

TEST_CASE(removeParentheses12); // Ticket #2760 ',(b)='

TEST_CASE(removeParentheses13);

TEST_CASE(removeParentheses14); // Ticket #3309

TEST_CASE(removeParentheses15); // Ticket #4142

TEST_CASE(removeParentheses16); // Ticket #4423 '*(x.y)='

TEST_CASE(removeParentheses17); // Don't remove parentheses in 'a ? b : (c>0 ? d : e);'

TEST_CASE(removeParentheses18); // 'float(*a)[2]' => 'float *a[2]'

TEST_CASE(removeParentheses19); // ((typeof(x) *)0)

TEST_CASE(removeParentheses20); // Ticket #5479: a<b<int>>(2);

TEST_CASE(removeParentheses21); // Don't "simplify" casts

TEST_CASE(removeParentheses22);

TEST_CASE(removeParentheses23); // Ticket #6103 - Infinite loop upon valid input

TEST_CASE(removeParentheses24); // Ticket #7040

TEST_CASE(removeParentheses25); // daca@home - a=(b,c)

TEST_CASE(removeParentheses26); // Ticket #8875 a[0](0)

TEST_CASE(removeParentheses27);

TEST_CASE(removeParentheses28); // #12164 - don't remove parentheses in '(expr1) ? (expr2) : (expr3);'

TEST_CASE(removeParantheses29); // #13735

TEST_CASE(removeParentheses30);

TEST_CASE(tokenize_double);

TEST_CASE(tokenize_strings);

TEST_CASE(simplifyStructDecl);

TEST_CASE(vardecl1);

TEST_CASE(vardecl2);

TEST_CASE(vardecl3);

TEST_CASE(vardecl4);

TEST_CASE(vardecl5); // #7048

TEST_CASE(vardec_static);

TEST_CASE(vardecl6);

TEST_CASE(vardecl7);

TEST_CASE(vardecl8);

TEST_CASE(vardecl9);

TEST_CASE(vardecl10);

TEST_CASE(vardecl11);

TEST_CASE(vardecl12);

TEST_CASE(vardecl13);

TEST_CASE(vardecl14);

TEST_CASE(vardecl15);

TEST_CASE(vardecl16);

TEST_CASE(vardecl17);

TEST_CASE(vardecl18);

TEST_CASE(vardecl19);

TEST_CASE(vardecl20); // #3700 - register const int H = 0;

TEST_CASE(vardecl21); // #4042 - a::b const *p = 0;

TEST_CASE(vardecl22); // #4211 - segmentation fault

TEST_CASE(vardecl23); // #4276 - segmentation fault

TEST_CASE(vardecl24); // #4187 - variable declaration within lambda function

TEST_CASE(vardecl25); // #4799 - segmentation fault

TEST_CASE(vardecl26); // #5907 - incorrect handling of extern declarations

TEST_CASE(vardecl27); // #7850 - crash on valid C code

TEST_CASE(vardecl28);

TEST_CASE(vardecl29); // #9282

TEST_CASE(vardecl30);

TEST_CASE(vardecl31); // function pointer init

TEST_CASE(vardecl32);

TEST_CASE(vardecl33);

TEST_CASE(vardecl34);

TEST_CASE(vardecl_stl_1);

TEST_CASE(vardecl_stl_2);

TEST_CASE(vardecl_stl_3);

TEST_CASE(vardecl_template_1);

TEST_CASE(vardecl_template_2);

TEST_CASE(vardecl_union);

TEST_CASE(vardecl_par); // #2743 - set links if variable type contains parentheses

TEST_CASE(vardecl_par2); // #3912 - set correct links

TEST_CASE(vardecl_par3); // #6556 - Fred x1(a), x2(b);

TEST_CASE(vardecl_class_ref);

TEST_CASE(volatile_variables);

// unsigned i; => unsigned int i;

TEST_CASE(implicitIntConst);

TEST_CASE(implicitIntExtern);

TEST_CASE(implicitIntSigned1);

TEST_CASE(implicitIntUnsigned1);

TEST_CASE(implicitIntUnsigned2);

TEST_CASE(implicitIntUnsigned3); // template arguments

TEST_CASE(simplifyStdType); // #4947, #4950, #4951

TEST_CASE(createLinks);

TEST_CASE(createLinks2);

TEST_CASE(simplifyString);

TEST_CASE(simplifyConst);

TEST_CASE(switchCase);

TEST_CASE(simplifyPointerToStandardType);

TEST_CASE(simplifyFunctionPointers1);

TEST_CASE(simplifyFunctionPointers2);

TEST_CASE(simplifyFunctionPointers3);

TEST_CASE(simplifyFunctionPointers4);

TEST_CASE(simplifyFunctionPointers5);

TEST_CASE(simplifyFunctionPointers6);

TEST_CASE(simplifyFunctionPointers7);

TEST_CASE(simplifyFunctionPointers8); // #7410 - throw

TEST_CASE(simplifyFunctionPointers9); // #6113 - function call with function pointer

TEST_CASE(removedeclspec);

TEST_CASE(removeattribute);

TEST_CASE(functionAttributeBefore1);

TEST_CASE(functionAttributeBefore2);

TEST_CASE(functionAttributeBefore3);

TEST_CASE(functionAttributeBefore4);

TEST_CASE(functionAttributeBefore5); // __declspec(dllexport)

TEST_CASE(functionAttributeAfter1);

TEST_CASE(functionAttributeAfter2);

TEST_CASE(functionAttributeListBefore);

TEST_CASE(functionAttributeListAfter);

TEST_CASE(functionAttributeListAfter2);

TEST_CASE(cppMaybeUnusedBefore);

TEST_CASE(cppMaybeUnusedAfter1);

TEST_CASE(cppMaybeUnusedAfter2);

TEST_CASE(cppMaybeUnusedStructuredBinding);

TEST_CASE(splitTemplateRightAngleBrackets);

TEST_CASE(cpp03template1);

TEST_CASE(cpp0xtemplate1);

TEST_CASE(cpp0xtemplate2);

TEST_CASE(cpp0xtemplate3);

TEST_CASE(cpp0xtemplate4); // Ticket #6181: Mishandled C++11 syntax

TEST_CASE(cpp0xtemplate5); // Ticket #9154 change >> to > >

TEST_CASE(cpp14template); // Ticket #6708

TEST_CASE(arraySize);

TEST_CASE(arraySizeAfterValueFlow);

TEST_CASE(labels);

TEST_CASE(simplifyInitVar);

TEST_CASE(simplifyInitVar2);

TEST_CASE(simplifyInitVar3);

TEST_CASE(simplifyInitVar4);

TEST_CASE(simplifyInitVar5);

TEST_CASE(bitfields1);

TEST_CASE(bitfields2);

TEST_CASE(bitfields3);

TEST_CASE(bitfields4); // ticket #1956

TEST_CASE(bitfields5); // ticket #1956

TEST_CASE(bitfields6); // ticket #2595

TEST_CASE(bitfields7); // ticket #1987

TEST_CASE(bitfields8);

TEST_CASE(bitfields9); // ticket #2706

TEST_CASE(bitfields10);

TEST_CASE(bitfields12); // ticket #3485 (segmentation fault)

TEST_CASE(bitfields13); // ticket #3502 (segmentation fault)

TEST_CASE(bitfields15); // ticket #7747 (enum Foo {A,B}:4;)

TEST_CASE(bitfields16); // Save bitfield bit count

TEST_CASE(bitfields17);

TEST_CASE(bitfields18);

TEST_CASE(bitfields19); // ticket #13733

TEST_CASE(bitfields20);

TEST_CASE(bitfields21);

TEST_CASE(simplifyNamespaceStd);

TEST_CASE(microsoftMemory);

TEST_CASE(microsoftString);

TEST_CASE(borland);

TEST_CASE(simplifySQL);

TEST_CASE(simplifyCAlternativeTokens);

// x = ({ 123; }); => { x = 123; }

TEST_CASE(simplifyCompoundStatements);

TEST_CASE(simplifyOperatorName1);

TEST_CASE(simplifyOperatorName2);

TEST_CASE(simplifyOperatorName3);

TEST_CASE(simplifyOperatorName4);

TEST_CASE(simplifyOperatorName5);

TEST_CASE(simplifyOperatorName6); // ticket #3194

TEST_CASE(simplifyOperatorName7); // ticket #4619

TEST_CASE(simplifyOperatorName8); // ticket #5706

TEST_CASE(simplifyOperatorName9); // ticket #5709 - comma operator not properly tokenized

TEST_CASE(simplifyOperatorName10); // #8746 - using a::operator=

TEST_CASE(simplifyOperatorName11); // #8889

TEST_CASE(simplifyOperatorName12); // #9110

TEST_CASE(simplifyOperatorName13); // user defined literal

TEST_CASE(simplifyOperatorName14); // std::complex operator "" if

TEST_CASE(simplifyOperatorName15); // ticket #9468 syntaxError

TEST_CASE(simplifyOperatorName16); // ticket #9472

TEST_CASE(simplifyOperatorName17);

TEST_CASE(simplifyOperatorName18); // global namespace

TEST_CASE(simplifyOperatorName19);

TEST_CASE(simplifyOperatorName20);

TEST_CASE(simplifyOperatorName21);

TEST_CASE(simplifyOperatorName22);

TEST_CASE(simplifyOperatorName23);

TEST_CASE(simplifyOperatorName24);

TEST_CASE(simplifyOperatorName25);

TEST_CASE(simplifyOperatorName26);

TEST_CASE(simplifyOperatorName27);

TEST_CASE(simplifyOperatorName28);

TEST_CASE(simplifyOperatorName29); // spaceship operator

TEST_CASE(simplifyOperatorName30);

TEST_CASE(simplifyOperatorName31); // #6342

TEST_CASE(simplifyOperatorName32); // #10256

TEST_CASE(simplifyOperatorName33); // #10138

TEST_CASE(simplifyOverloadedOperators1);

TEST_CASE(simplifyOverloadedOperators2); // (*this)(123)

TEST_CASE(simplifyOverloadedOperators3); // #9881 - hang

TEST_CASE(simplifyNullArray);

// Some simple cleanups of unhandled macros in the global scope

TEST_CASE(removeMacrosInGlobalScope);

TEST_CASE(addSemicolonAfterUnknownMacro);

// a = b = 0;

TEST_CASE(multipleAssignment);

TEST_CASE(platformWin);

TEST_CASE(platformWin32A);

TEST_CASE(platformWin32W);

TEST_CASE(platformWin32AStringCat); // ticket #5015

TEST_CASE(platformWin32WStringCat); // ticket #5015

TEST_CASE(platformWinWithNamespace);

TEST_CASE(simplifyStaticConst);

TEST_CASE(simplifyCPPAttribute);

TEST_CASE(simplifyCaseRange);

TEST_CASE(simplifyEmptyNamespaces);

TEST_CASE(prepareTernaryOpForAST);

// AST data

TEST_CASE(astexpr);

TEST_CASE(astexpr2); // limit large expressions

TEST_CASE(astpar);

TEST_CASE(astnewdelete);

TEST_CASE(astbrackets);

TEST_CASE(astunaryop);

TEST_CASE(astfunction);

TEST_CASE(asttemplate);

TEST_CASE(astrequires);

TEST_CASE(astcast);

TEST_CASE(astlambda);

TEST_CASE(astcase);

TEST_CASE(astrefqualifier);

TEST_CASE(astthrowdelete);

TEST_CASE(asttrailingdecltype);

TEST_CASE(astnoexcept);

TEST_CASE(astvardecl);

TEST_CASE(astnewscoped);

TEST_CASE(astdecltypescope);

TEST_CASE(astdesignatedinit);

TEST_CASE(astrvaluedecl);

TEST_CASE(astorkeyword);

TEST_CASE(astenumdecl);

TEST_CASE(astcompound);

TEST_CASE(startOfExecutableScope);

TEST_CASE(removeMacroInClassDef); // #6058

TEST_CASE(sizeofAddParentheses);

TEST_CASE(reportUnknownMacros);

// Make sure the Tokenizer::findGarbageCode() does not have false positives

// The TestGarbage ensures that there are true positives

TEST_CASE(findGarbageCode);

TEST_CASE(checkEnableIf);

TEST_CASE(checkTemplates);

TEST_CASE(checkNamespaces);

TEST_CASE(checkLambdas);

TEST_CASE(checkIfCppCast);

TEST_CASE(checkRefQualifiers);

TEST_CASE(checkConditionBlock);

TEST_CASE(checkUnknownCircularVar);

TEST_CASE(checkRequires);

// #9052

TEST_CASE(noCrash1);

TEST_CASE(noCrash2);

TEST_CASE(noCrash3);

TEST_CASE(noCrash4);

TEST_CASE(noCrash5); // #10603

TEST_CASE(noCrash6); // #10212

TEST_CASE(noCrash7);

// --check-config

TEST_CASE(checkConfiguration);

TEST_CASE(unknownMacroBeforeReturn);

TEST_CASE(cppKeywordInCSource);

TEST_CASE(cppcast);

TEST_CASE(checkHeader1);

TEST_CASE(removeExtraTemplateKeywords);

TEST_CASE(removeAlignas1);

TEST_CASE(removeAlignas2); // Do not remove alignof in the same way

TEST_CASE(removeAlignas3); // remove alignas in C11 code

TEST_CASE(dumpAlignas);

TEST_CASE(simplifyCoroutines);

TEST_CASE(simplifySpaceshipOperator);

TEST_CASE(simplifyIfSwitchForInit1);

TEST_CASE(simplifyIfSwitchForInit2);

TEST_CASE(simplifyIfSwitchForInit3);

TEST_CASE(simplifyIfSwitchForInit4);

TEST_CASE(simplifyIfSwitchForInit5);

TEST_CASE(newPlacementArgsCppInit); // #13775

TEST_CASE(cpp20_default_bitfield_initializer);

TEST_CASE(cpp11init);

TEST_CASE(testDirectiveIncludeTypes);

TEST_CASE(testDirectiveIncludeLocations);

TEST_CASE(testDirectiveIncludeComments);

TEST_CASE(testDirectiveRelativePath);

TEST_CASE(funcnameInParenthesis1); // #13554

TEST_CASE(funcnameInParenthesis2); // #13578

TEST_CASE(funcnameInParenthesis3); // #13585

TEST_CASE(genericInIf); // #13561

TEST_CASE(preincrementInLambda); // #13312

TEST_CASE(atomicCast); // #12605

TEST_CASE(constFunctionPtrTypedef); // #12135

TEST_CASE(simplifyPlatformTypes);

TEST_CASE(dumpFallthrough);

TEST_CASE(simplifyRedundantParentheses);

TEST_CASE(simplifyEnum1);

TEST_CASE(simplifyEnum2);

}

class TokenizerTest final : public Tokenizer

{

friend class TestTokenizer;

public:

TokenizerTest(TokenList tokenList, ErrorLogger &errorLogger)

: Tokenizer(std::move(tokenList), errorLogger)

{}

};

struct TokenizeOptions

{

bool expand = true;

Platform::Type platform = Platform::Type::Native;

bool cpp = true;

Standards::cppstd_t cppstd = Standards::CPP11;

Standards::cstd_t cstd = Standards::C11;

};

#define tokenizeAndStringify(...) tokenizeAndStringify_(__FILE__, __LINE__, __VA_ARGS__)

template<size_t size>

std::string tokenizeAndStringify_(const char* file, int linenr, const char (&code)[size], const TokenizeOptions& opt = make_default_obj{}) {

const Settings settings = settingsBuilder(settings1).cpp(opt.cppstd).c(opt.cstd).platform(opt.platform).build();

// tokenize..

SimpleTokenizer tokenizer(settings, *this, opt.cpp);

ASSERT_LOC(tokenizer.tokenize(code), file, linenr);

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, opt.expand, false, true, false, nullptr, nullptr);

return "";

}

// TODO: get rid of this

std::string tokenizeAndStringify_(const char* file, int linenr, const std::string& code) {

SimpleTokenizer tokenizer(settings2, *this);

ASSERT_LOC(tokenizer.tokenize(code), file, linenr);

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr);

return "";

}

template<size_t size>

std::string tokenizeAndStringify_(const char* file, int line, const char (&code)[size], const Settings &settings, bool cpp = true) {

// tokenize..

SimpleTokenizer tokenizer(settings, *this, cpp);

ASSERT_LOC(tokenizer.tokenize(code), file, line);

if (!tokenizer.tokens())

return "";

return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr);

}

#define tokenizeAndStringifyWindows(...) tokenizeAndStringifyWindows_(__FILE__, __LINE__, __VA_ARGS__)

template<size_t size>

std::string tokenizeAndStringifyWindows_(const char* file, int linenr, const char (&code)[size], Platform::Type platform = Platform::Type::Native) {

const Settings settings = settingsBuilder(settings_windows).platform(platform).build();

// tokenize..

SimpleTokenizer tokenizer(settings, *this);

ASSERT_LOC(tokenizer.tokenize(code), file, linenr);

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr);

return "";

}

#define tokenizeDebugListing(...) tokenizeDebugListing_(__FILE__, __LINE__, __VA_ARGS__)

template<size_t size>

std::string tokenizeDebugListing_(const char* file, int line, const char (&code)[size], bool cpp = true) {

SimpleTokenizer tokenizer(settings3, *this, cpp);

ASSERT_LOC(tokenizer.tokenize(code), file, line);

// result..

return tokenizer.tokens()->stringifyList(true,true,true,true,false);

}

template<size_t size>

void directiveDump(const char (&code)[size], std::ostream& ostr) {

directiveDump(code, "test.c", settingsDefault, ostr);

}

template<size_t size>

void directiveDump(const char (&code)[size], const char filename[], const Settings& settings, std::ostream& ostr) {

simplecpp::OutputList outputList;

std::vector<std::string> files;

simplecpp::TokenList tokens1(code, files, filename, &outputList);

Preprocessor preprocessor(tokens1, settings, *this, Path::identify(tokens1.getFiles()[0], false));

(void)preprocessor.reportOutput(outputList, true);

ASSERT(preprocessor.loadFiles(files));

std::list<Directive> directives = preprocessor.createDirectives();

TokenList tokenlist{settings, Path::identify(filename, false)};

Tokenizer tokenizer(std::move(tokenlist), *this);

tokenizer.setDirectives(std::move(directives));

tokenizer.dump(ostr);

}

void tokenize1() {

const char code[] = "void f ( )\n"

"{ if ( p . y ( ) > yof ) { } }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

ASSERT_EQUALS(

"[test.cpp:2:20]: (debug) valueFlowConditionExpressions bailout: Skipping function due to incomplete variable yof [valueFlowBailoutIncompleteVar]\n",

errout_str());

}

void tokenize2() {

const char code[] = "{ sizeof a, sizeof b }";

ASSERT_EQUALS("{ sizeof ( a ) , sizeof ( b ) }", tokenizeAndStringify(code));

}

void tokenize4() {

const char code[] = "class foo\n"

"{\n"

"public:\n"

" const int i;\n"

"}";

ASSERT_EQUALS("class foo\n"

"{\n"

"public:\n"

"const int i ;\n"

"}", tokenizeAndStringify(code));

ASSERT_EQUALS("", errout_str());

}

void tokenize5() {

// Tokenize values

ASSERT_EQUALS("; + 1E3 ;", tokenizeAndStringify("; +1E3 ;"));

ASSERT_EQUALS("; 1E-2 ;", tokenizeAndStringify("; 1E-2 ;"));

}

void tokenize7() {

const char code[] = "void f() {\n"

" int x1 = 1;\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("void f ( ) {\nint x1 ; x1 = 1 ;\nint x2 ( x1 ) ;\n}",

tokenizeAndStringify(code));

}

void tokenize8() {

const char code[] = "void f() {\n"

" int x1(g());\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("1: void f ( ) {\n"

"2: int x1@1 ( g ( ) ) ;\n"

"3: int x2@2 ( x1@1 ) ;\n"

"4: }\n",

tokenizeDebugListing(code));

}

void tokenize9() {

const char code[] = "typedef void (*fp)();\n"

"typedef fp (*fpp)();\n"

"void f() {\n"

" fpp x = (fpp)f();\n"

"}";

(void)tokenizeAndStringify(code);

ASSERT_EQUALS("", errout_str());

}

void tokenize11() {

ASSERT_EQUALS("X * sizeof ( Y ( ) ) ;", tokenizeAndStringify("X * sizeof(Y());"));

}

// bailout if there is "@" - it is not handled well

void tokenize13() {

const char code[] = "@implementation\n"

"-(Foo *)foo: (Bar *)bar\n"

"{ }\n"

"@end\n";

ASSERT_THROW_INTERNAL(tokenizeAndStringify(code), SYNTAX);

}

// Ticket #2361: 0X10 => 16

void tokenize14() {

ASSERT_EQUALS("; 0x10 ;", tokenizeAndStringify(";0x10;"));

ASSERT_EQUALS("; 0X10 ;", tokenizeAndStringify(";0X10;"));

ASSERT_EQUALS("; 0444 ;", tokenizeAndStringify(";0444;"));

}

// Ticket #8050

void tokenizeHexWithSuffix() {

ASSERT_EQUALS("; 0xFFFFFF ;", tokenizeAndStringify(";0xFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFul;"));

// Number of digits decides about internal representation...

ASSERT_EQUALS("; 0xFFFFFFFF ;", tokenizeAndStringify(";0xFFFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFFFul;"));

}

// Ticket #2429: 0.125

void tokenize15() {

ASSERT_EQUALS("0.125 ;", tokenizeAndStringify(".125;"));

ASSERT_EQUALS("005.125 ;", tokenizeAndStringify("005.125;")); // Don't confuse with octal values

}

void tokenize17() { // #2759

ASSERT_EQUALS("class B : private :: A { } ;", tokenizeAndStringify("class B : private ::A { };"));

}

void tokenize18() { // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

ASSERT_EQUALS("( X && Y ) ;", tokenizeAndStringify("(X&&Y);"));

}

void tokenize19() {

// #3006 - added hasComplicatedSyntaxErrorsInTemplates to avoid segmentation fault

ASSERT_THROW_INTERNAL(tokenizeAndStringify("x < () <"), SYNTAX);

// #3496 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("void a ( Fred * f ) { for ( ; n < f . x ( ) ; ) { } }",

tokenizeAndStringify("void a(Fred* f) MACRO { for (;n < f->x();) {} }"));

// #6216 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("C :: C ( )\n"

": v { }\n"

"{\n"

"for ( int dim = 0 ; dim < v . size ( ) ; ++ dim ) {\n"

"v [ dim ] . f ( ) ;\n"

"}\n"

"} ;",

tokenizeAndStringify("C::C()\n"

":v{}\n"

"{\n"

" for (int dim = 0; dim < v.size(); ++dim) {\n"

" v[dim]->f();\n"

" }\n"

"};"));

ignore_errout(); // we do not care about the output

}

void tokenize21() { // tokenize 0x0E-7

ASSERT_EQUALS("0x0E - 7 ;", tokenizeAndStringify("0x0E-7;"));

}

void tokenize22() { // tokenize special marker $ from preprocessor

ASSERT_EQUALS("a$b", tokenizeAndStringify("a$b"));

ASSERT_EQUALS("a $b\nc", tokenizeAndStringify("a $b\nc"));

ASSERT_EQUALS("a = $0 ;", tokenizeAndStringify("a = $0;"));

ASSERT_EQUALS("a$ ++ ;", tokenizeAndStringify("a$++;"));

ASSERT_EQUALS("$if ( ! p )", tokenizeAndStringify("$if(!p)"));

}

// #4239 - segfault for "f ( struct { int typedef T x ; } ) { }"

void tokenize25() {

ASSERT_THROW_INTERNAL(tokenizeAndStringify("f ( struct { int typedef T x ; } ) { }"), SYNTAX);

}

// #4245 - segfault

void tokenize26() {

ASSERT_THROW_INTERNAL(tokenizeAndStringify("class x { protected : template < int y = } ;"), SYNTAX); // Garbage code

}

void tokenize27() {

// #4525 - segfault

(void)tokenizeAndStringify("struct except_spec_d_good : except_spec_a, except_spec_b {\n"

"~except_spec_d_good();\n"

"};\n"

"struct S { S(); };\n"

"S::S() __attribute((pure)) = default;"

);

// original code: glibc-2.18/posix/bug-regex20.c

(void)tokenizeAndStringify("static unsigned int re_string_context_at (const re_string_t *input, int idx, int eflags) internal_function __attribute__ ((pure));");

}

// #3503 - don't "simplify" SetFunction member function to a variable

void tokenize31() {

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) { }\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)()) { }\n"

"};"));

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) ;\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)());\n"

"};"));

}

// #5884 - Avoid left shift of negative integer value.

void tokenize32() {

// Do not simplify negative integer left shifts.

const char code[] = "void f ( ) { int max_x ; max_x = -10000 << 16 ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

// #5780 Various crashes on valid template code in Tokenizer::setVarId()

void tokenize33() {

const char code[] = "template<typename T, typename A = Alloc<T>> struct vector {};\n"

"void z() {\n"

" vector<int> VI;\n"

"}\n";

(void)tokenizeAndStringify(code);

}

void tokenize34() { // #8031

{

const char code[] = "struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"Container::Container() : mElements(nullptr) {}\n"

"Container intContainer;";

const char exp[] = "1: struct Container {\n"

"2: Container ( ) ;\n"

"3: int * mElements@1 ;\n"

"4: } ;\n"

"5: Container :: Container ( ) : mElements@1 ( nullptr ) { }\n"

"6: Container intContainer@2 ;\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

{

const char code[] = "template<class T> struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"template <class T> Container<T>::Container() : mElements(nullptr) {}\n"

"Container<int> intContainer;";

const char exp[] = "1: struct Container<int> ;\n"

"2:\n"

"|\n"

"5:\n"

"6: Container<int> intContainer@1 ;\n"

"1: struct Container<int> {\n"

"2: Container<int> ( ) ;\n"

"3: int * mElements@2 ;\n"

"4: } ;\n"

"5: Container<int> :: Container<int> ( ) : mElements@2 ( nullptr ) { }\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

}

void tokenize35() { // #8361

ASSERT_NO_THROW(tokenizeAndStringify("typedef int CRCWord; "

"template<typename T> ::CRCWord const Compute(T const t) { return 0; }"));

}

void tokenize36() { // #8436

const char code[] = "int foo ( int i ) { return i ? * new int { 5 } : int { i ? 0 : 1 } ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

void tokenize37() { // #8550

const char codeC[] = "class name { public: static void init ( ) {} } ; "

"typedef class name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expC[] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expC, tokenizeAndStringify(codeC));

const char codeS[] = "class name { public: static void init ( ) {} } ; "

"typedef struct name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expS[] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expS, tokenizeAndStringify(codeS));

}

void tokenize38() { // #9569

const char code[] = "using Binary = std::vector<char>; enum Type { Binary };";

const char exp[] = "enum Type { Binary } ;";

ASSERT_EQUALS(exp, tokenizeAndStringify(code));

}

void tokenize39() { // #9771

const char code[] = "template <typename T> class Foo;"

"template <typename T> bool operator!=(const Foo<T> &, const Foo<T> &);"

"template <typename T> class Foo { friend bool operator!= <> (const Foo<T> &, const Foo<T> &); };";

const char exp[] = "template < typename T > class Foo ; "

"template < typename T > bool operator!= ( const Foo < T > & , const Foo < T > & ) ; "

"template < typename T > class Foo { friend bool operator!= < > ( const Foo < T > & , const Foo < T > & ) ; } ;";

ASSERT_EQUALS(exp, tokenizeAndStringify(code));

}

void tokenize40() { // #13181

const char code[] = "struct A { double eps(double); };\n"

"A operator \"\"_a(long double);\n"

"void f() {\n"

" double d = 1.23;\n"

" if (d == 1.2_a .eps(.1)) {}\n"

"}\n";

(void) tokenizeAndStringify(code);

ASSERT_EQUALS("", errout_str());

}

void tokenize41() { // #13847

const char code[] = "int main() {\n"

" using x::a;\n"

" b = a + 2;\n"

"}\n";

ASSERT_EQUALS("int main ( ) {\n"

"\n"

"b = x :: a + 2 ;\n"

"}", tokenizeAndStringify(code));

(void)errout_str();

}

void tokenize42() { // #13861

const char code[] = "struct AB { int a; int b; };\n"

"namespace ns { typedef AB S[10]; }\n"

"void foo(void) {\n"

" ns::S x = {0};\n"

" x[1].a = 2;\n"

"}\n";

ASSERT_EQUALS("struct AB { int a ; int b ; } ;\n"

"\n"

"void foo ( ) {\n"

"AB x [ 10 ] = { 0 } ;\n"

"x [ 1 ] . a = 2 ;\n"

"}", tokenizeAndStringify(code));

(void)errout_str();

}

void validate() {

// C++ code in C file

ASSERT_THROW_INTERNAL(tokenizeAndStringify(";using namespace std;",dinit(TokenizeOptions, $.expand = false, $.cpp = false)), SYNTAX);

ASSERT_THROW_INTERNAL(tokenizeAndStringify(";std::map<int,int> m;",dinit(TokenizeOptions, $.expand = false, $.cpp = false)), SYNTAX);

ASSERT_THROW_INTERNAL(tokenizeAndStringify(";template<class T> class X { };",dinit(TokenizeOptions, $.expand = false, $.cpp = false)), SYNTAX);

ASSERT_THROW_INTERNAL(tokenizeAndStringify("int X<Y>() {};",dinit(TokenizeOptions, $.expand = false, $.cpp = false)), SYNTAX);

{

TokenList tokenlist{settings1, Standards::Language::C}; // headers are treated as C files

const char code[] = "void foo(int i) { reinterpret_cast<char>(i) };";

tokenlist.appendFileIfNew("test.h");

ASSERT(tokenlist.createTokensFromString(code));

Tokenizer tokenizer(std::move(tokenlist), *this);

ASSERT_THROW_INTERNAL(tokenizer.simplifyTokens1(""), SYNTAX);

}

}

void objectiveC() {

ASSERT_THROW_INTERNAL(tokenizeAndStringify("void f() { [foo bar]; }"), SYNTAX);

}

void syntax_case_default() { // correct syntax

ASSERT_NO_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case 0: z(); break;}}"));

ASSERT_EQUALS("", errout_str());

(void)tokenizeAndStringify("void f(int n) {switch (n) { case 0:; break;}}");

ASSERT_EQUALS("", errout_str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case 0?1:2 : z(); break;}}"), InternalError);

//ASSERT_EQUALS("", errout_str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case 0?(1?3:4):2 : z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout_str());

//allow GCC '({ %name%|%num%|%bool% ; })' statement expression extension

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case 0?({0;}):1: z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout_str());

//'b' can be or a macro or an undefined enum

ASSERT_NO_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case b: z(); break;}}"));

ASSERT_EQUALS("", errout_str());

//valid, when there's this declaration: 'constexpr int g() { return 2; }'

ASSERT_NO_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case g(): z(); break;}}"));

ASSERT_EQUALS("", errout_str());

//valid, when there's also this declaration: 'constexpr int g[1] = {0};'

ASSERT_NO_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case g[0]: z(); break;}}"));

ASSERT_EQUALS(

"[test.cpp:1:34]: (debug) valueFlowConditionExpressions bailout: Skipping function due to incomplete variable g [valueFlowBailoutIncompleteVar]\n",

errout_str());

//valid, similar to above case

ASSERT_NO_THROW(tokenizeAndStringify("void f(int n) {switch (n) { case *g: z(); break;}}"));

ASSERT_EQUALS("", errout_str());

//valid, when 'x' and 'y' are constexpr.