From 7dfa362cad61666c3fd69ce619a67d25c9d952c0 Mon Sep 17 00:00:00 2001 From: Alexander Mai Date: Mon, 8 May 2017 20:52:49 +0200 Subject: [PATCH 001/381] Fix build with clang + small refactoring --- simplecpp.cpp | 12 +++--------- simplecpp.h | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a906d2d1..114153d9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -349,7 +349,7 @@ static unsigned short getAndSkipBOM(std::istream &istr) return 0; } -bool isNameChar(unsigned char ch) +static bool isNameChar(unsigned char ch) { return std::isalnum(ch) || ch == '_' || ch == '$'; } @@ -1737,16 +1737,10 @@ namespace { } } + const char * const altopData[] = {"and","or","bitand","bitor","not","not_eq","xor"}; + const std::set altop(&altopData[0], &altopData[7]); void simplifyName(simplecpp::TokenList &expr) { - std::set altop; - altop.insert("and"); - altop.insert("or"); - altop.insert("bitand"); - altop.insert("bitor"); - altop.insert("not"); - altop.insert("not_eq"); - altop.insert("xor"); for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->name) { if (altop.find(tok->str) != altop.end()) { diff --git a/simplecpp.h b/simplecpp.h index a1a5ca7d..04639006 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -268,6 +268,7 @@ namespace simplecpp { }; struct SIMPLECPP_LIB DUI { + DUI() {} std::list defines; std::set undefined; std::list includePaths; From b84234716046450baa0204e547a928c09a3fa3fa Mon Sep 17 00:00:00 2001 From: amai2012 Date: Thu, 8 Jun 2017 17:02:24 +0200 Subject: [PATCH 002/381] Avoid warning of NOMINMAX redefined building with cygwin/mingw --- simplecpp.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 114153d9..8ba9b734 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1,4 +1,4 @@ -/* + /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library * Copyright (C) 2016 Daniel Marjamäki. * @@ -16,6 +16,9 @@ * License along with this library. If not, see . */ +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) +#define NOMINMAX +#endif #include "simplecpp.h" #include @@ -36,7 +39,6 @@ #include #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) -#define NOMINMAX #include #undef ERROR #undef TRUE From e0a92fe250af26de6c7df36d65f5071c453a38d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 8 Jun 2017 22:50:54 +0200 Subject: [PATCH 003/381] Use static instead of anonymous namespace for functions and variables --- simplecpp.cpp | 254 +++++++++++++++++++++++++------------------------- 1 file changed, 126 insertions(+), 128 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 27dfb5f3..c60d8e3a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1,4 +1,4 @@ - /* +/* * simplecpp - A simple and high-fidelity C/C++ preprocessor library * Copyright (C) 2016 Daniel Marjamäki. * @@ -1755,161 +1755,159 @@ namespace simplecpp { } } -namespace { - /** Evaluate sizeof(type) */ - void simplifySizeof(simplecpp::TokenList &expr, const std::map &sizeOfType) - { - for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->str != "sizeof") - continue; - simplecpp::Token *tok1 = tok->next; - if (!tok1) { - throw std::runtime_error("missed sizeof argument"); - } - simplecpp::Token *tok2 = tok1->next; - if (!tok2) { - throw std::runtime_error("missed sizeof argument"); - } - if (tok1->op == '(') { - tok1 = tok1->next; - while (tok2->op != ')') - tok2 = tok2->next; - } - - std::string type; - for (simplecpp::Token *typeToken = tok1; typeToken != tok2; typeToken = typeToken->next) { - if ((typeToken->str == "unsigned" || typeToken->str == "signed") && typeToken->next->name) - continue; - if (typeToken->str == "*" && type.find('*') != std::string::npos) - continue; - if (!type.empty()) - type += ' '; - type += typeToken->str; - } +/** Evaluate sizeof(type) */ +static void simplifySizeof(simplecpp::TokenList &expr, const std::map &sizeOfType) +{ + for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { + if (tok->str != "sizeof") + continue; + simplecpp::Token *tok1 = tok->next; + if (!tok1) { + throw std::runtime_error("missed sizeof argument"); + } + simplecpp::Token *tok2 = tok1->next; + if (!tok2) { + throw std::runtime_error("missed sizeof argument"); + } + if (tok1->op == '(') { + tok1 = tok1->next; + while (tok2->op != ')') + tok2 = tok2->next; + } - const std::map::const_iterator it = sizeOfType.find(type); - if (it != sizeOfType.end()) - tok->setstr(toString(it->second)); - else + std::string type; + for (simplecpp::Token *typeToken = tok1; typeToken != tok2; typeToken = typeToken->next) { + if ((typeToken->str == "unsigned" || typeToken->str == "signed") && typeToken->next->name) continue; - - tok2 = tok2->next; - while (tok->next != tok2) - expr.deleteToken(tok->next); + if (typeToken->str == "*" && type.find('*') != std::string::npos) + continue; + if (!type.empty()) + type += ' '; + type += typeToken->str; } + + const std::map::const_iterator it = sizeOfType.find(type); + if (it != sizeOfType.end()) + tok->setstr(toString(it->second)); + else + continue; + + tok2 = tok2->next; + while (tok->next != tok2) + expr.deleteToken(tok->next); } +} - const char * const altopData[] = {"and","or","bitand","bitor","not","not_eq","xor"}; - const std::set altop(&altopData[0], &altopData[7]); - void simplifyName(simplecpp::TokenList &expr) - { - for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->name) { - if (altop.find(tok->str) != altop.end()) { - bool alt; - if (tok->str == "not") { - alt = isAlternativeUnaryOp(tok,tok->str); - } else { - alt = isAlternativeBinaryOp(tok,tok->str); - } - if (alt) - continue; +static const char * const altopData[] = {"and","or","bitand","bitor","not","not_eq","xor"}; +static const std::set altop(&altopData[0], &altopData[7]); +static void simplifyName(simplecpp::TokenList &expr) +{ + for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { + if (tok->name) { + if (altop.find(tok->str) != altop.end()) { + bool alt; + if (tok->str == "not") { + alt = isAlternativeUnaryOp(tok,tok->str); + } else { + alt = isAlternativeBinaryOp(tok,tok->str); } - tok->setstr("0"); + if (alt) + continue; } + tok->setstr("0"); } } +} - void simplifyNumbers(simplecpp::TokenList &expr) - { - for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->str.size() == 1U) - continue; - if (tok->str.compare(0,2,"0x") == 0) - tok->setstr(toString(stringToULL(tok->str))); - else if (tok->str[0] == '\'') - tok->setstr(toString(tok->str[1] & 0xffU)); - } - } - - long long evaluate(simplecpp::TokenList &expr, const std::map &sizeOfType) - { - simplifySizeof(expr, sizeOfType); - simplifyName(expr); - simplifyNumbers(expr); - expr.constFold(); - // TODO: handle invalid expressions - return expr.cfront() && expr.cfront() == expr.cback() && expr.cfront()->number ? stringToLL(expr.cfront()->str) : 0LL; +static void simplifyNumbers(simplecpp::TokenList &expr) +{ + for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { + if (tok->str.size() == 1U) + continue; + if (tok->str.compare(0,2,"0x") == 0) + tok->setstr(toString(stringToULL(tok->str))); + else if (tok->str[0] == '\'') + tok->setstr(toString(tok->str[1] & 0xffU)); } +} - const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) - { - const unsigned int line = tok->location.line; - const unsigned int file = tok->location.fileIndex; - while (tok && tok->location.line == line && tok->location.fileIndex == file) - tok = tok->next; - return tok; - } +static long long evaluate(simplecpp::TokenList &expr, const std::map &sizeOfType) +{ + simplifySizeof(expr, sizeOfType); + simplifyName(expr); + simplifyNumbers(expr); + expr.constFold(); + // TODO: handle invalid expressions + return expr.cfront() && expr.cfront() == expr.cback() && expr.cfront()->number ? stringToLL(expr.cfront()->str) : 0LL; +} - std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) - { - if (!systemheader) { - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - f.open(s.c_str()); - if (f.is_open()) - return simplecpp::simplifyPath(s); - } else { - f.open(header.c_str()); - if (f.is_open()) - return simplecpp::simplifyPath(header); - } - } +static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) +{ + const unsigned int line = tok->location.line; + const unsigned int file = tok->location.fileIndex; + while (tok && tok->location.line == line && tok->location.fileIndex == file) + tok = tok->next; + return tok; +} - for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = *it; - if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') - s += '/'; - s += header; +static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) +{ + if (!systemheader) { + if (sourcefile.find_first_of("\\/") != std::string::npos) { + const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; f.open(s.c_str()); if (f.is_open()) return simplecpp::simplifyPath(s); + } else { + f.open(header.c_str()); + if (f.is_open()) + return simplecpp::simplifyPath(header); } + } - return ""; + for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { + std::string s = *it; + if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') + s += '/'; + s += header; + f.open(s.c_str()); + if (f.is_open()) + return simplecpp::simplifyPath(s); } - std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) - { - if (!systemheader) { - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s(simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header)); - if (filedata.find(s) != filedata.end()) - return s; - } else { - std::string s = simplecpp::simplifyPath(header); - if (filedata.find(s) != filedata.end()) - return s; - } - } + return ""; +} - for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = *it; - if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') - s += '/'; - s += header; - s = simplecpp::simplifyPath(s); +static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +{ + if (!systemheader) { + if (sourcefile.find_first_of("\\/") != std::string::npos) { + const std::string s(simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header)); + if (filedata.find(s) != filedata.end()) + return s; + } else { + std::string s = simplecpp::simplifyPath(header); if (filedata.find(s) != filedata.end()) return s; } - - return ""; } - bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) - { - return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); + for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { + std::string s = *it; + if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') + s += '/'; + s += header; + s = simplecpp::simplifyPath(s); + if (filedata.find(s) != filedata.end()) + return s; } + + return ""; +} + +static bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +{ + return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); } std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &fileNumbers, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) From 047e4e7000f2f06b691e73c20e6b37d3e14684ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 9 Jun 2017 13:50:52 +0200 Subject: [PATCH 004/381] Fixed #81 (C++11 raw literal causes wrong line numbers) --- simplecpp.cpp | 8 ++++++-- test.cpp | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c60d8e3a..8919f31f 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -136,7 +136,7 @@ void simplecpp::Location::adjust(const std::string &str) for (std::size_t i = 0U; i < str.size(); ++i) { col++; if (str[i] == '\n' || str[i] == '\r') { - col = 0; + col = 1; line++; if (str[i] == '\r' && (i+1)setstr(escapeString(currentToken)); - location.col += currentToken.size() + 2U + 2 * delim.size(); + location.adjust(currentToken); + if (currentToken.find_first_of("\r\n") == std::string::npos) + location.col += 2 + 2 * delim.size(); + else + location.col += 1 + delim.size(); continue; } diff --git a/test.cpp b/test.cpp index b7f8acd7..0237d7b8 100644 --- a/test.cpp +++ b/test.cpp @@ -1078,6 +1078,7 @@ void readfile_rawstring() ASSERT_EQUALS("A = \"\\\\\"", readfile("A = R\"(\\)\"")); ASSERT_EQUALS("A = \"\\\"\"", readfile("A = R\"(\")\"")); ASSERT_EQUALS("A = \"abc\"", readfile("A = R\"\"\"(abc)\"\"\"")); + ASSERT_EQUALS("A = \"a\nb\nc\";", readfile("A = R\"foo(a\nb\nc)foo\";")); } void stringify1() From c21b2a0daeefa0390b4465d6b07bc98c9f71e8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 16 Jun 2017 11:26:33 +0200 Subject: [PATCH 005/381] Fixed #83 Header not found, absolute path --- simplecpp.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 8919f31f..04108df2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1719,8 +1719,19 @@ namespace simplecpp { return f; return ostr.str(); } + + bool isAbsolutePath(const std::string &path) { + if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) + return true; + return path.length() > 1U && (path[0] == '/' || path[0] == '/'); + } + #else #define realFilename(f) f + + bool isAbsolutePath(const std::string &path) { + return path.length() > 1U && path[0] == '/'; + } #endif /** @@ -1856,6 +1867,11 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { + if (simplecpp::isAbsolutePath(header)) { + f.open(header.c_str()); + return f.is_open() ? simplecpp::simplifyPath(header) : ""; + } + if (!systemheader) { if (sourcefile.find_first_of("\\/") != std::string::npos) { const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; @@ -1884,6 +1900,10 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { + if (simplecpp::isAbsolutePath(header)) { + return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; + } + if (!systemheader) { if (sourcefile.find_first_of("\\/") != std::string::npos) { const std::string s(simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header)); From 5f2cd0686816569455a5b98458a6e29301c21d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 20 Jun 2017 22:19:35 +0200 Subject: [PATCH 006/381] astyle formatting --- simplecpp.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 04108df2..5919406f 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1720,7 +1720,8 @@ namespace simplecpp { return ostr.str(); } - bool isAbsolutePath(const std::string &path) { + bool isAbsolutePath(const std::string &path) + { if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) return true; return path.length() > 1U && (path[0] == '/' || path[0] == '/'); @@ -1729,7 +1730,8 @@ namespace simplecpp { #else #define realFilename(f) f - bool isAbsolutePath(const std::string &path) { + bool isAbsolutePath(const std::string &path) + { return path.length() > 1U && path[0] == '/'; } #endif From eff31dc75db2c7295177158e59e4e91f647a45b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 20 Jun 2017 23:23:04 +0200 Subject: [PATCH 007/381] Fixed #80 (Cannot find include files with relative paths on Windows) --- simplecpp.cpp | 48 +++++++++++++++++++++++++++++------- test.cpp | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5919406f..fb79b1dc 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1746,6 +1746,14 @@ namespace simplecpp { // replace backslash separators std::replace(path.begin(), path.end(), '\\', '/'); + const bool unc(path.compare(0,2,"//") == 0); + + // replace "//" with "/" + pos = 0; + while ((pos = path.find("//",pos)) != std::string::npos) { + path.erase(pos,1); + } + // remove "./" pos = 0; while ((pos = path.find("./",pos)) != std::string::npos) { @@ -1755,19 +1763,41 @@ namespace simplecpp { pos += 2; } - // remove "xyz/../" - pos = 1U; - while ((pos = path.find("/../", pos)) != std::string::npos) { - const std::string::size_type pos1 = path.rfind('/', pos - 1U); - if (pos1 == std::string::npos) { - path.erase(0,pos+4); - pos = 0; + // remove trailing dot if path ends with "/." + if (endsWith(path,"/.")) + path.erase(path.size()-1); + + // simplify ".." + pos = 1; // don't simplify ".." if path starts with that + while ((pos = path.find("/..", pos)) != std::string::npos) { + // not end of path, then string must be "/../" + if (pos + 3 < path.size() && path[pos + 3] != '/') { + ++pos; + continue; + } + // get previous subpath + const std::string::size_type pos1 = path.rfind('/', pos - 1U) + 1U; + const std::string previousSubPath = path.substr(pos1, pos-pos1); + if (previousSubPath == "..") { + // don't simplify + ++pos; } else { - path.erase(pos1,pos-pos1+3); - pos = std::min((std::string::size_type)1, pos1); + // remove previous subpath and ".." + path.erase(pos1,pos-pos1+4); + if (path.empty()) + path = "."; + // update pos + pos = (pos1 == 0) ? 1 : (pos1 - 1); } } + // Remove trailing '/'? + //if (path.size() > 1 && endsWith(path, "/")) + // path.erase(path.size()-1); + + if (unc) + path = '/' + path; + return realFilename(path); } } diff --git a/test.cpp b/test.cpp index 0237d7b8..87bf0171 100644 --- a/test.cpp +++ b/test.cpp @@ -1256,6 +1256,71 @@ void simplifyPath() ASSERT_EQUALS("../1.c", simplecpp::simplifyPath("../a/../1.c")); ASSERT_EQUALS("/../1.c", simplecpp::simplifyPath("/../1.c")); ASSERT_EQUALS("/../1.c", simplecpp::simplifyPath("/../a/../1.c")); + + ASSERT_EQUALS("a/..b/1.c", simplecpp::simplifyPath("a/..b/1.c")); + ASSERT_EQUALS("../../1.c", simplecpp::simplifyPath("../../1.c")); + ASSERT_EQUALS("../../../1.c", simplecpp::simplifyPath("../../../1.c")); + ASSERT_EQUALS("../../../1.c", simplecpp::simplifyPath("../../../a/../1.c")); + ASSERT_EQUALS("../../1.c", simplecpp::simplifyPath("a/../../../1.c")); +} + +// tests transferred from cppcheck +// https://github.com/danmar/cppcheck/blob/d3e79b71b5ec6e641ca3e516cfced623b27988af/test/testpath.cpp#L43 +void simplifyPath_cppcheck() +{ + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath(".//index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath(".///index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/index.h")); + ASSERT_EQUALS("/path/", simplecpp::simplifyPath("/path/")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("/")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("/.")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("/./")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/./index.h")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("/.//")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/.//index.h")); + ASSERT_EQUALS("../index.h", simplecpp::simplifyPath("../index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/../index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./path/../index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("path/../index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path//../index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./path//../index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("path//../index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/..//index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./path/..//index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("path/..//index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path//..//index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./path//..//index.h")); + ASSERT_EQUALS("index.h", simplecpp::simplifyPath("path//..//index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/../other/../index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/../other///././../index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/../other/././..///index.h")); + ASSERT_EQUALS("/index.h", simplecpp::simplifyPath("/path/../other///././..///index.h")); + ASSERT_EQUALS("../path/index.h", simplecpp::simplifyPath("../path/other/../index.h")); + ASSERT_EQUALS("a/index.h", simplecpp::simplifyPath("a/../a/index.h")); + ASSERT_EQUALS(".", simplecpp::simplifyPath("a/..")); + ASSERT_EQUALS(".", simplecpp::simplifyPath("./a/..")); + ASSERT_EQUALS("../../src/test.cpp", simplecpp::simplifyPath("../../src/test.cpp")); + ASSERT_EQUALS("../../../src/test.cpp", simplecpp::simplifyPath("../../../src/test.cpp")); + ASSERT_EQUALS("src/test.cpp", simplecpp::simplifyPath(".//src/test.cpp")); + ASSERT_EQUALS("src/test.cpp", simplecpp::simplifyPath(".///src/test.cpp")); + ASSERT_EQUALS("test.cpp", simplecpp::simplifyPath("./././././test.cpp")); + ASSERT_EQUALS("src/", simplecpp::simplifyPath("src/abc/..")); + ASSERT_EQUALS("src/", simplecpp::simplifyPath("src/abc/../")); + + // Handling of UNC paths on Windows + ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("//src/test.cpp")); + ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("///src/test.cpp")); +} + +void simplifyPath_New() +{ + ASSERT_EQUALS("", simplecpp::simplifyPath("")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("/")); + ASSERT_EQUALS("//", simplecpp::simplifyPath("//")); + ASSERT_EQUALS("//", simplecpp::simplifyPath("///")); + ASSERT_EQUALS("/", simplecpp::simplifyPath("\\")); } @@ -1379,6 +1444,8 @@ int main(int argc, char **argv) // utility functions. TEST_CASE(simplifyPath); + TEST_CASE(simplifyPath_cppcheck); + TEST_CASE(simplifyPath_New); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 982006cd90515d56fa364a97c2d2313cd2dd8dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jun 2017 00:12:21 +0200 Subject: [PATCH 008/381] Fix test failure in cygwin --- simplecpp.cpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index fb79b1dc..ed8a144b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1679,7 +1679,7 @@ namespace simplecpp { namespace simplecpp { #ifdef SIMPLECPP_WINDOWS - static bool realFileName(const std::vector &buf, std::ostream &ostr) + bool realFileName(const std::vector &buf, std::ostream &ostr) { // Detect root directory, see simplecpp:realFileName returns the wrong root path #45 if ((buf.size()==2 || (buf.size()>2 && buf[2]=='\0')) @@ -1697,6 +1697,19 @@ namespace simplecpp { return true; } + const char *dotsPath(const std::string &f, const std::string::size_type sep) + { + if (sep >= 1 && f[sep-1]=='.') { + unsigned int dots = 1; + if (sep >= 2 && f[sep-2] == '.') + dots = 2; + const unsigned char c = (sep > dots) ? f[sep-dots-1] : '/'; + if (c=='/' || c=='\\') + return ((dots==1) ? "." : ".."); + } + return NULL; + } + std::string realFilename(const std::string &f) { std::vector buf(f.size()+1U, 0); @@ -1705,16 +1718,26 @@ namespace simplecpp { std::ostringstream ostr; std::string::size_type sep = 0; while ((sep = f.find_first_of("\\/", sep + 1U)) != std::string::npos) { - if (sep >= 2 && f.compare(sep-2,2,"..",0,2) == 0) { - ostr << "../"; + // do not convert ".." or "." + const char *s = dotsPath(f,sep); + if (s) { + ostr << s << '/'; continue; - } + } buf[sep] = 0; if (!realFileName(buf,ostr)) return f; ostr << '/'; buf[sep] = '/'; } + + if (endsWith(f,".")) { + const char *s = dotsPath(f,f.size()); + if (s) { + return ostr.str() + s; + } + } + if (!realFileName(buf, ostr)) return f; return ostr.str(); From 45c3e49cc4acd776be4c2050a7beac226e69fb27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jun 2017 10:49:29 +0200 Subject: [PATCH 009/381] rewrote realFileName() --- simplecpp.cpp | 114 +++++++++++++++++++++--------------- testsuite/realFileName1.cpp | 5 ++ 2 files changed, 73 insertions(+), 46 deletions(-) create mode 100644 testsuite/realFileName1.cpp diff --git a/simplecpp.cpp b/simplecpp.cpp index ed8a144b..f9661220 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1679,68 +1679,90 @@ namespace simplecpp { namespace simplecpp { #ifdef SIMPLECPP_WINDOWS - bool realFileName(const std::vector &buf, std::ostream &ostr) + bool realFileName(const std::string &f, std::string *result) { - // Detect root directory, see simplecpp:realFileName returns the wrong root path #45 - if ((buf.size()==2 || (buf.size()>2 && buf[2]=='\0')) - && std::isalpha(buf[0]) && buf[1]==':') { - ostr << (char)buf[0]; - ostr << (char)buf[1]; + // If path is a drive letter, uppercase it + if (f.size() == 2 && std::isalpha((unsigned char)f[0]) && f[1] == ':') { + *result = (char)std::toupper((unsigned char)f[0]) + std::string(":"); return true; } + + // are there alpha characters in last subpath? + bool alpha = false; + for (std::string::size_type pos = 1; pos < f.size(); ++pos) { + unsigned char c = f[f.size() - pos]; + if (c=='/' || c=='\\') + break; + if (std::isalpha(c)) { + alpha = true; + break; + } + } + + // do not convert this path if there are no alpha characters (either pointless or cause wrong results for . and ..) + if (!alpha) + return false; + + // Convert char path to CHAR path + std::vector buf(f.size()+1U, 0); + for (unsigned int i = 0; i < f.size(); ++i) + buf[i] = f[i]; + + // Lookup filename or foldername on file system WIN32_FIND_DATAA FindFileData; HANDLE hFind = FindFirstFileA(&buf[0], &FindFileData); if (hFind == INVALID_HANDLE_VALUE) return false; - ostr << FindFileData.cFileName; + *result = FindFileData.cFileName; FindClose(hFind); return true; } - const char *dotsPath(const std::string &f, const std::string::size_type sep) - { - if (sep >= 1 && f[sep-1]=='.') { - unsigned int dots = 1; - if (sep >= 2 && f[sep-2] == '.') - dots = 2; - const unsigned char c = (sep > dots) ? f[sep-dots-1] : '/'; - if (c=='/' || c=='\\') - return ((dots==1) ? "." : ".."); - } - return NULL; - } - + /** Change case in given path to match filesystem */ std::string realFilename(const std::string &f) { - std::vector buf(f.size()+1U, 0); - for (unsigned int i = 0; i < f.size(); ++i) - buf[i] = f[i]; - std::ostringstream ostr; - std::string::size_type sep = 0; - while ((sep = f.find_first_of("\\/", sep + 1U)) != std::string::npos) { - // do not convert ".." or "." - const char *s = dotsPath(f,sep); - if (s) { - ostr << s << '/'; - continue; - } - buf[sep] = 0; - if (!realFileName(buf,ostr)) - return f; - ostr << '/'; - buf[sep] = '/'; - } - - if (endsWith(f,".")) { - const char *s = dotsPath(f,f.size()); - if (s) { - return ostr.str() + s; + std::string ret; + ret.reserve(f.size()); // this will be the final size + + // Current subpath + std::string subpath; + + for (std::string::size_type pos = 0; pos < f.size(); ++pos) { + unsigned char c = f[pos]; + + // Separator.. add subpath and separator + if (c == '/' || c == '\\') { + // if subpath is empty just add separator + if (subpath.empty()) { + ret += c; + continue; + } + + // Append real filename (proper case) + std::string f2; + if (realFileName(f.substr(0,pos),&f2)) + ret += f2; + else + ret += subpath; + + subpath.clear(); + + // Append separator + ret += c; + } else { + subpath += c; } } - if (!realFileName(buf, ostr)) - return f; - return ostr.str(); + if (!subpath.empty()) { + std::string f2; + if (realFileName(f,&f2)) + ret += f2; + else + ret += subpath; + } + + return ret; } bool isAbsolutePath(const std::string &path) diff --git a/testsuite/realFileName1.cpp b/testsuite/realFileName1.cpp new file mode 100644 index 00000000..55f743ed --- /dev/null +++ b/testsuite/realFileName1.cpp @@ -0,0 +1,5 @@ +// Run: +// simplecpp.exe realFileName1.cpp | grep main.cpp + +#include "../MAIN.CPP" + From e0c4c490e5d07825ce241984aec2f55d8444e9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jun 2017 10:51:27 +0200 Subject: [PATCH 010/381] testsuite/realFileName1.cpp: add run command for absolute windows path --- testsuite/realFileName1.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/realFileName1.cpp b/testsuite/realFileName1.cpp index 55f743ed..0c007e73 100644 --- a/testsuite/realFileName1.cpp +++ b/testsuite/realFileName1.cpp @@ -1,5 +1,6 @@ // Run: // simplecpp.exe realFileName1.cpp | grep main.cpp +// simplecpp.exe c:\...\realFileName1.cpp | grep main.cpp #include "../MAIN.CPP" From df090d333113d531022466318c14c3f0c00aea54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jun 2017 18:24:28 +0200 Subject: [PATCH 011/381] don't include realFileName in simplecpp namespace --- simplecpp.cpp | 153 +++++++++++++++++++++++++------------------------- 1 file changed, 75 insertions(+), 78 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f9661220..cfb96c87 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1675,112 +1675,109 @@ namespace simplecpp { }; } - -namespace simplecpp { #ifdef SIMPLECPP_WINDOWS +static bool realFileName(const std::string &f, std::string *result) +{ + // If path is a drive letter, uppercase it + if (f.size() == 2 && std::isalpha((unsigned char)f[0]) && f[1] == ':') { + *result = (char)std::toupper((unsigned char)f[0]) + std::string(":"); + return true; + } - bool realFileName(const std::string &f, std::string *result) - { - // If path is a drive letter, uppercase it - if (f.size() == 2 && std::isalpha((unsigned char)f[0]) && f[1] == ':') { - *result = (char)std::toupper((unsigned char)f[0]) + std::string(":"); - return true; - } - - // are there alpha characters in last subpath? - bool alpha = false; - for (std::string::size_type pos = 1; pos < f.size(); ++pos) { - unsigned char c = f[f.size() - pos]; - if (c=='/' || c=='\\') - break; - if (std::isalpha(c)) { - alpha = true; - break; - } + // are there alpha characters in last subpath? + bool alpha = false; + for (std::string::size_type pos = 1; pos < f.size(); ++pos) { + unsigned char c = f[f.size() - pos]; + if (c=='/' || c=='\\') + break; + if (std::isalpha(c)) { + alpha = true; + break; } - - // do not convert this path if there are no alpha characters (either pointless or cause wrong results for . and ..) - if (!alpha) - return false; - - // Convert char path to CHAR path - std::vector buf(f.size()+1U, 0); - for (unsigned int i = 0; i < f.size(); ++i) - buf[i] = f[i]; - - // Lookup filename or foldername on file system - WIN32_FIND_DATAA FindFileData; - HANDLE hFind = FindFirstFileA(&buf[0], &FindFileData); - if (hFind == INVALID_HANDLE_VALUE) - return false; - *result = FindFileData.cFileName; - FindClose(hFind); - return true; } - /** Change case in given path to match filesystem */ - std::string realFilename(const std::string &f) - { - std::string ret; - ret.reserve(f.size()); // this will be the final size + // do not convert this path if there are no alpha characters (either pointless or cause wrong results for . and ..) + if (!alpha) + return false; - // Current subpath - std::string subpath; + // Convert char path to CHAR path + std::vector buf(f.size()+1U, 0); + for (unsigned int i = 0; i < f.size(); ++i) + buf[i] = f[i]; - for (std::string::size_type pos = 0; pos < f.size(); ++pos) { - unsigned char c = f[pos]; + // Lookup filename or foldername on file system + WIN32_FIND_DATAA FindFileData; + HANDLE hFind = FindFirstFileA(&buf[0], &FindFileData); + if (hFind == INVALID_HANDLE_VALUE) + return false; + *result = FindFileData.cFileName; + FindClose(hFind); + return true; +} - // Separator.. add subpath and separator - if (c == '/' || c == '\\') { - // if subpath is empty just add separator - if (subpath.empty()) { - ret += c; - continue; - } +/** Change case in given path to match filesystem */ +static std::string realFilename(const std::string &f) +{ + std::string ret; + ret.reserve(f.size()); // this will be the final size - // Append real filename (proper case) - std::string f2; - if (realFileName(f.substr(0,pos),&f2)) - ret += f2; - else - ret += subpath; + // Current subpath + std::string subpath; - subpath.clear(); + for (std::string::size_type pos = 0; pos < f.size(); ++pos) { + unsigned char c = f[pos]; - // Append separator + // Separator.. add subpath and separator + if (c == '/' || c == '\\') { + // if subpath is empty just add separator + if (subpath.empty()) { ret += c; - } else { - subpath += c; + continue; } - } - if (!subpath.empty()) { + // Append real filename (proper case) std::string f2; - if (realFileName(f,&f2)) + if (realFileName(f.substr(0,pos),&f2)) ret += f2; else ret += subpath; - } - return ret; + subpath.clear(); + + // Append separator + ret += c; + } else { + subpath += c; + } } - bool isAbsolutePath(const std::string &path) - { - if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) - return true; - return path.length() > 1U && (path[0] == '/' || path[0] == '/'); + if (!subpath.empty()) { + std::string f2; + if (realFileName(f,&f2)) + ret += f2; + else + ret += subpath; } + return ret; +} + +static bool isAbsolutePath(const std::string &path) +{ + if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) + return true; + return path.length() > 1U && (path[0] == '/' || path[0] == '/'); +} #else #define realFilename(f) f - bool isAbsolutePath(const std::string &path) - { - return path.length() > 1U && path[0] == '/'; - } +static bool isAbsolutePath(const std::string &path) +{ + return path.length() > 1U && path[0] == '/'; +} #endif +namespace simplecpp { /** * perform path simplifications for . and .. */ From b0f718520d4bd7ca12d0a1c1eadeb07be51024d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 22 Jun 2017 21:51:31 +0200 Subject: [PATCH 012/381] Fixed compiler errors --- simplecpp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index cfb96c87..e8c835ca 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1941,7 +1941,7 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { - if (simplecpp::isAbsolutePath(header)) { + if (isAbsolutePath(header)) { f.open(header.c_str()); return f.is_open() ? simplecpp::simplifyPath(header) : ""; } @@ -1974,7 +1974,7 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { - if (simplecpp::isAbsolutePath(header)) { + if (isAbsolutePath(header)) { return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; } From 0699ec5b2ebef3740880efe3fbf3834a2ae46fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 22 Jun 2017 23:03:33 +0200 Subject: [PATCH 013/381] Fixed #41 (Test failures on cygwin) There was FAILED tests in cygwin but there was no bug in simplecpp. --- run-tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/run-tests.py b/run-tests.py index 0398de04..712ba5e6 100644 --- a/run-tests.py +++ b/run-tests.py @@ -31,11 +31,19 @@ def cleanup(out): # skipping tests.. skip = ['assembler-with-cpp.c', 'builtin_line.c', + 'c99-6_10_3_3_p4.c', + 'clang_headers.c', # missing include 'comment_save.c', # _Pragma 'has_attribute.c', + 'has_attribute.cpp', 'header_lookup1.c', # missing include 'line-directive-output.c', + 'macro_paste_hashhash.c', 'microsoft-ext.c', + 'normalize-3.c', # gcc has different output \uAC00 vs \U0000AC00 on cygwin/linux + 'pr63831-1.c', # __has_attribute => works differently on cygwin/linux + 'pr63831-2.c', # __has_attribute => works differently on cygwin/linux + 'pr65238-1.c', # __has_attribute => works differently on cygwin/linux '_Pragma-location.c', '_Pragma-dependency.c', '_Pragma-dependency2.c', @@ -43,8 +51,7 @@ def cleanup(out): 'pragma-pushpop-macro.c', # pragma push/pop 'x86_target_features.c', 'warn-disabled-macro-expansion.c', - 'c99-6_10_3_3_p4.c', - 'macro_paste_hashhash.c' + 'ucnid-2011-1.c' # \u00A8 generates different output on cygwin/linux ] todo = [ From 5be8f237fc0d3f78805e06ea0bfd0cd21ae196a8 Mon Sep 17 00:00:00 2001 From: x29a <0.x29a.0@gmail.com> Date: Fri, 23 Jun 2017 14:04:07 +0200 Subject: [PATCH 014/381] dont autocapitalize windows drive letters. this fixes issue #85 --- simplecpp.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e8c835ca..c03e8682 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1678,12 +1678,6 @@ namespace simplecpp { #ifdef SIMPLECPP_WINDOWS static bool realFileName(const std::string &f, std::string *result) { - // If path is a drive letter, uppercase it - if (f.size() == 2 && std::isalpha((unsigned char)f[0]) && f[1] == ':') { - *result = (char)std::toupper((unsigned char)f[0]) + std::string(":"); - return true; - } - // are there alpha characters in last subpath? bool alpha = false; for (std::string::size_type pos = 1; pos < f.size(); ++pos) { From 593509d20bcb830231974db3bcde3997b398f3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 23 Jun 2017 19:59:37 +0200 Subject: [PATCH 015/381] allow that simplecpp::simplifyPath is reused --- simplecpp.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simplecpp.h b/simplecpp.h index cefd3d1c..a30385c1 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -294,6 +294,9 @@ namespace simplecpp { * Deallocate data */ SIMPLECPP_LIB void cleanup(std::map &filedata); + + /** Simplify path */ + SIMPLECPP_LIB std::string simplifyPath(std::string path); } #endif From 406f3e5124f8fc412c1f67004285e91207c9cc16 Mon Sep 17 00:00:00 2001 From: orbitcowboy Date: Thu, 29 Jun 2017 14:45:03 +0200 Subject: [PATCH 016/381] Added missing '\' check in isAbsolutePath(). --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c03e8682..96d2d55b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1760,7 +1760,7 @@ static bool isAbsolutePath(const std::string &path) { if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) return true; - return path.length() > 1U && (path[0] == '/' || path[0] == '/'); + return path.length() > 1U && (path[0] == '/' || path[0] == '\\'); } #else #define realFilename(f) f From b5767105232e28ed8bac7d2c13ee6eda73b3cf78 Mon Sep 17 00:00:00 2001 From: Huemac Date: Mon, 31 Jul 2017 20:04:12 +0300 Subject: [PATCH 017/381] Fix Visual Studio Warning C4458 - declaration of 'files' hides class member --- simplecpp.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 96d2d55b..72d31812 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1023,25 +1023,25 @@ namespace simplecpp { /** * Expand macro. This will recursively expand inner macros. - * @param output destination tokenlist - * @param rawtok macro token - * @param macros list of macros - * @param files the files + * @param output destination tokenlist + * @param rawtok macro token + * @param macros list of macros + * @param inputFiles the input files * @return token after macro * @throw Can throw wrongNumberOfParameters or invalidHashHash */ const Token * expand(TokenList * const output, const Token * rawtok, const std::map ¯os, - std::vector &files) const { + std::vector &inputFiles) const { std::set expandedmacros; - TokenList output2(files); + TokenList output2(inputFiles); if (functionLike() && rawtok->next && rawtok->next->op == '(') { // Copy macro call to a new tokenlist with no linebreaks const Token * const rawtok1 = rawtok; - TokenList rawtokens2(files); + TokenList rawtokens2(inputFiles); rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); rawtok = rawtok->next; rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); @@ -1084,7 +1084,7 @@ namespace simplecpp { const std::map::const_iterator macro = macros.find(macro2tok->str); if (macro == macros.end() || !macro->second.functionLike()) break; - TokenList rawtokens2(files); + TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); while (macro2tok) { Token *next = macro2tok->next; From 48a34f77af902891b95e4b4bf7c78bdc6d12f094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Kr=C3=BCger?= Date: Wed, 9 Aug 2017 22:13:38 +0200 Subject: [PATCH 018/381] fix some cppcheck and compiler warnings (#93) * fix cppcheck findings: [simplecpp.h:182] -> [simplecpp.cpp:226]: (style, inconclusive) Function 'push_back' argument 1 names different: declaration 'token' definition 'tok'. [simplecpp.h:249] -> [simplecpp.cpp:872]: (style, inconclusive) Function 'constFoldQuestionOp' argument 1 names different: declaration 'tok' definition 'tok1'. [simplecpp.cpp:1643]: (performance, inconclusive) Technically the member function 'simplecpp::Macro::isReplaced' can be static. * makefile: add -Warning flags that cppcheck uses to simplecpp build. Namely: -Wabi -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow * fix -Wshadow warning. Was: main.cpp:65:35: warning: declaration shadows a local variable [-Wshadow] for (const simplecpp::Output &output : outputList) { ^ main.cpp:60:26: note: previous declaration is here simplecpp::TokenList output(files); ^ 1 warning generated. --- Makefile | 2 +- main.cpp | 6 +++--- simplecpp.cpp | 2 +- simplecpp.h | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 484ba137..222d23c7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -g -std=c++0x +CXXFLAGS = -Wall -Wextra -Wabi -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -pedantic -g -std=c++0x LDFLAGS = -g %.o: %.cpp diff --git a/main.cpp b/main.cpp index f4ca0af3..fd205a1d 100644 --- a/main.cpp +++ b/main.cpp @@ -57,11 +57,11 @@ int main(int argc, char **argv) std::map included = simplecpp::load(rawtokens, files, dui, &outputList); for (std::pair i : included) i.second->removeComments(); - simplecpp::TokenList output(files); - simplecpp::preprocess(output, rawtokens, files, included, dui, &outputList); + simplecpp::TokenList outputTokens(files); + simplecpp::preprocess(outputTokens, rawtokens, files, included, dui, &outputList); // Output - std::cout << output.stringify() << std::endl; + std::cout << outputTokens.stringify() << std::endl; for (const simplecpp::Output &output : outputList) { std::cerr << output.location.file() << ':' << output.location.line << ": "; switch (output.type) { diff --git a/simplecpp.cpp b/simplecpp.cpp index 72d31812..3b5142e2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1640,7 +1640,7 @@ namespace simplecpp { return nextTok; } - bool isReplaced(const std::set &expandedmacros) const { + static bool isReplaced(const std::set &expandedmacros) { // return true if size > 1 std::set::const_iterator it = expandedmacros.begin(); if (it == expandedmacros.end()) diff --git a/simplecpp.h b/simplecpp.h index a30385c1..9f4ab8b2 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -179,7 +179,7 @@ namespace simplecpp { bool empty() const { return !frontToken; } - void push_back(Token *token); + void push_back(Token *tok); void dump() const; std::string stringify() const; @@ -246,7 +246,7 @@ namespace simplecpp { void constFoldComparison(Token *tok); void constFoldBitwise(Token *tok); void constFoldLogicalOp(Token *tok); - void constFoldQuestionOp(Token **tok); + void constFoldQuestionOp(Token **tok1); std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList); From 971da162d1da59c439c4c132171271be327fcf1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 13 Aug 2017 11:19:09 +0200 Subject: [PATCH 019/381] return reference to self in operator= --- simplecpp.cpp | 15 ++++++++------- simplecpp.h | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3b5142e2..22fac508 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -202,14 +202,15 @@ simplecpp::TokenList::~TokenList() clear(); } -void simplecpp::TokenList::operator=(const TokenList &other) +simplecpp::TokenList &simplecpp::TokenList::operator=(const TokenList &other) { - if (this == &other) - return; - clear(); - for (const Token *tok = other.cfront(); tok; tok = tok->next) - push_back(new Token(*tok)); - sizeOfType = other.sizeOfType; + if (this != &other) { + clear(); + for (const Token *tok = other.cfront(); tok; tok = tok->next) + push_back(new Token(*tok)); + sizeOfType = other.sizeOfType; + } + return *this; } void simplecpp::TokenList::clear() diff --git a/simplecpp.h b/simplecpp.h index 9f4ab8b2..1bf97a30 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -173,7 +173,7 @@ namespace simplecpp { TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = 0); TokenList(const TokenList &other); ~TokenList(); - void operator=(const TokenList &other); + TokenList &operator=(const TokenList &other); void clear(); bool empty() const { From 1f69b193050d529815dc1d1ceab03091e64d2ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 13 Aug 2017 13:51:57 +0200 Subject: [PATCH 020/381] Thread safety: Avoid static data in method --- simplecpp.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 1bf97a30..7c973b62 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -77,14 +77,15 @@ namespace simplecpp { } const std::string& file() const { - static const std::string temp; - return fileIndex < files.size() ? files[fileIndex] : temp; + return fileIndex < files.size() ? files[fileIndex] : emptyFileName; } const std::vector &files; unsigned int fileIndex; unsigned int line; unsigned int col; + private: + const std::string emptyFileName; }; /** From 85f487518e098632c5bbe14accb2b5ff21d8e8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 13 Aug 2017 13:57:30 +0200 Subject: [PATCH 021/381] Fix Cppcheck warnings, missing copy constructor, missing assignment operator --- simplecpp.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/simplecpp.h b/simplecpp.h index 7c973b62..21fa5525 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -52,6 +52,8 @@ namespace simplecpp { public: explicit Location(const std::vector &f) : files(f), fileIndex(0), line(1U), col(0U) {} + Location(const Location &loc) : files(loc.files), fileIndex(loc.fileIndex), line(loc.line), col(loc.col) {} + Location &operator=(const Location &other) { if (this != &other) { fileIndex = other.fileIndex; @@ -148,6 +150,9 @@ namespace simplecpp { void printOut() const; private: TokenString string; + + // Not implemented - prevent assignment + Token &operator=(const Token &tok); }; /** Output from preprocessor */ From f704c1b42ff5b83cf831e03b7170dc4e1c91ce5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 15 Aug 2017 22:31:28 +0200 Subject: [PATCH 022/381] better diagnostics when fail to evaluate sizeof --- simplecpp.cpp | 14 ++++++++++---- test.cpp | 23 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 22fac508..9c147fa1 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1847,16 +1847,20 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::mapnext; if (!tok1) { - throw std::runtime_error("missed sizeof argument"); + throw std::runtime_error("missing sizeof argument"); } simplecpp::Token *tok2 = tok1->next; if (!tok2) { - throw std::runtime_error("missed sizeof argument"); + throw std::runtime_error("missing sizeof argument"); } if (tok1->op == '(') { tok1 = tok1->next; - while (tok2->op != ')') + while (tok2->op != ')') { tok2 = tok2->next; + if (!tok2) { + throw std::runtime_error("invalid sizeof expression"); + } + } } std::string type; @@ -2353,12 +2357,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } try { conditionIsTrue = (evaluate(expr, sizeOfType) != 0); - } catch (const std::exception &) { + } catch (const std::exception &e) { if (outputList) { Output out(rawtok->location.files); out.type = Output::SYNTAX_ERROR; out.location = rawtok->location; out.msg = "failed to evaluate " + std::string(rawtok->str == IF ? "#if" : "#elif") + " condition"; + if (e.what() && *e.what()) + out.msg += std::string(", ") + e.what(); outputList->push_back(out); } output.clear(); diff --git a/test.cpp b/test.cpp index 87bf0171..9626009f 100644 --- a/test.cpp +++ b/test.cpp @@ -1233,10 +1233,6 @@ void warning() ASSERT_EQUALS("file0,1,#warning,#warning MSG\n", toString(outputList)); } -namespace simplecpp { - std::string simplifyPath(std::string); -} - void simplifyPath() { ASSERT_EQUALS("1.c", simplecpp::simplifyPath("./1.c")); @@ -1323,6 +1319,23 @@ void simplifyPath_New() ASSERT_EQUALS("/", simplecpp::simplifyPath("\\")); } +void preprocessSizeOf() +{ + simplecpp::OutputList outputList; + + preprocess("#if 3 > sizeof", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); + + outputList.clear(); + + preprocess("#if 3 > sizeof A", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); + + outputList.clear(); + + preprocess("#if 3 > sizeof(int", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, invalid sizeof expression\n", toString(outputList)); +} int main(int argc, char **argv) { @@ -1447,5 +1460,7 @@ int main(int argc, char **argv) TEST_CASE(simplifyPath_cppcheck); TEST_CASE(simplifyPath_New); + TEST_CASE(preprocessSizeOf); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 0c58f2c216407446e0f4854d24cb599f9e29023f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 2 Sep 2017 10:43:42 +0200 Subject: [PATCH 023/381] ensure that endToken is assigned in Macro::parseDefine --- simplecpp.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9c147fa1..351c24fc 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1192,6 +1192,7 @@ namespace simplecpp { argtok = argtok->next; } if (!sameline(nametoken, argtok)) { + endToken = argtok ? argtok->previous : argtok; return false; } valueToken = argtok ? argtok->next : NULL; From 143b74130051e027ae745b8925d280e1188ebd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 7 Sep 2017 17:43:27 +0200 Subject: [PATCH 024/381] Handle C++14 digit separators (#98) --- simplecpp.cpp | 3 +++ test.cpp | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 351c24fc..4149a3f8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -449,9 +449,12 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // number or name if (isNameChar(ch)) { + const bool num = std::isdigit(ch); while (istr.good() && isNameChar(ch)) { currentToken += ch; ch = readChar(istr,bom); + if (num && ch=='\'' && isNameChar(peekChar(istr,bom))) + ch = readChar(istr,bom); } ungetChar(istr,bom); diff --git a/test.cpp b/test.cpp index 9626009f..26e6bc4c 100644 --- a/test.cpp +++ b/test.cpp @@ -1081,6 +1081,11 @@ void readfile_rawstring() ASSERT_EQUALS("A = \"a\nb\nc\";", readfile("A = R\"foo(a\nb\nc)foo\";")); } +void readfile_cpp14_number() +{ + ASSERT_EQUALS("A = 12345 ;", readfile("A = 12\'345;")); +} + void stringify1() { const char code_c[] = "#include \"A.h\"\n" @@ -1437,6 +1442,7 @@ int main(int argc, char **argv) TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_string); TEST_CASE(readfile_rawstring); + TEST_CASE(readfile_cpp14_number); TEST_CASE(stringify1); From e020598a662dcc3a62e0bbf856a5444779e04b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 7 Sep 2017 17:46:21 +0200 Subject: [PATCH 025/381] Fix compiler warnings --- test.cpp | 194 +++++++++++++++++++++++++++---------------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/test.cpp b/test.cpp index 26e6bc4c..39cd534d 100644 --- a/test.cpp +++ b/test.cpp @@ -97,7 +97,7 @@ static std::string toString(const simplecpp::OutputList &outputList) return ostr.str(); } -void backslash() +static void backslash() { // preprocessed differently simplecpp::OutputList outputList; @@ -114,7 +114,7 @@ void backslash() ASSERT_EQUALS("file0,1,portability_backslash,Combination 'backslash space newline' is not portable.\n", toString(outputList)); } -void builtin() +static void builtin() { ASSERT_EQUALS("\"\" 1 0", preprocess("__FILE__ __LINE__ __COUNTER__")); ASSERT_EQUALS("\n\n3", preprocess("\n\n__LINE__")); @@ -141,7 +141,7 @@ static std::string testConstFold(const char code[]) return expr.stringify(); } -void combineOperators_floatliteral() +static void combineOperators_floatliteral() { ASSERT_EQUALS("1.", preprocess("1.")); ASSERT_EQUALS("1.f", preprocess("1.f")); @@ -154,19 +154,19 @@ void combineOperators_floatliteral() ASSERT_EQUALS("0x1E + 7", preprocess("0x1E+7")); } -void combineOperators_increment() +static void combineOperators_increment() { ASSERT_EQUALS("; ++ x ;", preprocess(";++x;")); ASSERT_EQUALS("; x ++ ;", preprocess(";x++;")); ASSERT_EQUALS("1 + + 2", preprocess("1++2")); } -void combineOperators_coloncolon() +static void combineOperators_coloncolon() { ASSERT_EQUALS("x ? y : :: z", preprocess("x ? y : ::z")); } -void comment() +static void comment() { ASSERT_EQUALS("// abc", readfile("// abc")); ASSERT_EQUALS("", preprocess("// abc")); @@ -176,7 +176,7 @@ void comment() ASSERT_EQUALS("* p = a / * b / * c ;", preprocess("*p=a/ *b/ *c;")); } -void comment_multiline() +static void comment_multiline() { const char code[] = "#define ABC {// \\\n" "}\n" @@ -199,7 +199,7 @@ static void constFold() ASSERT_EQUALS("exception", testConstFold("?2:3")); } -void define1() +static void define1() { const char code[] = "#define A 1+2\n" "a=A+3;"; @@ -210,7 +210,7 @@ void define1() preprocess(code)); } -void define2() +static void define2() { const char code[] = "#define ADD(A,B) A+B\n" "ADD(1+2,3);"; @@ -221,7 +221,7 @@ void define2() preprocess(code)); } -void define3() +static void define3() { const char code[] = "#define A 123\n" "#define B A\n" @@ -234,7 +234,7 @@ void define3() preprocess(code)); } -void define4() +static void define4() { const char code[] = "#define A 123\n" "#define B(C) A\n" @@ -247,42 +247,42 @@ void define4() preprocess(code)); } -void define5() +static void define5() { const char code[] = "#define add(x,y) x+y\n" "add(add(1,2),3)"; ASSERT_EQUALS("\n1 + 2 + 3", preprocess(code)); } -void define6() +static void define6() { const char code[] = "#define A() 1\n" "A()"; ASSERT_EQUALS("\n1", preprocess(code)); } -void define7() +static void define7() { const char code[] = "#define A(X) X+1\n" "A(1 /*23*/)"; ASSERT_EQUALS("\n1 + 1", preprocess(code)); } -void define8() // 6.10.3.10 +static void define8() // 6.10.3.10 { const char code[] = "#define A(X) \n" "int A[10];"; ASSERT_EQUALS("\nint A [ 10 ] ;", preprocess(code)); } -void define9() +static void define9() { const char code[] = "#define AB ab.AB\n" "AB.CD\n"; ASSERT_EQUALS("\nab . AB . CD", preprocess(code)); } -void define_invalid_1() +static void define_invalid_1() { std::istringstream istr("#define A(\nB\n"); std::vector files; @@ -293,7 +293,7 @@ void define_invalid_1() ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } -void define_invalid_2() +static void define_invalid_2() { std::istringstream istr("#define\nhas#"); std::vector files; @@ -304,7 +304,7 @@ void define_invalid_2() ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } -void define_define_1() +static void define_define_1() { const char code[] = "#define A(x) (x+1)\n" "#define B A(\n" @@ -312,7 +312,7 @@ void define_define_1() ASSERT_EQUALS("\n\n( ( i ) + 1 )", preprocess(code)); } -void define_define_2() +static void define_define_2() { const char code[] = "#define A(m) n=m\n" "#define B(x) A(x)\n" @@ -320,7 +320,7 @@ void define_define_2() ASSERT_EQUALS("\n\nn = 0", preprocess(code)); } -void define_define_3() +static void define_define_3() { const char code[] = "#define ABC 123\n" "#define A(B) A##B\n" @@ -328,7 +328,7 @@ void define_define_3() ASSERT_EQUALS("\n\n123", preprocess(code)); } -void define_define_4() +static void define_define_4() { const char code[] = "#define FOO1()\n" "#define TEST(FOO) FOO FOO()\n" @@ -336,7 +336,7 @@ void define_define_4() ASSERT_EQUALS("\n\nFOO1", preprocess(code)); } -void define_define_5() +static void define_define_5() { const char code[] = "#define X() Y\n" "#define Y() X\n" @@ -345,7 +345,7 @@ void define_define_5() ASSERT_EQUALS("\n\nA : Y", preprocess(code)); // <- match the output from gcc/clang/vc } -void define_define_6() +static void define_define_6() { const char code1[] = "#define f(a) a*g\n" "#define g f\n" @@ -358,7 +358,7 @@ void define_define_6() ASSERT_EQUALS("\n\na : 2 * 9 * g", preprocess(code2)); } -void define_define_7() +static void define_define_7() { const char code[] = "#define f(x) g(x\n" "#define g(x) x()\n" @@ -366,7 +366,7 @@ void define_define_7() ASSERT_EQUALS("\n\nf ( )", preprocess(code)); } -void define_define_8() // line break in nested macro call +static void define_define_8() // line break in nested macro call { const char code[] = "#define A(X,Y) ((X)*(Y))\n" "#define B(X,Y) ((X)+(Y))\n" @@ -375,7 +375,7 @@ void define_define_8() // line break in nested macro call ASSERT_EQUALS("\n\n( ( 0 ) + ( ( ( 255 ) * ( x + y ) ) ) )", preprocess(code)); } -void define_define_9() // line break in nested macro call +static void define_define_9() // line break in nested macro call { const char code[] = "#define A(X) X\n" "#define B(X) X\n" @@ -383,7 +383,7 @@ void define_define_9() // line break in nested macro call ASSERT_EQUALS("\n\ndostuff ( 1 , 2 )", preprocess(code)); } -void define_define_10() +static void define_define_10() { const char code[] = "#define glue(a, b) a ## b\n" "#define xglue(a, b) glue(a, b)\n" @@ -393,7 +393,7 @@ void define_define_10() ASSERT_EQUALS("\n\n\n\n1 2", preprocess(code)); } -void define_define_11() +static void define_define_11() { const char code[] = "#define XY(x, y) x ## y\n" "#define XY2(x, y) XY(x, y)\n" @@ -403,7 +403,7 @@ void define_define_11() ASSERT_EQUALS("\n\n\n\nP2DIR ;", preprocess(code)); } -void define_define_12() +static void define_define_12() { const char code[] = "#define XY(Z) Z\n" "#define X(ID) X##ID(0)\n" @@ -411,39 +411,39 @@ void define_define_12() ASSERT_EQUALS("\n\n0", preprocess(code)); } -void define_va_args_1() +static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" "A(1,2);"; ASSERT_EQUALS("\ndostuff ( 1 , 2 ) ;", preprocess(code)); } -void define_va_args_2() +static void define_va_args_2() { const char code[] = "#define A(X,...) X(#__VA_ARGS__)\n" "A(f,123);"; ASSERT_EQUALS("\nf ( \"123\" ) ;", preprocess(code)); } -void define_va_args_3() // min number of arguments +static void define_va_args_3() // min number of arguments { const char code[] = "#define A(x, y, z...) 1\n" "A(1, 2)\n"; ASSERT_EQUALS("\n1", preprocess(code)); } -void dollar() +static void dollar() { ASSERT_EQUALS("$ab", readfile("$ab")); ASSERT_EQUALS("a$b", readfile("a$b")); } -void dotDotDot() +static void dotDotDot() { ASSERT_EQUALS("1 . . . 2", readfile("1 ... 2")); } -void error() +static void error() { std::istringstream istr("#error hello world! \n"); std::vector files; @@ -454,7 +454,7 @@ void error() ASSERT_EQUALS("file0,1,#error,#error hello world!\n", toString(outputList)); } -void garbage() +static void garbage() { const simplecpp::DUI dui; simplecpp::OutputList outputList; @@ -472,7 +472,7 @@ void garbage() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'CON', Invalid ## usage when expanding 'CON'.\n", toString(outputList)); } -void garbage_endif() +static void garbage_endif() { const simplecpp::DUI dui; simplecpp::OutputList outputList; @@ -490,7 +490,7 @@ void garbage_endif() ASSERT_EQUALS("file0,1,syntax_error,#endif without #if\n", toString(outputList)); } -void hash() +static void hash() { ASSERT_EQUALS("x = \"1\"", preprocess("x=#__LINE__")); @@ -509,14 +509,14 @@ void hash() "B(123)")); } -void hashhash1() // #4703 +static void hashhash1() // #4703 { const char code[] = "#define MACRO( A, B, C ) class A##B##C##Creator {};\n" "MACRO( B\t, U , G )"; ASSERT_EQUALS("\nclass BUGCreator { } ;", preprocess(code)); } -void hashhash2() +static void hashhash2() { const char code[] = "#define A(x) a##x\n" "#define B 0\n" @@ -524,7 +524,7 @@ void hashhash2() ASSERT_EQUALS("\n\naB", preprocess(code)); } -void hashhash3() +static void hashhash3() { const char code[] = "#define A(B) A##B\n" "#define a(B) A(B)\n" @@ -532,7 +532,7 @@ void hashhash3() ASSERT_EQUALS("\n\nAAB", preprocess(code)); } -void hashhash4() // nonstandard gcc/clang extension for empty varargs +static void hashhash4() // nonstandard gcc/clang extension for empty varargs { const char *code; @@ -546,12 +546,12 @@ void hashhash4() // nonstandard gcc/clang extension for empty varargs ASSERT_EQUALS("\n\na ( 1 ) ;", preprocess(code)); } -void hashhash5() +static void hashhash5() { ASSERT_EQUALS("x1", preprocess("x##__LINE__")); } -void hashhash6() +static void hashhash6() { const char *code; @@ -566,7 +566,7 @@ void hashhash6() ASSERT_EQUALS("\n\n\nLOG ( 1 , ( int ) 2 )", preprocess(code)); } -void hashhash7() // # ## # (C standard; 6.10.3.3.p4) +static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) { const char *code; @@ -576,7 +576,7 @@ void hashhash7() // # ## # (C standard; 6.10.3.3.p4) } -void ifdef1() +static void ifdef1() { const char code[] = "#ifdef A\n" "1\n" @@ -586,7 +586,7 @@ void ifdef1() ASSERT_EQUALS("\n\n\n2", preprocess(code)); } -void ifdef2() +static void ifdef2() { const char code[] = "#define A\n" "#ifdef A\n" @@ -597,7 +597,7 @@ void ifdef2() ASSERT_EQUALS("\n\n1", preprocess(code)); } -void ifndef() +static void ifndef() { const char code1[] = "#define A\n" "#ifndef A\n" @@ -611,7 +611,7 @@ void ifndef() ASSERT_EQUALS("\n1", preprocess(code2)); } -void ifA() +static void ifA() { const char code[] = "#if A==1\n" "X\n" @@ -623,7 +623,7 @@ void ifA() ASSERT_EQUALS("\nX", preprocess(code, dui)); } -void ifCharLiteral() +static void ifCharLiteral() { const char code[] = "#if ('A'==0x41)\n" "123\n" @@ -631,7 +631,7 @@ void ifCharLiteral() ASSERT_EQUALS("\n123", preprocess(code)); } -void ifDefined() +static void ifDefined() { const char code[] = "#if defined(A)\n" "X\n" @@ -642,7 +642,7 @@ void ifDefined() ASSERT_EQUALS("\nX", preprocess(code, dui)); } -void ifDefinedNoPar() +static void ifDefinedNoPar() { const char code[] = "#if defined A\n" "X\n" @@ -653,7 +653,7 @@ void ifDefinedNoPar() ASSERT_EQUALS("\nX", preprocess(code, dui)); } -void ifDefinedNested() +static void ifDefinedNested() { const char code[] = "#define FOODEF defined(FOO)\n" "#if FOODEF\n" @@ -665,7 +665,7 @@ void ifDefinedNested() ASSERT_EQUALS("\n\nX", preprocess(code, dui)); } -void ifDefinedNestedNoPar() +static void ifDefinedNestedNoPar() { const char code[] = "#define FOODEF defined FOO\n" "#if FOODEF\n" @@ -677,7 +677,7 @@ void ifDefinedNestedNoPar() ASSERT_EQUALS("\n\nX", preprocess(code, dui)); } -void ifDefinedInvalid1() // #50 - invalid unterminated defined +static void ifDefinedInvalid1() // #50 - invalid unterminated defined { const char code[] = "#if defined(A"; simplecpp::DUI dui; @@ -690,7 +690,7 @@ void ifDefinedInvalid1() // #50 - invalid unterminated defined ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } -void ifDefinedInvalid2() +static void ifDefinedInvalid2() { const char code[] = "#if defined"; simplecpp::DUI dui; @@ -703,7 +703,7 @@ void ifDefinedInvalid2() ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } -void ifLogical() +static void ifLogical() { const char code[] = "#if defined(A) || defined(B)\n" "X\n" @@ -718,7 +718,7 @@ void ifLogical() ASSERT_EQUALS("\nX", preprocess(code, dui)); } -void ifSizeof() +static void ifSizeof() { const char code[] = "#if sizeof(unsigned short)==2\n" "X\n" @@ -728,7 +728,7 @@ void ifSizeof() ASSERT_EQUALS("\nX", preprocess(code)); } -void elif() +static void elif() { const char code1[] = "#ifndef X\n" "1\n" @@ -758,7 +758,7 @@ void elif() ASSERT_EQUALS("\n\n\n\n\n3", preprocess(code3)); } -void ifif() +static void ifif() { // source code from LLVM const char code[] = "#if defined(__has_include)\n" @@ -768,7 +768,7 @@ void ifif() ASSERT_EQUALS("", preprocess(code)); } -void ifoverflow() +static void ifoverflow() { // source code from CLANG const char code[] = "#if 0x7FFFFFFFFFFFFFFF*2\n" @@ -787,7 +787,7 @@ void ifoverflow() (void)preprocess(code); } -void ifdiv0() +static void ifdiv0() { const char code[] = "#if 1000/0\n" "#endif\n" @@ -795,7 +795,7 @@ void ifdiv0() ASSERT_EQUALS("", preprocess(code)); } -void ifalt() // using "and", "or", etc +static void ifalt() // using "and", "or", etc { const char *code; @@ -814,7 +814,7 @@ void ifalt() // using "and", "or", etc ASSERT_EQUALS("\n1", preprocess(code)); } -void missingHeader1() +static void missingHeader1() { const simplecpp::DUI dui; std::istringstream istr("#include \"notexist.h\"\n"); @@ -826,7 +826,7 @@ void missingHeader1() ASSERT_EQUALS("file0,1,missing_header,Header not found: \"notexist.h\"\n", toString(outputList)); } -void missingHeader2() +static void missingHeader2() { const simplecpp::DUI dui; std::istringstream istr("#include \"foo.h\"\n"); // this file exists @@ -839,7 +839,7 @@ void missingHeader2() ASSERT_EQUALS("", toString(outputList)); } -void missingHeader3() +static void missingHeader3() { const simplecpp::DUI dui; std::istringstream istr("#ifdef UNDEFINED\n#include \"notexist.h\"\n#endif\n"); // this file is not included @@ -851,7 +851,7 @@ void missingHeader3() ASSERT_EQUALS("", toString(outputList)); } -void nestedInclude() +static void nestedInclude() { std::istringstream istr("#include \"test.h\"\n"); std::vector files; @@ -867,7 +867,7 @@ void nestedInclude() ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } -void multiline1() +static void multiline1() { const char code[] = "#define A \\\n" "1\n" @@ -881,7 +881,7 @@ void multiline1() ASSERT_EQUALS("\n\n1", tokens2.stringify()); } -void multiline2() +static void multiline2() { const char code[] = "#define A /*\\\n" "*/1\n" @@ -898,7 +898,7 @@ void multiline2() ASSERT_EQUALS("\n\n1", tokens2.stringify()); } -void multiline3() // #28 - macro with multiline comment +static void multiline3() // #28 - macro with multiline comment { const char code[] = "#define A /*\\\n" " */ 1\n" @@ -915,7 +915,7 @@ void multiline3() // #28 - macro with multiline comment ASSERT_EQUALS("\n\n1", tokens2.stringify()); } -void multiline4() // #28 - macro with multiline comment +static void multiline4() // #28 - macro with multiline comment { const char code[] = "#define A \\\n" " /*\\\n" @@ -933,7 +933,7 @@ void multiline4() // #28 - macro with multiline comment ASSERT_EQUALS("\n\n\n1", tokens2.stringify()); } -void multiline5() // column +static void multiline5() // column { const char code[] = "#define A\\\n" "("; @@ -945,19 +945,19 @@ void multiline5() // column ASSERT_EQUALS(11, rawtokens.back()->location.col); } -void include1() +static void include1() { const char code[] = "#include \"A.h\"\n"; ASSERT_EQUALS("# include \"A.h\"", readfile(code)); } -void include2() +static void include2() { const char code[] = "#include \n"; ASSERT_EQUALS("# include ", readfile(code)); } -void include3() // #16 - crash when expanding macro from header +static void include3() // #16 - crash when expanding macro from header { const char code_c[] = "#include \"A.h\"\n" "glue(1,2,3,4)\n" ; @@ -986,7 +986,7 @@ void include3() // #16 - crash when expanding macro from header } -void include4() // #27 - -include +static void include4() // #27 - -include { const char code_c[] = "X\n" ; const char code_h[] = "#define X 123\n"; @@ -1015,7 +1015,7 @@ void include4() // #27 - -include ASSERT_EQUALS("123", out.stringify()); } -void include5() // #3 - handle #include MACRO +static void include5() // #3 - handle #include MACRO { const char code_c[] = "#define A \"3.h\"\n#include A\n"; const char code_h[] = "123\n"; @@ -1037,7 +1037,7 @@ void include5() // #3 - handle #include MACRO ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } -void include6() // #57 - incomplete macro #include MACRO(,) +static void include6() // #57 - incomplete macro #include MACRO(,) { const char code[] = "#define MACRO(X,Y) X##Y\n#include MACRO(,)\n"; @@ -1053,7 +1053,7 @@ void include6() // #57 - incomplete macro #include MACRO(,) simplecpp::preprocess(out, rawtokens, files, filedata, dui); } -void readfile_nullbyte() +static void readfile_nullbyte() { const char code[] = "ab\0cd"; simplecpp::OutputList outputList; @@ -1061,7 +1061,7 @@ void readfile_nullbyte() ASSERT_EQUALS(true, outputList.empty()); // should warning be written? } -void readfile_string() +static void readfile_string() { const char code[] = "A = \"abc\'def\""; ASSERT_EQUALS("A = \"abc\'def\"", readfile(code)); @@ -1070,7 +1070,7 @@ void readfile_string() ASSERT_EQUALS("x = \"a b\"\n;", readfile("x=\"a\\\r\n b\";")); } -void readfile_rawstring() +static void readfile_rawstring() { ASSERT_EQUALS("A = \"abc\\\\\\\\def\"", readfile("A = R\"(abc\\\\def)\"")); ASSERT_EQUALS("A = \"abc\\\\\\\\def\"", readfile("A = R\"x(abc\\\\def)x\"")); @@ -1081,12 +1081,12 @@ void readfile_rawstring() ASSERT_EQUALS("A = \"a\nb\nc\";", readfile("A = R\"foo(a\nb\nc)foo\";")); } -void readfile_cpp14_number() +static void readfile_cpp14_number() { ASSERT_EQUALS("A = 12345 ;", readfile("A = 12\'345;")); } -void stringify1() +static void stringify1() { const char code_c[] = "#include \"A.h\"\n" "#include \"A.h\"\n"; @@ -1114,7 +1114,7 @@ void stringify1() ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); } -void tokenMacro1() +static void tokenMacro1() { const char code[] = "#define A 123\n" "A"; @@ -1127,7 +1127,7 @@ void tokenMacro1() ASSERT_EQUALS("A", tokenList.cback()->macro); } -void tokenMacro2() +static void tokenMacro2() { const char code[] = "#define ADD(X,Y) X+Y\n" "ADD(1,2)"; @@ -1148,7 +1148,7 @@ void tokenMacro2() ASSERT_EQUALS("", tok->macro); } -void tokenMacro3() +static void tokenMacro3() { const char code[] = "#define ADD(X,Y) X+Y\n" "#define FRED 1\n" @@ -1170,7 +1170,7 @@ void tokenMacro3() ASSERT_EQUALS("", tok->macro); } -void tokenMacro4() +static void tokenMacro4() { const char code[] = "#define A B\n" "#define B 1\n" @@ -1186,7 +1186,7 @@ void tokenMacro4() ASSERT_EQUALS("A", tok->macro); } -void undef() +static void undef() { std::istringstream istr("#define A\n" "#undef A\n" @@ -1201,7 +1201,7 @@ void undef() ASSERT_EQUALS("", tokenList.stringify()); } -void userdef() +static void userdef() { std::istringstream istr("#ifdef A\n123\n#endif\n"); simplecpp::DUI dui; @@ -1214,19 +1214,19 @@ void userdef() ASSERT_EQUALS("\n123", tokens2.stringify()); } -void utf8() +static void utf8() { ASSERT_EQUALS("123", readfile("\xEF\xBB\xBF 123")); } -void unicode() +static void unicode() { ASSERT_EQUALS("12", readfile("\xFE\xFF\x00\x31\x00\x32", 6)); ASSERT_EQUALS("12", readfile("\xFF\xFE\x31\x00\x32\x00", 6)); ASSERT_EQUALS("\n//1", readfile("\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00",16)); } -void warning() +static void warning() { std::istringstream istr("#warning MSG\n1"); std::vector files; @@ -1238,7 +1238,7 @@ void warning() ASSERT_EQUALS("file0,1,#warning,#warning MSG\n", toString(outputList)); } -void simplifyPath() +static void simplifyPath() { ASSERT_EQUALS("1.c", simplecpp::simplifyPath("./1.c")); ASSERT_EQUALS("1.c", simplecpp::simplifyPath("././1.c")); @@ -1267,7 +1267,7 @@ void simplifyPath() // tests transferred from cppcheck // https://github.com/danmar/cppcheck/blob/d3e79b71b5ec6e641ca3e516cfced623b27988af/test/testpath.cpp#L43 -void simplifyPath_cppcheck() +static void simplifyPath_cppcheck() { ASSERT_EQUALS("index.h", simplecpp::simplifyPath("index.h")); ASSERT_EQUALS("index.h", simplecpp::simplifyPath("./index.h")); @@ -1315,7 +1315,7 @@ void simplifyPath_cppcheck() ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("///src/test.cpp")); } -void simplifyPath_New() +static void simplifyPath_New() { ASSERT_EQUALS("", simplecpp::simplifyPath("")); ASSERT_EQUALS("/", simplecpp::simplifyPath("/")); @@ -1324,7 +1324,7 @@ void simplifyPath_New() ASSERT_EQUALS("/", simplecpp::simplifyPath("\\")); } -void preprocessSizeOf() +static void preprocessSizeOf() { simplecpp::OutputList outputList; From 2e2b7e61b2d229a42ed616a59851c6cb1cf3b263 Mon Sep 17 00:00:00 2001 From: Dmitry-Me Date: Sat, 9 Sep 2017 00:15:54 +0300 Subject: [PATCH 026/381] Ensure valueToken is assigned in Macro::parseDefine (#100) --- simplecpp.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4149a3f8..3309f271 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1196,6 +1196,7 @@ namespace simplecpp { } if (!sameline(nametoken, argtok)) { endToken = argtok ? argtok->previous : argtok; + valueToken = NULL; return false; } valueToken = argtok ? argtok->next : NULL; From de91025ea76f41330a22a0e576228ca846b6acdb Mon Sep 17 00:00:00 2001 From: Lars Ljung Date: Fri, 8 Sep 2017 23:17:42 +0200 Subject: [PATCH 027/381] Handle constant folding for the shift operators (#99) --- simplecpp.cpp | 24 ++++++++++++++++++++++++ simplecpp.h | 1 + 2 files changed, 25 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3309f271..0b3da388 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -589,6 +589,7 @@ void simplecpp::TokenList::constFold() constFoldUnaryNotPosNeg(tok); constFoldMulDivRem(tok); constFoldAddSub(tok); + constFoldShift(tok); constFoldComparison(tok); constFoldBitwise(tok); constFoldLogicalOp(tok); @@ -769,6 +770,29 @@ void simplecpp::TokenList::constFoldAddSub(Token *tok) } } +void simplecpp::TokenList::constFoldShift(Token *tok) +{ + for (; tok && tok->op != ')'; tok = tok->next) { + if (!tok->previous || !tok->previous->number) + continue; + if (!tok->next || !tok->next->number) + continue; + + long long result; + if (tok->str == "<<") + result = stringToLL(tok->previous->str) << stringToLL(tok->next->str); + else if (tok->str == ">>") + result = stringToLL(tok->previous->str) >> stringToLL(tok->next->str); + else + continue; + + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); + } +} + static const std::string NOTEQ("not_eq"); void simplecpp::TokenList::constFoldComparison(Token *tok) { diff --git a/simplecpp.h b/simplecpp.h index 21fa5525..2e608d22 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -249,6 +249,7 @@ namespace simplecpp { void constFoldUnaryNotPosNeg(Token *tok); void constFoldMulDivRem(Token *tok); void constFoldAddSub(Token *tok); + void constFoldShift(Token *tok); void constFoldComparison(Token *tok); void constFoldBitwise(Token *tok); void constFoldLogicalOp(Token *tok); From a7606e047fa1be45e4b10e59c8f478bb159ac197 Mon Sep 17 00:00:00 2001 From: Dmitry-Me Date: Sat, 9 Sep 2017 00:19:14 +0300 Subject: [PATCH 028/381] Omit repeated search (#101) --- simplecpp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0b3da388..1c44c6c5 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1287,8 +1287,8 @@ namespace simplecpp { } else { if (!expandArg(tokens, tok, tok->location, macros, expandedmacros, parametertokens)) { bool expanded = false; - if (macros.find(tok->str) != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { - const std::map::const_iterator it = macros.find(tok->str); + const std::map::const_iterator it = macros.find(tok->str); + if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { m.expand(tokens, tok, macros, files); From 7129b11963156d7a35f4495fc5aab3b341512e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 12 Sep 2017 22:26:25 +0200 Subject: [PATCH 029/381] Reject unicode and extended ascii code --- simplecpp.cpp | 16 +++++++++++++++- simplecpp.h | 3 ++- test.cpp | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1c44c6c5..e290421a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -401,6 +401,20 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (ch < ' ' && ch != '\t' && ch != '\n' && ch != '\r') ch = ' '; + if (ch >= 0x80) { + if (outputList) { + simplecpp::Output err(files); + err.type = simplecpp::Output::UNHANDLED_CHAR_ERROR; + err.location = location; + std::ostringstream s; + s << (int)ch; + err.msg = "The code contains unhandled character(s) (character code=" + s.str() + "). Neither unicode nor extended ascii is supported."; + outputList->push_back(err); + } + clear(); + return; + } + if (ch == '\n') { if (cback() && cback()->op == '\\') { if (location.col > cback()->location.col + 1U) @@ -1288,7 +1302,7 @@ namespace simplecpp { if (!expandArg(tokens, tok, tok->location, macros, expandedmacros, parametertokens)) { bool expanded = false; const std::map::const_iterator it = macros.find(tok->str); - if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { + if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { m.expand(tokens, tok, macros, files); diff --git a/simplecpp.h b/simplecpp.h index 2e608d22..c74ccf89 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -164,7 +164,8 @@ namespace simplecpp { MISSING_HEADER, INCLUDE_NESTED_TOO_DEEPLY, SYNTAX_ERROR, - PORTABILITY_BACKSLASH + PORTABILITY_BACKSLASH, + UNHANDLED_CHAR_ERROR } type; Location location; std::string msg; diff --git a/test.cpp b/test.cpp index 39cd534d..1a84e424 100644 --- a/test.cpp +++ b/test.cpp @@ -90,6 +90,8 @@ static std::string toString(const simplecpp::OutputList &outputList) case simplecpp::Output::Type::PORTABILITY_BACKSLASH: ostr << "portability_backslash,"; break; + case simplecpp::Output::Type::UNHANDLED_CHAR_ERROR: + ostr << "unhandled_char_error,"; } ostr << output.msg << '\n'; @@ -1086,6 +1088,17 @@ static void readfile_cpp14_number() ASSERT_EQUALS("A = 12345 ;", readfile("A = 12\'345;")); } +static void readfile_unhandled_chars() +{ + simplecpp::OutputList outputList; + readfile("// 你好世界", -1, &outputList); + ASSERT_EQUALS("", toString(outputList)); + readfile("s=\"你好世界\"", -1, &outputList); + ASSERT_EQUALS("", toString(outputList)); + readfile("int 你好世界=0;", -1, &outputList); + ASSERT_EQUALS("file0,1,unhandled_char_error,The code contains unhandled character(s) (character code=228). Neither unicode nor extended ascii is supported.\n", toString(outputList)); +} + static void stringify1() { const char code_c[] = "#include \"A.h\"\n" @@ -1443,6 +1456,7 @@ int main(int argc, char **argv) TEST_CASE(readfile_string); TEST_CASE(readfile_rawstring); TEST_CASE(readfile_cpp14_number); + TEST_CASE(readfile_unhandled_chars); TEST_CASE(stringify1); From 3ddd94b8ebcfed6e437b873663d0b3512f962e42 Mon Sep 17 00:00:00 2001 From: Lars Ljung Date: Thu, 14 Sep 2017 09:14:02 +0200 Subject: [PATCH 030/381] Support for bitwise not (#102) * Handle constant folding for the shift operators * Support for bitwise not. --- run-tests.py | 1 - simplecpp.cpp | 13 ++++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/run-tests.py b/run-tests.py index 712ba5e6..7ad35dfd 100644 --- a/run-tests.py +++ b/run-tests.py @@ -69,7 +69,6 @@ def cleanup(out): # todo, high priority 'c99-6_10_3_4_p5.c', 'c99-6_10_3_4_p6.c', - 'cxx_compl.cpp', # if A compl B 'cxx_oper_keyword_ms_compat.cpp', 'expr_usual_conversions.c', # condition is true: 4U - 30 >= 0 'stdint.c', diff --git a/simplecpp.cpp b/simplecpp.cpp index e290421a..be3bf59b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -699,6 +699,7 @@ void simplecpp::TokenList::combineOperators() } } +static const std::string COMPL("compl"); static const std::string NOT("not"); void simplecpp::TokenList::constFoldUnaryNotPosNeg(simplecpp::Token *tok) { @@ -706,10 +707,16 @@ void simplecpp::TokenList::constFoldUnaryNotPosNeg(simplecpp::Token *tok) // "not" might be ! if (isAlternativeUnaryOp(tok, NOT)) tok->op = '!'; + // "compl" might be ~ + else if (isAlternativeUnaryOp(tok, COMPL)) + tok->op = '~'; if (tok->op == '!' && tok->next && tok->next->number) { tok->setstr(tok->next->str == "0" ? "1" : "0"); deleteToken(tok->next); + } else if (tok->op == '~' && tok->next && tok->next->number) { + tok->setstr(toString(~stringToLL(tok->next->str))); + deleteToken(tok->next); } else { if (tok->previous && (tok->previous->number || tok->previous->name)) continue; @@ -1929,15 +1936,15 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::map altop(&altopData[0], &altopData[7]); +static const char * const altopData[] = {"and","or","bitand","bitor","compl","not","not_eq","xor"}; +static const std::set altop(&altopData[0], &altopData[8]); static void simplifyName(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->name) { if (altop.find(tok->str) != altop.end()) { bool alt; - if (tok->str == "not") { + if (tok->str == "not" || tok->str == "compl") { alt = isAlternativeUnaryOp(tok,tok->str); } else { alt = isAlternativeBinaryOp(tok,tok->str); From 12993c05d6a77e388c0bdb376d54a9f716e7435e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 15 Sep 2017 22:48:10 +0200 Subject: [PATCH 031/381] add missing enum constant in switch --- main.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.cpp b/main.cpp index fd205a1d..68f3cc70 100644 --- a/main.cpp +++ b/main.cpp @@ -83,6 +83,9 @@ int main(int argc, char **argv) case simplecpp::Output::PORTABILITY_BACKSLASH: std::cerr << "portability: "; break; + case simplecpp::Output::UNHANDLED_CHAR_ERROR: + std::cerr << "unhandled char error: "; + break; } std::cerr << output.msg << std::endl; } From b9944cb6dc6df9a636b3c521145ac2a163b97273 Mon Sep 17 00:00:00 2001 From: Dmitry-Me Date: Sat, 23 Sep 2017 17:00:18 +0300 Subject: [PATCH 032/381] Cast result of istream::get() uniformly with surrounding code (#104) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index be3bf59b..14419d8c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -338,7 +338,7 @@ static unsigned short getAndSkipBOM(std::istream &istr) // Skip UTF-8 BOM 0xefbbbf if (ch1 == 0xef) { - istr.get(); + (void)istr.get(); if (istr.get() == 0xbb && istr.peek() == 0xbf) { (void)istr.get(); } else { From b32e778720095111c53b3f2485880145962465e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 5 Nov 2017 11:06:23 +0100 Subject: [PATCH 033/381] Write comment for DUI. Fixes #106 --- simplecpp.h | 4 ++++ 1 file changed, 4 insertions(+) mode change 100644 => 100755 simplecpp.h diff --git a/simplecpp.h b/simplecpp.h old mode 100644 new mode 100755 index c74ccf89..19624e1d --- a/simplecpp.h +++ b/simplecpp.h @@ -275,6 +275,10 @@ namespace simplecpp { Location useLocation; }; + /** + * Command line preprocessor settings. + * On the command line these are configured by -D, -U, -I, --include + */ struct SIMPLECPP_LIB DUI { DUI() {} std::list defines; From 2425abce39c4eb82f457134e6ea0e00fc435b0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 29 Nov 2017 11:15:07 +0100 Subject: [PATCH 034/381] Refactoring #if for windows --- simplecpp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 14419d8c..0ba4faf3 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -17,6 +17,7 @@ */ #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) +#define SIMPLECPP_WINDOWS #define NOMINMAX #endif #include "simplecpp.h" @@ -33,11 +34,10 @@ #include #include -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) +#ifdef SIMPLECPP_WINDOWS #include #undef ERROR #undef TRUE -#define SIMPLECPP_WINDOWS #endif static bool isHex(const std::string &s) From d2f5d34c40a0966082246fe58b284e73321919d3 Mon Sep 17 00:00:00 2001 From: BNT Date: Tue, 5 Dec 2017 20:41:15 +0100 Subject: [PATCH 035/381] Improve performance on windows (#110) * use faster WIN API if possible * return early if nothing searchable exists * remove conversion to vector of char because const char* (via c_str()) is LPCSTR * implement common (http://en.cppreference.com/w/cpp/filesystem/path step 1) quick return part of algo for empty paths * couldnt find any performance diff with FIND_FIRST_EX_LARGE_FETCH so took the check for OS version out because all other OS support the call with NULL. Also removes need for static * cleanup --- simplecpp.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0ba4faf3..9a9551c1 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1745,15 +1745,11 @@ static bool realFileName(const std::string &f, std::string *result) if (!alpha) return false; - // Convert char path to CHAR path - std::vector buf(f.size()+1U, 0); - for (unsigned int i = 0; i < f.size(); ++i) - buf[i] = f[i]; - // Lookup filename or foldername on file system WIN32_FIND_DATAA FindFileData; - HANDLE hFind = FindFirstFileA(&buf[0], &FindFileData); - if (hFind == INVALID_HANDLE_VALUE) + HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); + + if (INVALID_HANDLE_VALUE == hFind) return false; *result = FindFileData.cFileName; FindClose(hFind); @@ -1828,6 +1824,9 @@ namespace simplecpp { */ std::string simplifyPath(std::string path) { + if (path.empty()) + return path; + std::string::size_type pos; // replace backslash separators @@ -2023,6 +2022,9 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { + if (filedata.empty()) { + return ""; + } if (isAbsolutePath(header)) { return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; } From 26bc3c9b73c55cf2be0b4034f3dc41edbd4d373d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 13 Jan 2018 16:09:22 +0100 Subject: [PATCH 036/381] Update astyle to version 3.0.1 --- main.cpp | 2 +- runastyle | 6 +++--- simplecpp.cpp | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/main.cpp b/main.cpp index 68f3cc70..2f513083 100644 --- a/main.cpp +++ b/main.cpp @@ -85,7 +85,7 @@ int main(int argc, char **argv) break; case simplecpp::Output::UNHANDLED_CHAR_ERROR: std::cerr << "unhandled char error: "; - break; + break; } std::cerr << output.msg << std::endl; } diff --git a/runastyle b/runastyle index c8183fd9..64298273 100755 --- a/runastyle +++ b/runastyle @@ -6,7 +6,7 @@ # If project management wishes to take a newer astyle version into use # just change this string to match the start of astyle version string. -ASTYLE_VERSION="Artistic Style Version 2.05.1" +ASTYLE_VERSION="Artistic Style Version 3.0.1" ASTYLE="astyle" DETECTED_VERSION=`$ASTYLE --version 2>&1` @@ -16,8 +16,8 @@ if [[ "$DETECTED_VERSION" != ${ASTYLE_VERSION}* ]]; then exit 1; fi -style="--style=stroustrup --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0" -options="--options=none --pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines" +style="--style=kr --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0" +options="--options=none --pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines --attach-classes --attach-namespaces" $ASTYLE $style $options *.cpp $ASTYLE $style $options *.h diff --git a/simplecpp.cpp b/simplecpp.cpp index 9a9551c1..672ec005 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -715,8 +715,8 @@ void simplecpp::TokenList::constFoldUnaryNotPosNeg(simplecpp::Token *tok) tok->setstr(tok->next->str == "0" ? "1" : "0"); deleteToken(tok->next); } else if (tok->op == '~' && tok->next && tok->next->number) { - tok->setstr(toString(~stringToLL(tok->next->str))); - deleteToken(tok->next); + tok->setstr(toString(~stringToLL(tok->next->str))); + deleteToken(tok->next); } else { if (tok->previous && (tok->previous->number || tok->previous->name)) continue; @@ -1747,7 +1747,7 @@ static bool realFileName(const std::string &f, std::string *result) // Lookup filename or foldername on file system WIN32_FIND_DATAA FindFileData; - HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); + HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); if (INVALID_HANDLE_VALUE == hFind) return false; @@ -2022,9 +2022,9 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { - if (filedata.empty()) { - return ""; - } + if (filedata.empty()) { + return ""; + } if (isAbsolutePath(header)) { return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; } From 5f19e505a149e40387c4bf3586389f3d12730e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 13 Jan 2018 16:34:55 +0100 Subject: [PATCH 037/381] Fix handling of ##. Fixes #114 --- simplecpp.cpp | 7 ++++++- test.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 672ec005..a4c720d2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1669,7 +1669,12 @@ namespace simplecpp { if (varargs && tokensB.empty() && tok->previous->str == ",") output->deleteToken(A); - else { + else if (strAB != "," && macros.find(strAB) == macros.end()) { + A->setstr(strAB); + for (Token *b = tokensB.front(); b; b = b->next) + b->location = loc; + output->takeTokens(tokensB); + } else { output->deleteToken(A); TokenList tokens(files); tokens.push_back(new Token(strAB, tok->location)); diff --git a/test.cpp b/test.cpp index 1a84e424..4d3dad67 100644 --- a/test.cpp +++ b/test.cpp @@ -575,7 +575,13 @@ static void hashhash7() // # ## # (C standard; 6.10.3.3.p4) code = "#define hash_hash # ## #\n" "x hash_hash y"; ASSERT_EQUALS("\nx ## y", preprocess(code)); +} +static void hashhash8() +{ + const char code[] = "#define a(xy) x##y = xy\n" + "a(123);"; + ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); } static void ifdef1() @@ -1414,6 +1420,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash5); TEST_CASE(hashhash6); TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) + TEST_CASE(hashhash8); TEST_CASE(ifdef1); TEST_CASE(ifdef2); From c368c82e425bfab12178ad11cb1b3db160acbff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 13 Jan 2018 22:08:17 +0100 Subject: [PATCH 038/381] Fix handling of raw string prefixes L/u/U/u8. Fixes #113 --- simplecpp.cpp | 15 +++++++++++++-- test.cpp | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a4c720d2..dfa950a2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -521,7 +521,13 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { // C++11 raw string literal - if (ch == '\"' && cback() && cback()->op == 'R') { + std::set rawString; + rawString.insert("R"); + rawString.insert("uR"); + rawString.insert("UR"); + rawString.insert("LR"); + rawString.insert("u8R"); + if (ch == '\"' && cback() && rawString.find(cback()->str) != rawString.end()) { std::string delim; ch = readChar(istr,bom); while (istr.good() && ch != '(' && ch != '\n') { @@ -539,7 +545,12 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // TODO report return; currentToken.erase(currentToken.size() - endOfRawString.size(), endOfRawString.size() - 1U); - back()->setstr(escapeString(currentToken)); + if (cback()->op == 'R') + back()->setstr(escapeString(currentToken)); + else { + back()->setstr(cback()->str.substr(0, cback()->str.size() - 1)); + push_back(new Token(currentToken, location)); // push string without newlines + } location.adjust(currentToken); if (currentToken.find_first_of("\r\n") == std::string::npos) location.col += 2 + 2 * delim.size(); diff --git a/test.cpp b/test.cpp index 4d3dad67..a58de726 100644 --- a/test.cpp +++ b/test.cpp @@ -1087,6 +1087,10 @@ static void readfile_rawstring() ASSERT_EQUALS("A = \"\\\"\"", readfile("A = R\"(\")\"")); ASSERT_EQUALS("A = \"abc\"", readfile("A = R\"\"\"(abc)\"\"\"")); ASSERT_EQUALS("A = \"a\nb\nc\";", readfile("A = R\"foo(a\nb\nc)foo\";")); + ASSERT_EQUALS("A = L \"abc\"", readfile("A = LR\"(abc)\"")); + ASSERT_EQUALS("A = u \"abc\"", readfile("A = uR\"(abc)\"")); + ASSERT_EQUALS("A = U \"abc\"", readfile("A = UR\"(abc)\"")); + ASSERT_EQUALS("A = u8 \"abc\"", readfile("A = u8R\"(abc)\"")); } static void readfile_cpp14_number() From 4bb133ae2162e0859df74459db1ecb5cb0f2e269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 14 Jan 2018 15:16:10 +0100 Subject: [PATCH 039/381] Refactoring rawstring string match --- simplecpp.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index dfa950a2..aa3651a4 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -380,6 +380,11 @@ static void portabilityBackslash(simplecpp::OutputList *outputList, const std::v outputList->push_back(err); } +static bool isRawStringId(const std::string &str) +{ + return str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; +} + void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filename, OutputList *outputList) { std::stack loc; @@ -521,13 +526,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { // C++11 raw string literal - std::set rawString; - rawString.insert("R"); - rawString.insert("uR"); - rawString.insert("UR"); - rawString.insert("LR"); - rawString.insert("u8R"); - if (ch == '\"' && cback() && rawString.find(cback()->str) != rawString.end()) { + if (ch == '\"' && cback() && cback()->name && isRawStringId(cback()->str)) { std::string delim; ch = readChar(istr,bom); while (istr.good() && ch != '(' && ch != '\n') { From ec50c813afc27f0a9f6bf3b6b2a7fc1f85e34147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 11 Feb 2018 11:22:33 +0100 Subject: [PATCH 040/381] Better handling of '#include MACRO' when filename is inside <>. However spaces in the filename is not handled currently. --- simplecpp.cpp | 11 +++++++++++ test.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index aa3651a4..193c249d 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2313,6 +2313,17 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL inc2.takeTokens(inc1); } + if (!inc2.empty() && inc2.cfront()->op == '<' && inc2.cback()->op == '>') { + TokenString hdr; + // TODO: Sometimes spaces must be added in the string + // Somehow preprocessToken etc must be told that the location should be source location not destination location + for (const Token *tok = inc2.cfront(); tok; tok = tok->next) { + hdr += tok->str; + } + inc2.clear(); + inc2.push_back(new Token(hdr, inc1.cfront()->location)); + } + if (inc2.empty() || inc2.cfront()->str.size() <= 2U) { if (outputList) { simplecpp::Output err(files); diff --git a/test.cpp b/test.cpp index a58de726..a1e6e65a 100644 --- a/test.cpp +++ b/test.cpp @@ -1061,6 +1061,30 @@ static void include6() // #57 - incomplete macro #include MACRO(,) simplecpp::preprocess(out, rawtokens, files, filedata, dui); } + +static void include7() // #include MACRO +{ + const char code_c[] = "#define HDR <3.h>\n" + "#include HDR\n"; + const char code_h[] = "123\n"; + + std::vector files; + std::istringstream istr_c(code_c); + simplecpp::TokenList rawtokens_c(istr_c, files, "3.c"); + std::istringstream istr_h(code_h); + simplecpp::TokenList rawtokens_h(istr_h, files, "3.h"); + + std::map filedata; + filedata["3.c"] = &rawtokens_c; + filedata["3.h"] = &rawtokens_h; + + simplecpp::TokenList out(files); + simplecpp::DUI dui; + simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + + ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); +} + static void readfile_nullbyte() { const char code[] = "ab\0cd"; @@ -1456,6 +1480,7 @@ int main(int argc, char **argv) TEST_CASE(include4); // -include TEST_CASE(include5); // #include MACRO TEST_CASE(include6); // invalid code: #include MACRO(,) + TEST_CASE(include7); // #include MACRO TEST_CASE(multiline1); TEST_CASE(multiline2); From 6df52bfb9df4ef54145e7df0ca93b3d987ed81fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 11 Feb 2018 15:08:23 +0100 Subject: [PATCH 041/381] Dont tokenize #error and #warning messages. Fixed #118 --- simplecpp.cpp | 10 ++++++++++ test.cpp | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 193c249d..586a68e9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -466,6 +466,16 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen TokenString currentToken; + if (cback() && cback()->previous && cback()->previous->op == '#' && (lastLine() == "# error" || lastLine() == "# warning")) { + while (istr.good() && ch != '\r' && ch != '\n') { + currentToken += ch; + ch = readChar(istr, bom); + } + istr.unget(); + push_back(new Token(currentToken, location)); + continue; + } + // number or name if (isNameChar(ch)) { const bool num = std::isdigit(ch); diff --git a/test.cpp b/test.cpp index a1e6e65a..a6fb7eab 100644 --- a/test.cpp +++ b/test.cpp @@ -445,9 +445,9 @@ static void dotDotDot() ASSERT_EQUALS("1 . . . 2", readfile("1 ... 2")); } -static void error() +static void error1() { - std::istringstream istr("#error hello world! \n"); + std::istringstream istr("#error hello world!\n"); std::vector files; std::map filedata; simplecpp::OutputList outputList; @@ -456,6 +456,17 @@ static void error() ASSERT_EQUALS("file0,1,#error,#error hello world!\n", toString(outputList)); } +static void error2() +{ + std::istringstream istr("#error it's an error\n"); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,#error,#error it's an error\n", toString(outputList)); +} + static void garbage() { const simplecpp::DUI dui; @@ -1435,7 +1446,8 @@ int main(int argc, char **argv) TEST_CASE(dotDotDot); // ... - TEST_CASE(error); + TEST_CASE(error1); + TEST_CASE(error2); TEST_CASE(garbage); TEST_CASE(garbage_endif); From 0a9cc70d80b7f99767aa10f31aa9321635f3536f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 11 Feb 2018 22:22:27 +0100 Subject: [PATCH 042/381] TODO testcase works, at least in linux --- run-tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run-tests.py b/run-tests.py index 7ad35dfd..785fd89c 100644 --- a/run-tests.py +++ b/run-tests.py @@ -69,7 +69,6 @@ def cleanup(out): # todo, high priority 'c99-6_10_3_4_p5.c', 'c99-6_10_3_4_p6.c', - 'cxx_oper_keyword_ms_compat.cpp', 'expr_usual_conversions.c', # condition is true: 4U - 30 >= 0 'stdint.c', 'stringize_misc.c', From 2979908f99733c0969e9f27805aa0e005cf9696b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 11 Feb 2018 22:50:18 +0100 Subject: [PATCH 043/381] Fixed define in define problem. Macro expects parentheses but none given. Fixes #49 --- simplecpp.cpp | 5 +++++ test.cpp | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 586a68e9..dcd706f0 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1500,6 +1500,11 @@ namespace simplecpp { return tok->next; } + if (!sameline(tok, tok->next)) { + output->takeTokens(temp); + return tok->next; + } + const std::map::const_iterator it = macros.find(temp.cback()->str); if (it == macros.end() || expandedmacros.find(temp.cback()->str) != expandedmacros.end()) { output->takeTokens(temp); diff --git a/test.cpp b/test.cpp index a6fb7eab..57042bd8 100644 --- a/test.cpp +++ b/test.cpp @@ -413,6 +413,14 @@ static void define_define_12() ASSERT_EQUALS("\n\n0", preprocess(code)); } +static void define_define_13() // issue #49 - empty macro +{ + const char code[] = "#define f()\n" + "#define t(a) a\n" + "(t(f))\n"; + ASSERT_EQUALS("\n\n( f )", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -1438,6 +1446,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_10); TEST_CASE(define_define_11); TEST_CASE(define_define_12); // expand result of ## + TEST_CASE(define_define_13); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From f9a3fc1b179a73184fc1cd2ce7b8d7f428510a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 12 Feb 2018 17:01:55 +0100 Subject: [PATCH 044/381] Fix handling of multiline strings in macros. Fixed #92 --- simplecpp.cpp | 10 ++++++++-- test.cpp | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index dcd706f0..6d3f09c9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -575,14 +575,20 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen std::string s = currentToken; std::string::size_type pos; + int newlines = 0; while ((pos = s.find_first_of("\r\n")) != std::string::npos) { s.erase(pos,1); + newlines++; } push_back(new Token(s, location)); // push string without newlines - location.adjust(currentToken); - + if (newlines > 0 && lastLine().compare(0,9,"# define ") == 0) { + multiline += newlines; + location.adjust(s); + } else { + location.adjust(currentToken); + } continue; } diff --git a/test.cpp b/test.cpp index 57042bd8..f15dca1d 100644 --- a/test.cpp +++ b/test.cpp @@ -972,6 +972,20 @@ static void multiline5() // column ASSERT_EQUALS(11, rawtokens.back()->location.col); } +static void multiline6() // multiline string in macro +{ + const char code[] = "#define string (\"\\\n" + "x\")\n" + "string\n"; + const simplecpp::DUI dui; + std::istringstream istr(code); + std::vector files; + simplecpp::TokenList rawtokens(istr,files); + ASSERT_EQUALS("# define string ( \"x\" )\n" + "\n" + "string", rawtokens.stringify()); +} + static void include1() { const char code[] = "#include \"A.h\"\n"; @@ -1508,6 +1522,7 @@ int main(int argc, char **argv) TEST_CASE(multiline3); TEST_CASE(multiline4); TEST_CASE(multiline5); // column + TEST_CASE(multiline6); // multiline string in macro TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_string); From 4f874b81f5697491a5cf446258b0ce524f099105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 12 Feb 2018 17:18:01 +0100 Subject: [PATCH 045/381] Fixed handling of empty #error --- simplecpp.cpp | 2 +- test.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6d3f09c9..a6e9044e 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -466,7 +466,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen TokenString currentToken; - if (cback() && cback()->previous && cback()->previous->op == '#' && (lastLine() == "# error" || lastLine() == "# warning")) { + if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && (lastLine() == "# error" || lastLine() == "# warning")) { while (istr.good() && ch != '\r' && ch != '\n') { currentToken += ch; ch = readChar(istr, bom); diff --git a/test.cpp b/test.cpp index f15dca1d..144e25f8 100644 --- a/test.cpp +++ b/test.cpp @@ -1166,6 +1166,14 @@ static void readfile_unhandled_chars() ASSERT_EQUALS("file0,1,unhandled_char_error,The code contains unhandled character(s) (character code=228). Neither unicode nor extended ascii is supported.\n", toString(outputList)); } +static void readfile_error() +{ + ASSERT_EQUALS("# if ! A\n" + "# error\n" + "# endif\n" + "X",readfile("#if !A\n#error\n#endif\nX\n")); +} + static void stringify1() { const char code_c[] = "#include \"A.h\"\n" @@ -1529,6 +1537,7 @@ int main(int argc, char **argv) TEST_CASE(readfile_rawstring); TEST_CASE(readfile_cpp14_number); TEST_CASE(readfile_unhandled_chars); + TEST_CASE(readfile_error); TEST_CASE(stringify1); From 7cbc0394a7e0cbdd26a7f7ec8f5931fd9d1489a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 14 Feb 2018 22:07:59 +0100 Subject: [PATCH 046/381] Fix endless recursion for unterminated macros. Fixes #58 --- simplecpp.cpp | 2 +- test.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a6e9044e..4fea1062 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1338,7 +1338,7 @@ namespace simplecpp { if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { - m.expand(tokens, tok, macros, files); + m.expand(tokens, tok->location, tok, macros, expandedmacros); expanded = true; } } diff --git a/test.cpp b/test.cpp index 144e25f8..51d04d20 100644 --- a/test.cpp +++ b/test.cpp @@ -421,6 +421,15 @@ static void define_define_13() // issue #49 - empty macro ASSERT_EQUALS("\n\n( f )", preprocess(code)); } +static void define_define_14() // issue #58 - endless recursion +{ + const char code[] = "#define z f(w\n" + "#define f()\n" + "#define w f(z\n" + "w\n"; + ASSERT_EQUALS("\n\n\nf ( f ( w", preprocess(code)); // Don't crash +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -1469,6 +1478,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_11); TEST_CASE(define_define_12); // expand result of ## TEST_CASE(define_define_13); + TEST_CASE(define_define_14); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From 8086d0a64415c11fd8c7d540ac6be450342ea5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 25 Feb 2018 22:30:18 +0100 Subject: [PATCH 047/381] Fix for '#include MACRO(header)'. Fixed #114 --- simplecpp.cpp | 3 ++- test.cpp | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4fea1062..59946ede 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2343,6 +2343,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } inc2.clear(); inc2.push_back(new Token(hdr, inc1.cfront()->location)); + inc2.front()->op = '<'; } if (inc2.empty() || inc2.cfront()->str.size() <= 2U) { @@ -2376,7 +2377,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL simplecpp::Output out(files); out.type = Output::MISSING_HEADER; out.location = rawtok->location; - out.msg = "Header not found: " + rawtok->next->str; + out.msg = "Header not found: " + inctok->str; outputList->push_back(out); } } else if (includetokenstack.size() >= 400) { diff --git a/test.cpp b/test.cpp index 51d04d20..2cf361f7 100644 --- a/test.cpp +++ b/test.cpp @@ -1122,11 +1122,27 @@ static void include7() // #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; + dui.includePaths.push_back("."); simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } +static void include8() // #include MACRO(X) +{ + const char code[] = "#define INCLUDE_LOCATION ../somewhere\n" + "#define INCLUDE_FILE(F) \n" + "#include INCLUDE_FILE(header)\n"; + + std::istringstream istr(code); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,3,missing_header,Header not found: <../somewhere/header.h>\n", toString(outputList)); +} + static void readfile_nullbyte() { const char code[] = "ab\0cd"; @@ -1534,6 +1550,7 @@ int main(int argc, char **argv) TEST_CASE(include5); // #include MACRO TEST_CASE(include6); // invalid code: #include MACRO(,) TEST_CASE(include7); // #include MACRO + TEST_CASE(include8); // #include MACRO(X) TEST_CASE(multiline1); TEST_CASE(multiline2); From 81499d947564615d14e9717e05815408aef3d248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 15 Mar 2018 21:21:39 +0100 Subject: [PATCH 048/381] better checking of invalid ## usage. Fixes #121 --- simplecpp.cpp | 5 +++++ test.cpp | 22 +++++++++++++++++++ .../macro_paste_identifier_error.c | 8 ------- 3 files changed, 27 insertions(+), 8 deletions(-) delete mode 100644 testsuite/clang-preprocessor-tests/macro_paste_identifier_error.c diff --git a/simplecpp.cpp b/simplecpp.cpp index 59946ede..edb9f180 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1676,8 +1676,13 @@ namespace simplecpp { throw invalidHashHash(tok->location, name()); if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) throw invalidHashHash(tok->location, name()); + if (!A->name && !A->number && A->op != ',' && !A->str.empty()) + throw invalidHashHash(tok->location, name()); Token *B = tok->next->next; + if (!B->name && !B->number && B->op && B->op != '#') + throw invalidHashHash(tok->location, name()); + std::string strAB; const bool varargs = variadic && args.size() >= 1U && B->str == args[args.size()-1U]; diff --git a/test.cpp b/test.cpp index 2cf361f7..e2e6fddd 100644 --- a/test.cpp +++ b/test.cpp @@ -612,6 +612,26 @@ static void hashhash8() ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); } +static void hashhash_invalid_1() { + std::istringstream istr("#define f(a) (##x)\nf(1)"); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); +} + +static void hashhash_invalid_2() { + std::istringstream istr("#define f(a) (x##)\nf(1)"); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); +} + static void ifdef1() { const char code[] = "#ifdef A\n" @@ -1518,6 +1538,8 @@ int main(int argc, char **argv) TEST_CASE(hashhash6); TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) TEST_CASE(hashhash8); + TEST_CASE(hashhash_invalid_1); + TEST_CASE(hashhash_invalid_2); TEST_CASE(ifdef1); TEST_CASE(ifdef2); diff --git a/testsuite/clang-preprocessor-tests/macro_paste_identifier_error.c b/testsuite/clang-preprocessor-tests/macro_paste_identifier_error.c deleted file mode 100644 index bba31723..00000000 --- a/testsuite/clang-preprocessor-tests/macro_paste_identifier_error.c +++ /dev/null @@ -1,8 +0,0 @@ -// RUN: %clang_cc1 -fms-extensions -Wno-invalid-token-paste %s -verify -// RUN: %clang_cc1 -E -fms-extensions -Wno-invalid-token-paste %s | FileCheck %s -// RUN: %clang_cc1 -E -fms-extensions -Wno-invalid-token-paste -x assembler-with-cpp %s | FileCheck %s -// expected-no-diagnostics - -#define foo a ## b ## = 0 -int foo; -// CHECK: int ab = 0; From 127340e83df4ec58c6b7da8942ec9c0ddd196c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 1 Apr 2018 09:05:23 +0200 Subject: [PATCH 049/381] TokenList::combineOperators: dont combine &= for anonymous reference parameter --- simplecpp.cpp | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ test.cpp | 14 ++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index edb9f180..da181a87 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -658,7 +658,26 @@ static bool isFloatSuffix(const simplecpp::Token *tok) void simplecpp::TokenList::combineOperators() { + std::stack executableScope; + executableScope.push(false); for (Token *tok = front(); tok; tok = tok->next) { + if (tok->op == '{') { + if (executableScope.top()) { + executableScope.push(true); + continue; + } + const Token *prev = tok->previous; + while (prev && prev->isOneOf(";{}()")) + prev = prev->previous; + executableScope.push(prev && prev->op == ')'); + continue; + } + if (tok->op == '}') { + if (executableScope.size() > 1) + executableScope.pop(); + continue; + } + if (tok->op == '.') { if (tok->previous && tok->previous->op == '.') continue; @@ -694,6 +713,39 @@ void simplecpp::TokenList::combineOperators() continue; if (tok->next->op == '=' && tok->isOneOf("=!<>+-*/%&|^")) { + if (tok->op == '&' && !executableScope.top()) { + // don't combine &= if it is a anonymous reference parameter with default value: + // void f(x&=2) + int indentlevel = 0; + const Token *start = tok; + while (indentlevel >= 0 && start) { + if (start->op == ')') + ++indentlevel; + else if (start->op == '(') + --indentlevel; + else if (start->isOneOf(";{}")) + break; + start = start->previous; + } + if (indentlevel == -1 && start) { + const Token *ftok = start; + bool isFuncDecl = ftok->name; + while (isFuncDecl) { + if (!start->name && start->str != "::" && start->op != '*' && start->op != '&') + isFuncDecl = false; + if (!start->previous) + break; + if (start->previous->isOneOf(";{}:")) + break; + start = start->previous; + } + isFuncDecl &= start != ftok && start->name; + if (isFuncDecl) { + // TODO: we could loop through the parameters here and check if they are correct. + continue; + } + } + } tok->setstr(tok->str + "="); deleteToken(tok->next); } else if ((tok->op == '|' || tok->op == '&') && tok->op == tok->next->op) { diff --git a/test.cpp b/test.cpp index e2e6fddd..497b436f 100644 --- a/test.cpp +++ b/test.cpp @@ -168,6 +168,13 @@ static void combineOperators_coloncolon() ASSERT_EQUALS("x ? y : :: z", preprocess("x ? y : ::z")); } +static void combineOperators_andequal() +{ + ASSERT_EQUALS("x &= 2 ;", preprocess("x &= 2;")); + ASSERT_EQUALS("void f ( x & = 2 ) ;", preprocess("void f(x &= 2);")); + ASSERT_EQUALS("f ( x &= 2 ) ;", preprocess("f(x &= 2);")); +} + static void comment() { ASSERT_EQUALS("// abc", readfile("// abc")); @@ -612,7 +619,8 @@ static void hashhash8() ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); } -static void hashhash_invalid_1() { +static void hashhash_invalid_1() +{ std::istringstream istr("#define f(a) (##x)\nf(1)"); std::vector files; std::map filedata; @@ -622,7 +630,8 @@ static void hashhash_invalid_1() { ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } -static void hashhash_invalid_2() { +static void hashhash_invalid_2() +{ std::istringstream istr("#define f(a) (x##)\nf(1)"); std::vector files; std::map filedata; @@ -1484,6 +1493,7 @@ int main(int argc, char **argv) TEST_CASE(combineOperators_floatliteral); TEST_CASE(combineOperators_increment); TEST_CASE(combineOperators_coloncolon); + TEST_CASE(combineOperators_andequal); TEST_CASE(comment); TEST_CASE(comment_multiline); From 0df1acbb6d228ba270040db0d900b8ef411ac2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 9 Apr 2018 10:46:43 +0200 Subject: [PATCH 050/381] Fix Path::simplifyPath() for glob patterns --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index da181a87..3e769a5c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1977,7 +1977,7 @@ namespace simplecpp { if (unc) path = '/' + path; - return realFilename(path); + return path.find_first_of("*?") == std::string::npos ? realFilename(path) : path; } } From 8c5304a39f451827a6ec790658b834b6c582ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 12 Apr 2018 21:50:49 +0200 Subject: [PATCH 051/381] Handle null directive. Fixes #125 --- simplecpp.cpp | 9 ++++++--- test.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3e769a5c..ca3fd21e 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2313,10 +2313,13 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) { + if (!sameline(rawtok, rawtok->next)) { + rawtok = rawtok->next; + continue; + } rawtok = rawtok->next; - if (!rawtok || !rawtok->name) { - if (rawtok) - rawtok = gotoNextLine(rawtok); + if (!rawtok->name) { + rawtok = gotoNextLine(rawtok); continue; } diff --git a/test.cpp b/test.cpp index 497b436f..92b37757 100644 --- a/test.cpp +++ b/test.cpp @@ -1024,6 +1024,30 @@ static void multiline6() // multiline string in macro "string", rawtokens.stringify()); } +static void nullDirective1() +{ + const char code[] = "#\n" + "#if 1\n" + "#define a 1\n" + "#endif\n" + "x = a;\n"; + + const simplecpp::DUI dui; + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); +} + +static void nullDirective2() +{ + const char code[] = "# // comment\n" + "#if 1\n" + "#define a 1\n" + "#endif\n" + "x = a;\n"; + + const simplecpp::DUI dui; + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); +} + static void include1() { const char code[] = "#include \"A.h\"\n"; @@ -1575,6 +1599,9 @@ int main(int argc, char **argv) TEST_CASE(missingHeader3); TEST_CASE(nestedInclude); + TEST_CASE(nullDirective1); + TEST_CASE(nullDirective2); + TEST_CASE(include1); TEST_CASE(include2); TEST_CASE(include3); From b9dd0c5cb3452a1662ba48d84e9eaf08ae096183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 17 Apr 2018 08:11:28 +0200 Subject: [PATCH 052/381] Write error when preprocessor directive is used as macro parameter --- simplecpp.cpp | 2 ++ test.cpp | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index ca3fd21e..969a23c9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1179,6 +1179,8 @@ namespace simplecpp { ++par; else if (rawtok->op == ')') --par; + else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) + throw Error(rawtok->location, "it is invalid to use a preprocessor directive as macro parameter"); rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); rawtok = rawtok->next; } diff --git a/test.cpp b/test.cpp index 92b37757..067913d8 100644 --- a/test.cpp +++ b/test.cpp @@ -458,6 +458,21 @@ static void define_va_args_3() // min number of arguments ASSERT_EQUALS("\n1", preprocess(code)); } +static void define_ifdef() +{ + const char code[] = "#define A(X) X\n" + "A(1\n" + "#ifdef CFG\n" + "#endif\n" + ")\n"; + + const simplecpp::DUI dui; + simplecpp::OutputList outputList; + preprocess(code, dui, &outputList); + ASSERT_EQUALS("file0,3,syntax_error,failed to expand 'A', it is invalid to use a preprocessor directive as macro parameter\n", toString(outputList)); + +} + static void dollar() { ASSERT_EQUALS("$ab", readfile("$ab")); @@ -1553,6 +1568,9 @@ int main(int argc, char **argv) TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); + // UB: #ifdef as macro parameter + TEST_CASE(define_ifdef); + TEST_CASE(dollar); TEST_CASE(dotDotDot); // ... From 600a89dea238eac5c8465c204e0772a2468cbc21 Mon Sep 17 00:00:00 2001 From: PKEuS Date: Mon, 14 May 2018 10:45:35 +0200 Subject: [PATCH 053/381] Optimized memory usage by simplecpp::Token: - One empty string object is sufficient, not one per Token... - Replaced reference to its own string by function str() --- simplecpp.cpp | 339 +++++++++++++++++++++++++------------------------- simplecpp.h | 16 +-- test.cpp | 14 +-- 3 files changed, 186 insertions(+), 183 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 969a23c9..70ab3729 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -112,7 +112,7 @@ static bool sameline(const simplecpp::Token *tok1, const simplecpp::Token *tok2) static bool isAlternativeBinaryOp(const simplecpp::Token *tok, const std::string &alt) { return (tok->name && - tok->str == alt && + tok->str() == alt && tok->previous && tok->next && (tok->previous->number || tok->previous->name || tok->previous->op == ')') && @@ -121,11 +121,14 @@ static bool isAlternativeBinaryOp(const simplecpp::Token *tok, const std::string static bool isAlternativeUnaryOp(const simplecpp::Token *tok, const std::string &alt) { - return ((tok->name && tok->str == alt) && + return ((tok->name && tok->str() == alt) && (!tok->previous || tok->previous->op == '(') && (tok->next && (tok->next->name || tok->next->number))); } + +const std::string simplecpp::Location::emptyFileName; + void simplecpp::Location::adjust(const std::string &str) { if (str.find_first_of("\r\n") == std::string::npos) { @@ -151,12 +154,12 @@ bool simplecpp::Token::isOneOf(const char ops[]) const bool simplecpp::Token::startsWithOneOf(const char c[]) const { - return std::strchr(c, str[0]) != 0; + return std::strchr(c, string[0]) != 0; } bool simplecpp::Token::endsWithOneOf(const char c[]) const { - return std::strchr(c, str[str.size() - 1U]) != 0; + return std::strchr(c, string[string.size() - 1U]) != 0; } void simplecpp::Token::printAll() const @@ -168,7 +171,7 @@ void simplecpp::Token::printAll() const if (tok->previous) { std::cout << (sameline(tok, tok->previous) ? ' ' : '\n'); } - std::cout << tok->str; + std::cout << tok->str(); } std::cout << std::endl; } @@ -179,7 +182,7 @@ void simplecpp::Token::printOut() const if (tok != this) { std::cout << (sameline(tok, tok->previous) ? ' ' : '\n'); } - std::cout << tok->str; + std::cout << tok->str(); } std::cout << std::endl; } @@ -257,9 +260,9 @@ std::string simplecpp::TokenList::stringify() const if (sameline(tok->previous, tok)) ret << ' '; - ret << tok->str; + ret << tok->str(); - loc.adjust(tok->str); + loc.adjust(tok->str()); } return ret.str(); @@ -439,15 +442,15 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (lastline == "# file %str%") { loc.push(location); - location.fileIndex = fileIndex(cback()->str.substr(1U, cback()->str.size() - 2U)); + location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = 1U; } else if (lastline == "# line %num%") { loc.push(location); - location.line = std::atol(cback()->str.c_str()); + location.line = std::atol(cback()->str().c_str()); } else if (lastline == "# line %num% %str%") { loc.push(location); - location.fileIndex = fileIndex(cback()->str.substr(1U, cback()->str.size() - 2U)); - location.line = std::atol(cback()->previous->str.c_str()); + location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); + location.line = std::atol(cback()->previous->str().c_str()); } // #endfile else if (lastline == "# endfile" && !loc.empty()) { @@ -536,7 +539,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { // C++11 raw string literal - if (ch == '\"' && cback() && cback()->name && isRawStringId(cback()->str)) { + if (ch == '\"' && cback() && cback()->name && isRawStringId(cback()->str())) { std::string delim; ch = readChar(istr,bom); while (istr.good() && ch != '(' && ch != '\n') { @@ -557,7 +560,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (cback()->op == 'R') back()->setstr(escapeString(currentToken)); else { - back()->setstr(cback()->str.substr(0, cback()->str.size() - 1)); + back()->setstr(cback()->str().substr(0, cback()->str().size() - 1)); push_back(new Token(currentToken, location)); // push string without newlines } location.adjust(currentToken); @@ -650,9 +653,9 @@ void simplecpp::TokenList::constFold() static bool isFloatSuffix(const simplecpp::Token *tok) { - if (!tok || tok->str.size() != 1U) + if (!tok || tok->str().size() != 1U) return false; - const char c = std::tolower(tok->str[0]); + const char c = std::tolower(tok->str()[0]); return c == 'f' || c == 'l'; } @@ -685,22 +688,22 @@ void simplecpp::TokenList::combineOperators() continue; // float literals.. if (tok->previous && tok->previous->number) { - tok->setstr(tok->previous->str + '.'); + tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); if (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("Ee"))) { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } } if (tok->next && tok->next->number) { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } } // match: [0-9.]+E [+-] [0-9]+ - const char lastChar = tok->str[tok->str.size() - 1]; - if (tok->number && !isHex(tok->str) && (lastChar == 'E' || lastChar == 'e') && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { - tok->setstr(tok->str + tok->next->op + tok->next->next->str); + const char lastChar = tok->str()[tok->str().size() - 1]; + if (tok->number && !isHex(tok->str()) && (lastChar == 'E' || lastChar == 'e') && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { + tok->setstr(tok->str() + tok->next->op + tok->next->next->str()); deleteToken(tok->next); deleteToken(tok->next); } @@ -731,7 +734,7 @@ void simplecpp::TokenList::combineOperators() const Token *ftok = start; bool isFuncDecl = ftok->name; while (isFuncDecl) { - if (!start->name && start->str != "::" && start->op != '*' && start->op != '&') + if (!start->name && start->str() != "::" && start->op != '*' && start->op != '&') isFuncDecl = false; if (!start->previous) break; @@ -746,22 +749,22 @@ void simplecpp::TokenList::combineOperators() } } } - tok->setstr(tok->str + "="); + tok->setstr(tok->str() + "="); deleteToken(tok->next); } else if ((tok->op == '|' || tok->op == '&') && tok->op == tok->next->op) { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } else if (tok->op == ':' && tok->next->op == ':') { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } else if (tok->op == '-' && tok->next->op == '>') { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } else if ((tok->op == '<' || tok->op == '>') && tok->op == tok->next->op) { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); if (tok->next && tok->next->op == '=') { - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } } else if ((tok->op == '+' || tok->op == '-') && tok->op == tok->next->op) { @@ -771,7 +774,7 @@ void simplecpp::TokenList::combineOperators() continue; if (tok->next->next && tok->next->next->number) continue; - tok->setstr(tok->str + tok->next->str); + tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } } @@ -790,10 +793,10 @@ void simplecpp::TokenList::constFoldUnaryNotPosNeg(simplecpp::Token *tok) tok->op = '~'; if (tok->op == '!' && tok->next && tok->next->number) { - tok->setstr(tok->next->str == "0" ? "1" : "0"); + tok->setstr(tok->next->str() == "0" ? "1" : "0"); deleteToken(tok->next); } else if (tok->op == '~' && tok->next && tok->next->number) { - tok->setstr(toString(~stringToLL(tok->next->str))); + tok->setstr(toString(~stringToLL(tok->next->str()))); deleteToken(tok->next); } else { if (tok->previous && (tok->previous->number || tok->previous->name)) @@ -802,11 +805,11 @@ void simplecpp::TokenList::constFoldUnaryNotPosNeg(simplecpp::Token *tok) continue; switch (tok->op) { case '+': - tok->setstr(tok->next->str); + tok->setstr(tok->next->str()); deleteToken(tok->next); break; case '-': - tok->setstr(tok->op + tok->next->str); + tok->setstr(tok->op + tok->next->str()); deleteToken(tok->next); break; } @@ -824,12 +827,12 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) long long result; if (tok->op == '*') - result = (stringToLL(tok->previous->str) * stringToLL(tok->next->str)); + result = (stringToLL(tok->previous->str()) * stringToLL(tok->next->str())); else if (tok->op == '/' || tok->op == '%') { - long long rhs = stringToLL(tok->next->str); + long long rhs = stringToLL(tok->next->str()); if (rhs == 0) throw std::overflow_error("division/modulo by zero"); - long long lhs = stringToLL(tok->previous->str); + long long lhs = stringToLL(tok->previous->str()); if (rhs == -1 && lhs == std::numeric_limits::min()) throw std::overflow_error("division overflow"); if (tok->op == '/') @@ -856,9 +859,9 @@ void simplecpp::TokenList::constFoldAddSub(Token *tok) long long result; if (tok->op == '+') - result = stringToLL(tok->previous->str) + stringToLL(tok->next->str); + result = stringToLL(tok->previous->str()) + stringToLL(tok->next->str()); else if (tok->op == '-') - result = stringToLL(tok->previous->str) - stringToLL(tok->next->str); + result = stringToLL(tok->previous->str()) - stringToLL(tok->next->str()); else continue; @@ -878,10 +881,10 @@ void simplecpp::TokenList::constFoldShift(Token *tok) continue; long long result; - if (tok->str == "<<") - result = stringToLL(tok->previous->str) << stringToLL(tok->next->str); - else if (tok->str == ">>") - result = stringToLL(tok->previous->str) >> stringToLL(tok->next->str); + if (tok->str() == "<<") + result = stringToLL(tok->previous->str()) << stringToLL(tok->next->str()); + else if (tok->str() == ">>") + result = stringToLL(tok->previous->str()) >> stringToLL(tok->next->str()); else continue; @@ -907,18 +910,18 @@ void simplecpp::TokenList::constFoldComparison(Token *tok) continue; int result; - if (tok->str == "==") - result = (stringToLL(tok->previous->str) == stringToLL(tok->next->str)); - else if (tok->str == "!=") - result = (stringToLL(tok->previous->str) != stringToLL(tok->next->str)); - else if (tok->str == ">") - result = (stringToLL(tok->previous->str) > stringToLL(tok->next->str)); - else if (tok->str == ">=") - result = (stringToLL(tok->previous->str) >= stringToLL(tok->next->str)); - else if (tok->str == "<") - result = (stringToLL(tok->previous->str) < stringToLL(tok->next->str)); - else if (tok->str == "<=") - result = (stringToLL(tok->previous->str) <= stringToLL(tok->next->str)); + if (tok->str() == "==") + result = (stringToLL(tok->previous->str()) == stringToLL(tok->next->str())); + else if (tok->str() == "!=") + result = (stringToLL(tok->previous->str()) != stringToLL(tok->next->str())); + else if (tok->str() == ">") + result = (stringToLL(tok->previous->str()) > stringToLL(tok->next->str())); + else if (tok->str() == ">=") + result = (stringToLL(tok->previous->str()) >= stringToLL(tok->next->str())); + else if (tok->str() == "<") + result = (stringToLL(tok->previous->str()) < stringToLL(tok->next->str())); + else if (tok->str() == "<=") + result = (stringToLL(tok->previous->str()) <= stringToLL(tok->next->str())); else continue; @@ -952,11 +955,11 @@ void simplecpp::TokenList::constFoldBitwise(Token *tok) continue; long long result; if (*op == '&') - result = (stringToLL(tok->previous->str) & stringToLL(tok->next->str)); + result = (stringToLL(tok->previous->str()) & stringToLL(tok->next->str())); else if (*op == '^') - result = (stringToLL(tok->previous->str) ^ stringToLL(tok->next->str)); + result = (stringToLL(tok->previous->str()) ^ stringToLL(tok->next->str())); else /*if (*op == '|')*/ - result = (stringToLL(tok->previous->str) | stringToLL(tok->next->str)); + result = (stringToLL(tok->previous->str()) | stringToLL(tok->next->str())); tok = tok->previous; tok->setstr(toString(result)); deleteToken(tok->next); @@ -976,7 +979,7 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) else if (isAlternativeBinaryOp(tok,OR)) tok->setstr("||"); } - if (tok->str != "&&" && tok->str != "||") + if (tok->str() != "&&" && tok->str() != "||") continue; if (!tok->previous || !tok->previous->number) continue; @@ -984,10 +987,10 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) continue; int result; - if (tok->str == "||") - result = (stringToLL(tok->previous->str) || stringToLL(tok->next->str)); - else /*if (tok->str == "&&")*/ - result = (stringToLL(tok->previous->str) && stringToLL(tok->next->str)); + if (tok->str() == "||") + result = (stringToLL(tok->previous->str()) || stringToLL(tok->next->str())); + else /*if (tok->str() == "&&")*/ + result = (stringToLL(tok->previous->str()) && stringToLL(tok->next->str())); tok = tok->previous; tok->setstr(toString(result)); @@ -1001,7 +1004,7 @@ void simplecpp::TokenList::constFoldQuestionOp(Token **tok1) bool gotoTok1 = false; for (Token *tok = *tok1; tok && tok->op != ')'; tok = gotoTok1 ? *tok1 : tok->next) { gotoTok1 = false; - if (tok->str != "?") + if (tok->str() != "?") continue; if (!tok->previous || !tok->next || !tok->next->next) throw std::runtime_error("invalid expression"); @@ -1015,10 +1018,10 @@ void simplecpp::TokenList::constFoldQuestionOp(Token **tok1) if (!falseTok) throw std::runtime_error("invalid expression"); if (condTok == *tok1) - *tok1 = (condTok->str != "0" ? trueTok : falseTok); + *tok1 = (condTok->str() != "0" ? trueTok : falseTok); deleteToken(condTok->next); // ? deleteToken(trueTok->next); // : - deleteToken(condTok->str == "0" ? trueTok : falseTok); + deleteToken(condTok->str() == "0" ? trueTok : falseTok); deleteToken(condTok); gotoTok1 = true; } @@ -1085,8 +1088,8 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const continue; if (!ret.empty()) ret = ' ' + ret; - ret = (tok->str[0] == '\"' ? std::string("%str%") - : std::isdigit(static_cast(tok->str[0])) ? std::string("%num%") : tok->str) + ret; + ret = (tok->str()[0] == '\"' ? std::string("%str%") + : std::isdigit(static_cast(tok->str()[0])) ? std::string("%num%") : tok->str()) + ret; if (++count > maxsize) return ""; } @@ -1116,7 +1119,7 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax"); const Token * const hashtok = tok; tok = tok->next; - if (!tok || tok->str != DEFINE) + if (!tok || tok->str() != DEFINE) throw std::runtime_error("bad macro syntax"); tok = tok->next; if (!tok || !tok->name || !sameline(hashtok,tok)) @@ -1169,9 +1172,9 @@ namespace simplecpp { // Copy macro call to a new tokenlist with no linebreaks const Token * const rawtok1 = rawtok; TokenList rawtokens2(inputFiles); - rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); rawtok = rawtok->next; - rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); rawtok = rawtok->next; int par = 1; while (rawtok && par > 0) { @@ -1181,7 +1184,7 @@ namespace simplecpp { --par; else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) throw Error(rawtok->location, "it is invalid to use a preprocessor directive as macro parameter"); - rawtokens2.push_back(new Token(rawtok->str, rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); rawtok = rawtok->next; } if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros)) @@ -1208,23 +1211,23 @@ namespace simplecpp { macro2tok = output2.back(); if (!macro2tok || !macro2tok->name) break; - if (output2.cfront() != output2.cback() && macro2tok->str == this->name()) + if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; - const std::map::const_iterator macro = macros.find(macro2tok->str); + const std::map::const_iterator macro = macros.find(macro2tok->str()); if (macro == macros.end() || !macro->second.functionLike()) break; TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); while (macro2tok) { Token *next = macro2tok->next; - rawtokens2.push_back(new Token(macro2tok->str, loc)); + rawtokens2.push_back(new Token(macro2tok->str(), loc)); output2.deleteToken(macro2tok); macro2tok = next; } par = (rawtokens2.cfront() != rawtokens2.cback()) ? 1U : 0U; const Token *rawtok2 = rawtok; for (; rawtok2; rawtok2 = rawtok2->next) { - rawtokens2.push_back(new Token(rawtok2->str, loc)); + rawtokens2.push_back(new Token(rawtok2->str(), loc)); if (rawtok2->op == '(') ++par; else if (rawtok2->op == ')') { @@ -1245,7 +1248,7 @@ namespace simplecpp { /** macro name */ const TokenString &name() const { - return nameTokDef->str; + return nameTokDef->str(); } /** location for macro definition */ @@ -1263,7 +1266,7 @@ namespace simplecpp { return nameTokDef->next && nameTokDef->next->op == '(' && sameline(nameTokDef, nameTokDef->next) && - nameTokDef->next->location.col == nameTokDef->location.col + nameTokDef->str.size(); + nameTokDef->next->location.col == nameTokDef->location.col + nameTokDef->str().size(); } /** base class for errors */ @@ -1287,7 +1290,7 @@ namespace simplecpp { Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced) const { Token *tok = new Token(str,loc); if (replaced) - tok->macro = nameTokDef->str; + tok->macro = nameTokDef->str(); return tok; } @@ -1316,7 +1319,7 @@ namespace simplecpp { break; } if (argtok->op != ',') - args.push_back(argtok->str); + args.push_back(argtok->str()); argtok = argtok->next; } if (!sameline(nametoken, argtok)) { @@ -1388,8 +1391,8 @@ namespace simplecpp { } else { if (!expandArg(tokens, tok, tok->location, macros, expandedmacros, parametertokens)) { bool expanded = false; - const std::map::const_iterator it = macros.find(tok->str); - if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { + const std::map::const_iterator it = macros.find(tok->str()); + if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { m.expand(tokens, tok->location, tok, macros, expandedmacros); @@ -1414,19 +1417,19 @@ namespace simplecpp { } const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const std::map ¯os, std::set expandedmacros) const { - expandedmacros.insert(nameTokInst->str); + expandedmacros.insert(nameTokInst->str()); usageList.push_back(loc); - if (nameTokInst->str == "__FILE__") { + if (nameTokInst->str() == "__FILE__") { output->push_back(new Token('\"'+loc.file()+'\"', loc)); return nameTokInst->next; } - if (nameTokInst->str == "__LINE__") { + if (nameTokInst->str() == "__LINE__") { output->push_back(new Token(toString(loc.line), loc)); return nameTokInst->next; } - if (nameTokInst->str == "__COUNTER__") { + if (nameTokInst->str() == "__COUNTER__") { output->push_back(new Token(toString(usageList.size()-1U), loc)); return nameTokInst->next; } @@ -1439,7 +1442,7 @@ namespace simplecpp { if (functionLike()) { // No arguments => not macro expansion if (nameTokInst->next && nameTokInst->next->op != '(') { - output->push_back(new Token(nameTokInst->str, loc)); + output->push_back(new Token(nameTokInst->str(), loc)); return nameTokInst->next; } @@ -1460,7 +1463,7 @@ namespace simplecpp { if (!parametertokens1.empty()) { bool counter = false; for (const Token *tok = parametertokens1[0]; tok != parametertokens1.back(); tok = tok->next) { - if (tok->str == "__COUNTER__") { + if (tok->str() == "__COUNTER__") { counter = true; break; } @@ -1474,7 +1477,7 @@ namespace simplecpp { const Macro &counterMacro = m->second; unsigned int par = 0; for (const Token *tok = parametertokens1[0]; tok && par < parametertokens1.size(); tok = tok->next) { - if (tok->str == "__COUNTER__") { + if (tok->str() == "__COUNTER__") { tokensparams.push_back(new Token(toString(counterMacro.usageList.size()), tok->location)); counterMacro.usageList.push_back(tok->location); } else { @@ -1534,7 +1537,7 @@ namespace simplecpp { if (!functionLike()) { for (Token *tok = output_end_1 ? output_end_1->next : output->front(); tok; tok = tok->next) { - tok->macro = nameTokInst->str; + tok->macro = nameTokInst->str(); } } @@ -1547,7 +1550,7 @@ namespace simplecpp { const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { - output->push_back(newMacroToken(tok->str, loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; } @@ -1565,8 +1568,8 @@ namespace simplecpp { return tok->next; } - const std::map::const_iterator it = macros.find(temp.cback()->str); - if (it == macros.end() || expandedmacros.find(temp.cback()->str) != expandedmacros.end()) { + const std::map::const_iterator it = macros.find(temp.cback()->str()); + if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) { output->takeTokens(temp); return tok->next; } @@ -1578,7 +1581,7 @@ namespace simplecpp { } TokenList temp2(files); - temp2.push_back(new Token(temp.cback()->str, tok->location)); + temp2.push_back(new Token(temp.cback()->str(), tok->location)); const Token *tok2 = appendTokens(&temp2, tok->next, macros, expandedmacros, parametertokens); if (!tok2) @@ -1593,27 +1596,27 @@ namespace simplecpp { } // Macro.. - const std::map::const_iterator it = macros.find(tok->str); - if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) { + const std::map::const_iterator it = macros.find(tok->str()); + if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { const Macro &calledMacro = it->second; if (!calledMacro.functionLike()) return calledMacro.expand(output, loc, tok, macros, expandedmacros); if (!sameline(tok, tok->next) || tok->next->op != '(') { - output->push_back(newMacroToken(tok->str, loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; } TokenList tokens(files); tokens.push_back(new Token(*tok)); const Token *tok2 = appendTokens(&tokens, tok->next, macros, expandedmacros, parametertokens); if (!tok2) { - output->push_back(newMacroToken(tok->str, loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; } calledMacro.expand(output, loc, tokens.cfront(), macros, expandedmacros); return tok2->next; } - else if (tok->str == DEFINED) { + else if (tok->str() == DEFINED) { const Token *tok2 = tok->next; const Token *tok3 = tok2 ? tok2->next : NULL; const Token *tok4 = tok3 ? tok3->next : NULL; @@ -1626,13 +1629,13 @@ namespace simplecpp { defToken = lastToken = tok2; } if (defToken) { - const bool def = (macros.find(defToken->str) != macros.end()); + const bool def = (macros.find(defToken->str()) != macros.end()); output->push_back(newMacroToken(def ? "1" : "0", loc, true)); return lastToken->next; } } - output->push_back(newMacroToken(tok->str, loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; } @@ -1640,7 +1643,7 @@ namespace simplecpp { if (!tok->name) return false; - const unsigned int argnr = getArgNum(tok->str); + const unsigned int argnr = getArgNum(tok->str()); if (argnr >= args.size()) return false; @@ -1657,17 +1660,17 @@ namespace simplecpp { bool expandArg(TokenList *output, const Token *tok, const Location &loc, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!tok->name) return false; - const unsigned int argnr = getArgNum(tok->str); + const unsigned int argnr = getArgNum(tok->str()); if (argnr >= args.size()) return false; if (variadic && argnr + 1U >= parametertokens.size()) // empty variadic parameter return true; for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) { - const std::map::const_iterator it = macros.find(partok->str); - if (it != macros.end() && (partok->str == name() || expandedmacros.find(partok->str) == expandedmacros.end())) + const std::map::const_iterator it = macros.find(partok->str()); + if (it != macros.end() && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) partok = it->second.expand(output, loc, partok, macros, expandedmacros); else { - output->push_back(newMacroToken(partok->str, loc, isReplaced(expandedmacros))); + output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros))); partok = partok->next; } } @@ -1685,10 +1688,10 @@ namespace simplecpp { if (expandArg(&tokens, tok, parametertokens)) { std::string s; for (const Token *tok2 = tokens.cfront(); tok2; tok2 = tok2->next) - s += tok2->str; + s += tok2->str(); return s; } - return tok->str; + return tok->str(); } /** @@ -1707,7 +1710,7 @@ namespace simplecpp { std::ostringstream ostr; ostr << '\"'; for (const Token *hashtok = tokenListHash.cfront(); hashtok; hashtok = hashtok->next) - ostr << hashtok->str; + ostr << hashtok->str(); ostr << '\"'; output->push_back(newMacroToken(escapeString(ostr.str()), loc, isReplaced(expandedmacros))); return tok; @@ -1730,7 +1733,7 @@ namespace simplecpp { throw invalidHashHash(tok->location, name()); if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) throw invalidHashHash(tok->location, name()); - if (!A->name && !A->number && A->op != ',' && !A->str.empty()) + if (!A->name && !A->number && A->op != ',' && !A->str().empty()) throw invalidHashHash(tok->location, name()); Token *B = tok->next->next; @@ -1739,25 +1742,25 @@ namespace simplecpp { std::string strAB; - const bool varargs = variadic && args.size() >= 1U && B->str == args[args.size()-1U]; + const bool varargs = variadic && args.size() >= 1U && B->str() == args[args.size()-1U]; TokenList tokensB(files); if (expandArg(&tokensB, B, parametertokens)) { if (tokensB.empty()) - strAB = A->str; + strAB = A->str(); else if (varargs && A->op == ',') { strAB = ","; } else { - strAB = A->str + tokensB.cfront()->str; + strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); } } else { - strAB = A->str + B->str; + strAB = A->str() + B->str(); } const Token *nextTok = B->next; - if (varargs && tokensB.empty() && tok->previous->str == ",") + if (varargs && tokensB.empty() && tok->previous->str() == ",") output->deleteToken(A); else if (strAB != "," && macros.find(strAB) == macros.end()) { A->setstr(strAB); @@ -1987,7 +1990,7 @@ namespace simplecpp { static void simplifySizeof(simplecpp::TokenList &expr, const std::map &sizeOfType) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->str != "sizeof") + if (tok->str() != "sizeof") continue; simplecpp::Token *tok1 = tok->next; if (!tok1) { @@ -2009,13 +2012,13 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::mapnext) { - if ((typeToken->str == "unsigned" || typeToken->str == "signed") && typeToken->next->name) + if ((typeToken->str() == "unsigned" || typeToken->str() == "signed") && typeToken->next->name) continue; - if (typeToken->str == "*" && type.find('*') != std::string::npos) + if (typeToken->str() == "*" && type.find('*') != std::string::npos) continue; if (!type.empty()) type += ' '; - type += typeToken->str; + type += typeToken->str(); } const std::map::const_iterator it = sizeOfType.find(type); @@ -2036,12 +2039,12 @@ static void simplifyName(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->name) { - if (altop.find(tok->str) != altop.end()) { + if (altop.find(tok->str()) != altop.end()) { bool alt; - if (tok->str == "not" || tok->str == "compl") { - alt = isAlternativeUnaryOp(tok,tok->str); + if (tok->str() == "not" || tok->str() == "compl") { + alt = isAlternativeUnaryOp(tok,tok->str()); } else { - alt = isAlternativeBinaryOp(tok,tok->str); + alt = isAlternativeBinaryOp(tok,tok->str()); } if (alt) continue; @@ -2054,12 +2057,12 @@ static void simplifyName(simplecpp::TokenList &expr) static void simplifyNumbers(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->str.size() == 1U) + if (tok->str().size() == 1U) continue; - if (tok->str.compare(0,2,"0x") == 0) - tok->setstr(toString(stringToULL(tok->str))); - else if (tok->str[0] == '\'') - tok->setstr(toString(tok->str[1] & 0xffU)); + if (tok->str().compare(0,2,"0x") == 0) + tok->setstr(toString(stringToULL(tok->str()))); + else if (tok->str()[0] == '\'') + tok->setstr(toString(tok->str()[1] & 0xffU)); } } @@ -2070,7 +2073,7 @@ static long long evaluate(simplecpp::TokenList &expr, const std::mapnumber ? stringToLL(expr.cfront()->str) : 0LL; + return expr.cfront() && expr.cfront() == expr.cback() && expr.cfront()->number ? stringToLL(expr.cfront()->str()) : 0LL; } static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) @@ -2191,7 +2194,7 @@ std::map simplecpp::load(const simplecpp::To continue; rawtok = rawtok->nextSkipComments(); - if (!rawtok || rawtok->str != INCLUDE) + if (!rawtok || rawtok->str() != INCLUDE) continue; const std::string &sourcefile = rawtok->location.file(); @@ -2200,9 +2203,9 @@ std::map simplecpp::load(const simplecpp::To if (!sameline(rawtok, htok)) continue; - bool systemheader = (htok->str[0] == '<'); + bool systemheader = (htok->str()[0] == '<'); - const std::string header(realFilename(htok->str.substr(1U, htok->str.size() - 2U))); + const std::string header(realFilename(htok->str().substr(1U, htok->str().size() - 2U))); if (hasFile(ret, sourcefile, header, dui, systemheader)) continue; @@ -2223,7 +2226,7 @@ std::map simplecpp::load(const simplecpp::To static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, std::map ¯os, std::vector &files, simplecpp::OutputList *outputList) { const simplecpp::Token *tok = *tok1; - const std::map::const_iterator it = macros.find(tok->str); + const std::map::const_iterator it = macros.find(tok->str()); if (it != macros.end()) { simplecpp::TokenList value(files); try { @@ -2233,7 +2236,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token simplecpp::Output out(files); out.type = simplecpp::Output::SYNTAX_ERROR; out.location = err.location; - out.msg = "failed to expand \'" + tok->str + "\', " + err.what; + out.msg = "failed to expand \'" + tok->str() + "\', " + err.what; outputList->push_back(out); } return false; @@ -2325,38 +2328,38 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } - if (ifstates.size() <= 1U && (rawtok->str == ELIF || rawtok->str == ELSE || rawtok->str == ENDIF)) { + if (ifstates.size() <= 1U && (rawtok->str() == ELIF || rawtok->str() == ELSE || rawtok->str() == ENDIF)) { if (outputList) { simplecpp::Output err(files); err.type = Output::SYNTAX_ERROR; err.location = rawtok->location; - err.msg = "#" + rawtok->str + " without #if"; + err.msg = "#" + rawtok->str() + " without #if"; outputList->push_back(err); } output.clear(); return; } - if (ifstates.top() == TRUE && (rawtok->str == ERROR || rawtok->str == WARNING)) { + if (ifstates.top() == TRUE && (rawtok->str() == ERROR || rawtok->str() == WARNING)) { if (outputList) { simplecpp::Output err(rawtok->location.files); - err.type = rawtok->str == ERROR ? Output::ERROR : Output::WARNING; + err.type = rawtok->str() == ERROR ? Output::ERROR : Output::WARNING; err.location = rawtok->location; for (const Token *tok = rawtok->next; tok && sameline(rawtok,tok); tok = tok->next) { - if (!err.msg.empty() && isNameChar(tok->str[0])) + if (!err.msg.empty() && isNameChar(tok->str()[0])) err.msg += ' '; - err.msg += tok->str; + err.msg += tok->str(); } - err.msg = '#' + rawtok->str + ' ' + err.msg; + err.msg = '#' + rawtok->str() + ' ' + err.msg; outputList->push_back(err); } - if (rawtok->str == ERROR) { + if (rawtok->str() == ERROR) { output.clear(); return; } } - if (rawtok->str == DEFINE) { + if (rawtok->str() == DEFINE) { if (ifstates.top() != TRUE) continue; try { @@ -2379,7 +2382,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL output.clear(); return; } - } else if (ifstates.top() == TRUE && rawtok->str == INCLUDE) { + } else if (ifstates.top() == TRUE && rawtok->str() == INCLUDE) { TokenList inc1(files); for (const Token *inctok = rawtok->next; sameline(rawtok,inctok); inctok = inctok->next) { if (!inctok->comment) @@ -2401,14 +2404,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL // TODO: Sometimes spaces must be added in the string // Somehow preprocessToken etc must be told that the location should be source location not destination location for (const Token *tok = inc2.cfront(); tok; tok = tok->next) { - hdr += tok->str; + hdr += tok->str(); } inc2.clear(); inc2.push_back(new Token(hdr, inc1.cfront()->location)); inc2.front()->op = '<'; } - if (inc2.empty() || inc2.cfront()->str.size() <= 2U) { + if (inc2.empty() || inc2.cfront()->str().size() <= 2U) { if (outputList) { simplecpp::Output err(files); err.type = Output::SYNTAX_ERROR; @@ -2423,7 +2426,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const Token *inctok = inc2.cfront(); const bool systemheader = (inctok->op == '<'); - const std::string header(realFilename(inctok->str.substr(1U, inctok->str.size() - 2U))); + const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui, systemheader); if (header2.empty()) { // try to load file.. @@ -2439,7 +2442,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL simplecpp::Output out(files); out.type = Output::MISSING_HEADER; out.location = rawtok->location; - out.msg = "Header not found: " + inctok->str; + out.msg = "Header not found: " + inctok->str(); outputList->push_back(out); } } else if (includetokenstack.size() >= 400) { @@ -2456,13 +2459,13 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok = includetokens ? includetokens->cfront() : 0; continue; } - } else if (rawtok->str == IF || rawtok->str == IFDEF || rawtok->str == IFNDEF || rawtok->str == ELIF) { + } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { if (!sameline(rawtok,rawtok->next)) { if (outputList) { simplecpp::Output out(files); out.type = Output::SYNTAX_ERROR; out.location = rawtok->location; - out.msg = "Syntax error in #" + rawtok->str; + out.msg = "Syntax error in #" + rawtok->str(); outputList->push_back(out); } output.clear(); @@ -2470,13 +2473,13 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } bool conditionIsTrue; - if (ifstates.top() == ALWAYS_FALSE || (ifstates.top() == ELSE_IS_TRUE && rawtok->str != ELIF)) + if (ifstates.top() == ALWAYS_FALSE || (ifstates.top() == ELSE_IS_TRUE && rawtok->str() != ELIF)) conditionIsTrue = false; - else if (rawtok->str == IFDEF) - conditionIsTrue = (macros.find(rawtok->next->str) != macros.end()); - else if (rawtok->str == IFNDEF) - conditionIsTrue = (macros.find(rawtok->next->str) == macros.end()); - else { /*if (rawtok->str == IF || rawtok->str == ELIF)*/ + else if (rawtok->str() == IFDEF) + conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end()); + else if (rawtok->str() == IFNDEF) + conditionIsTrue = (macros.find(rawtok->next->str()) == macros.end()); + else { /*if (rawtok->str() == IF || rawtok->str() == ELIF)*/ TokenList expr(files); for (const Token *tok = rawtok->next; tok && tok->location.sameline(rawtok->location); tok = tok->next) { if (!tok->name) { @@ -2484,13 +2487,13 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } - if (tok->str == DEFINED) { + if (tok->str() == DEFINED) { tok = tok->next; const bool par = (tok && tok->op == '('); if (par) tok = tok->next; if (tok) { - if (macros.find(tok->str) != macros.end()) + if (macros.find(tok->str()) != macros.end()) expr.push_back(new Token("1", tok->location)); else expr.push_back(new Token("0", tok->location)); @@ -2502,7 +2505,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL Output out(rawtok->location.files); out.type = Output::SYNTAX_ERROR; out.location = rawtok->location; - out.msg = "failed to evaluate " + std::string(rawtok->str == IF ? "#if" : "#elif") + " condition"; + out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; outputList->push_back(out); } output.clear(); @@ -2524,7 +2527,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL Output out(rawtok->location.files); out.type = Output::SYNTAX_ERROR; out.location = rawtok->location; - out.msg = "failed to evaluate " + std::string(rawtok->str == IF ? "#if" : "#elif") + " condition"; + out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; if (e.what() && *e.what()) out.msg += std::string(", ") + e.what(); outputList->push_back(out); @@ -2534,7 +2537,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } - if (rawtok->str != ELIF) { + if (rawtok->str() != ELIF) { // push a new ifstate.. if (ifstates.top() != TRUE) ifstates.push(ALWAYS_FALSE); @@ -2545,19 +2548,19 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (ifstates.top() == ELSE_IS_TRUE && conditionIsTrue) { ifstates.top() = TRUE; } - } else if (rawtok->str == ELSE) { + } else if (rawtok->str() == ELSE) { ifstates.top() = (ifstates.top() == ELSE_IS_TRUE) ? TRUE : ALWAYS_FALSE; - } else if (rawtok->str == ENDIF) { + } else if (rawtok->str() == ENDIF) { ifstates.pop(); - } else if (rawtok->str == UNDEF) { + } else if (rawtok->str() == UNDEF) { if (ifstates.top() == TRUE) { const Token *tok = rawtok->next; while (sameline(rawtok,tok) && tok->comment) tok = tok->next; if (sameline(rawtok, tok)) - macros.erase(tok->str); + macros.erase(tok->str()); } - } else if (ifstates.top() == TRUE && rawtok->str == PRAGMA && rawtok->next && rawtok->next->str == ONCE && sameline(rawtok,rawtok->next)) { + } else if (ifstates.top() == TRUE && rawtok->str() == PRAGMA && rawtok->next && rawtok->next->str() == ONCE && sameline(rawtok,rawtok->next)) { pragmaOnce.insert(rawtok->location.file()); } rawtok = gotoNextLine(rawtok); @@ -2592,11 +2595,11 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (hash || hashhash) { std::string s; for (const Token *hashtok = tokens.cfront(); hashtok; hashtok = hashtok->next) - s += hashtok->str; + s += hashtok->str(); if (hash) output.push_back(new Token('\"' + s + '\"', loc)); else if (output.back()) - output.back()->setstr(output.cback()->str + s); + output.back()->setstr(output.cback()->str() + s); else output.push_back(new Token(s, loc)); } else { diff --git a/simplecpp.h b/simplecpp.h index 19624e1d..62f7a586 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -87,7 +87,7 @@ namespace simplecpp { unsigned int line; unsigned int col; private: - const std::string emptyFileName; + static const std::string emptyFileName; }; /** @@ -97,22 +97,23 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc) : - str(string), location(loc), previous(NULL), next(NULL), string(s) { + location(loc), previous(NULL), next(NULL), string(s) { flags(); } Token(const Token &tok) : - str(string), macro(tok.macro), location(tok.location), previous(NULL), next(NULL), string(tok.str) { + macro(tok.macro), location(tok.location), previous(NULL), next(NULL), string(tok.string) { flags(); } void flags() { - name = (std::isalpha((unsigned char)str[0]) || str[0] == '_' || str[0] == '$'); - comment = (str.compare(0, 2, "//") == 0 || str.compare(0, 2, "/*") == 0); - number = std::isdigit((unsigned char)str[0]) || (str.size() > 1U && str[0] == '-' && std::isdigit((unsigned char)str[1])); - op = (str.size() == 1U) ? str[0] : '\0'; + name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$'); + comment = (string.compare(0, 2, "//") == 0 || string.compare(0, 2, "/*") == 0); + number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); + op = (string.size() == 1U) ? string[0] : '\0'; } + const TokenString& str() const { return string; } void setstr(const std::string &s) { string = s; flags(); @@ -122,7 +123,6 @@ namespace simplecpp { bool startsWithOneOf(const char c[]) const; bool endsWithOneOf(const char c[]) const; - const TokenString &str; TokenString macro; char op; bool comment; diff --git a/test.cpp b/test.cpp index 067913d8..bb12bdf2 100644 --- a/test.cpp +++ b/test.cpp @@ -1319,13 +1319,13 @@ static void tokenMacro2() simplecpp::TokenList tokenList(files); simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str); + ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("", tok->macro); tok = tok->next; - ASSERT_EQUALS("+", tok->str); + ASSERT_EQUALS("+", tok->str()); ASSERT_EQUALS("ADD", tok->macro); tok = tok->next; - ASSERT_EQUALS("2", tok->str); + ASSERT_EQUALS("2", tok->str()); ASSERT_EQUALS("", tok->macro); } @@ -1341,13 +1341,13 @@ static void tokenMacro3() simplecpp::TokenList tokenList(files); simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str); + ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("FRED", tok->macro); tok = tok->next; - ASSERT_EQUALS("+", tok->str); + ASSERT_EQUALS("+", tok->str()); ASSERT_EQUALS("ADD", tok->macro); tok = tok->next; - ASSERT_EQUALS("2", tok->str); + ASSERT_EQUALS("2", tok->str()); ASSERT_EQUALS("", tok->macro); } @@ -1363,7 +1363,7 @@ static void tokenMacro4() simplecpp::TokenList tokenList(files); simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); const simplecpp::Token *tok = tokenList.cfront(); - ASSERT_EQUALS("1", tok->str); + ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("A", tok->macro); } From 0427b19041288dc897c1dd28d318f772c994e8dd Mon Sep 17 00:00:00 2001 From: PKEuS Date: Mon, 14 May 2018 10:46:56 +0200 Subject: [PATCH 054/381] Simplified code: Explicit constructor call is unnecessary --- simplecpp.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 70ab3729..81b5e731 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2253,26 +2253,26 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage) { std::map sizeOfType(rawtokens.sizeOfType); - sizeOfType.insert(std::pair(std::string("char"), sizeof(char))); - sizeOfType.insert(std::pair(std::string("short"), sizeof(short))); - sizeOfType.insert(std::pair(std::string("short int"), sizeOfType["short"])); - sizeOfType.insert(std::pair(std::string("int"), sizeof(int))); - sizeOfType.insert(std::pair(std::string("long"), sizeof(long))); - sizeOfType.insert(std::pair(std::string("long int"), sizeOfType["long"])); - sizeOfType.insert(std::pair(std::string("long long"), sizeof(long long))); - sizeOfType.insert(std::pair(std::string("float"), sizeof(float))); - sizeOfType.insert(std::pair(std::string("double"), sizeof(double))); - sizeOfType.insert(std::pair(std::string("long double"), sizeof(long double))); - sizeOfType.insert(std::pair(std::string("char *"), sizeof(char *))); - sizeOfType.insert(std::pair(std::string("short *"), sizeof(short *))); - sizeOfType.insert(std::pair(std::string("short int *"), sizeOfType["short *"])); - sizeOfType.insert(std::pair(std::string("int *"), sizeof(int *))); - sizeOfType.insert(std::pair(std::string("long *"), sizeof(long *))); - sizeOfType.insert(std::pair(std::string("long int *"), sizeOfType["long *"])); - sizeOfType.insert(std::pair(std::string("long long *"), sizeof(long long *))); - sizeOfType.insert(std::pair(std::string("float *"), sizeof(float *))); - sizeOfType.insert(std::pair(std::string("double *"), sizeof(double *))); - sizeOfType.insert(std::pair(std::string("long double *"), sizeof(long double *))); + sizeOfType.insert(std::pair("char", sizeof(char))); + sizeOfType.insert(std::pair("short", sizeof(short))); + sizeOfType.insert(std::pair("short int", sizeOfType["short"])); + sizeOfType.insert(std::pair("int", sizeof(int))); + sizeOfType.insert(std::pair("long", sizeof(long))); + sizeOfType.insert(std::pair("long int", sizeOfType["long"])); + sizeOfType.insert(std::pair("long long", sizeof(long long))); + sizeOfType.insert(std::pair("float", sizeof(float))); + sizeOfType.insert(std::pair("double", sizeof(double))); + sizeOfType.insert(std::pair("long double", sizeof(long double))); + sizeOfType.insert(std::pair("char *", sizeof(char *))); + sizeOfType.insert(std::pair("short *", sizeof(short *))); + sizeOfType.insert(std::pair("short int *", sizeOfType["short *"])); + sizeOfType.insert(std::pair("int *", sizeof(int *))); + sizeOfType.insert(std::pair("long *", sizeof(long *))); + sizeOfType.insert(std::pair("long int *", sizeOfType["long *"])); + sizeOfType.insert(std::pair("long long *", sizeof(long long *))); + sizeOfType.insert(std::pair("float *", sizeof(float *))); + sizeOfType.insert(std::pair("double *", sizeof(double *))); + sizeOfType.insert(std::pair("long double *", sizeof(long double *))); std::map macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { From cabd704895abef9fd4c16ab4e77d3036bea760d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 7 Aug 2018 09:51:37 +0200 Subject: [PATCH 055/381] astyle formatting --- simplecpp.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/simplecpp.h b/simplecpp.h index 62f7a586..95e35085 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -113,7 +113,9 @@ namespace simplecpp { op = (string.size() == 1U) ? string[0] : '\0'; } - const TokenString& str() const { return string; } + const TokenString& str() const { + return string; + } void setstr(const std::string &s) { string = s; flags(); From e0c7b0b400b86e5b12f7f592f41a250b7fa1dda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 7 Aug 2018 10:17:15 +0200 Subject: [PATCH 056/381] code cleanup --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 81b5e731..1b655d78 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1089,7 +1089,7 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (!ret.empty()) ret = ' ' + ret; ret = (tok->str()[0] == '\"' ? std::string("%str%") - : std::isdigit(static_cast(tok->str()[0])) ? std::string("%num%") : tok->str()) + ret; + : tok->number ? std::string("%num%") : tok->str()) + ret; if (++count > maxsize) return ""; } From 4e83a5c95fd6eedf7437a9a1ca66ba58d377f2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 7 Aug 2018 21:20:18 +0200 Subject: [PATCH 057/381] Fix issue with multiline strings in macro. Fixes #128 --- Makefile | 4 ++-- simplecpp.h | 2 +- test.cpp | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 222d23c7..93da951d 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,13 @@ all: testrunner simplecpp CXXFLAGS = -Wall -Wextra -Wabi -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -pedantic -g -std=c++0x LDFLAGS = -g -%.o: %.cpp +%.o: %.cpp simplecpp.h $(CXX) $(CXXFLAGS) -c $< testrunner: test.o simplecpp.o $(CXX) $(LDFLAGS) simplecpp.o test.o -o testrunner - + test: testrunner simplecpp # The -std=c++03 makes sure that simplecpp.cpp is C++03 conformant. We don't require a C++11 compiler g++ -std=c++03 -fsyntax-only simplecpp.cpp && ./testrunner && python run-tests.py diff --git a/simplecpp.h b/simplecpp.h index 95e35085..d58fda07 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -260,7 +260,7 @@ namespace simplecpp { std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList); - std::string lastLine(int maxsize=10) const; + std::string lastLine(int maxsize=100000) const; unsigned int fileIndex(const std::string &filename); diff --git a/test.cpp b/test.cpp index bb12bdf2..d039d76e 100644 --- a/test.cpp +++ b/test.cpp @@ -1039,6 +1039,20 @@ static void multiline6() // multiline string in macro "string", rawtokens.stringify()); } +static void multiline7() // multiline string in macro +{ + const char code[] = "#define A(X) aaa { f(\"\\\n" + "a\"); }\n" + "A(1)"; + const simplecpp::DUI dui; + std::istringstream istr(code); + std::vector files; + simplecpp::TokenList rawtokens(istr,files); + ASSERT_EQUALS("# define A ( X ) aaa { f ( \"a\" ) ; }\n" + "\n" + "A ( 1 )", rawtokens.stringify()); +} + static void nullDirective1() { const char code[] = "#\n" @@ -1635,6 +1649,7 @@ int main(int argc, char **argv) TEST_CASE(multiline4); TEST_CASE(multiline5); // column TEST_CASE(multiline6); // multiline string in macro + TEST_CASE(multiline7); // multiline string in macro TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_string); From 00af8774c91ebea85c7ba4fc216133087a2cdb2e Mon Sep 17 00:00:00 2001 From: PKEuS Date: Tue, 25 Sep 2018 13:43:01 +0200 Subject: [PATCH 058/381] Updated runastyle.bat (synchronized with cppcheck) --- runastyle.bat | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/runastyle.bat b/runastyle.bat index e0fab6f9..b8f11561 100644 --- a/runastyle.bat +++ b/runastyle.bat @@ -1,7 +1,26 @@ @REM Script to run AStyle on the sources +@REM The version check in this script is used to avoid commit battles +@REM between different developers that use different astyle versions as +@REM different versions might have different output (this has happened in +@REM the past). -@SET STYLE=--style=stroustrup --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0 -@SET OPTIONS=--pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines +@REM If project management wishes to take a newer astyle version into use +@REM just change this string to match the start of astyle version string. +@SET ASTYLE_VERSION="Artistic Style Version 3.0.1" +@SET ASTYLE="astyle" -astyle %STYLE% %OPTIONS% *.h -astyle %STYLE% %OPTIONS% *.cpp +@SET DETECTED_VERSION="" +@FOR /F "tokens=*" %%i IN ('%ASTYLE% --version') DO SET DETECTED_VERSION=%%i +@ECHO %DETECTED_VERSION% | FINDSTR /B /C:%ASTYLE_VERSION% > nul && ( + ECHO "%DETECTED_VERSION%" matches %ASTYLE_VERSION% +) || ( + ECHO You should use: %ASTYLE_VERSION% + ECHO Detected: "%DETECTED_VERSION%" + GOTO EXIT_ERROR +) + +@SET STYLE=--style=kr --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0 +@SET OPTIONS=--pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines --attach-classes --attach-namespaces + +%ASTYLE% %STYLE% %OPTIONS% *.cpp +%ASTYLE% %STYLE% %OPTIONS% *.h \ No newline at end of file From 885a27fbfe65abff66c01dcb74f901e64e342674 Mon Sep 17 00:00:00 2001 From: PKEuS Date: Tue, 9 Oct 2018 20:58:54 +0200 Subject: [PATCH 059/381] Save if a macro value is known (cppcheck ticket #8404) --- simplecpp.cpp | 18 +++++++++++++----- simplecpp.h | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1b655d78..5d92006e 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1110,9 +1110,9 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) namespace simplecpp { class Macro { public: - explicit Macro(std::vector &f) : nameTokDef(NULL), variadic(false), valueToken(NULL), endToken(NULL), files(f), tokenListDefine(f) {} + explicit Macro(std::vector &f) : nameTokDef(NULL), variadic(false), valueToken(NULL), endToken(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(false) {} - Macro(const Token *tok, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f) { + Macro(const Token *tok, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previous, tok)) throw std::runtime_error("bad macro syntax"); if (tok->op != '#') @@ -1128,7 +1128,7 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax"); } - Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f) { + Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); std::istringstream istr(def); tokenListDefine.readfile(istr); @@ -1136,12 +1136,13 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax"); } - Macro(const Macro ¯o) : nameTokDef(NULL), files(macro.files), tokenListDefine(macro.files) { + Macro(const Macro ¯o) : nameTokDef(NULL), files(macro.files), tokenListDefine(macro.files), valueDefinedInCode_(macro.valueDefinedInCode_) { *this = macro; } void operator=(const Macro ¯o) { if (this != ¯o) { + valueDefinedInCode_ = macro.valueDefinedInCode_; if (macro.tokenListDefine.empty()) parseDefine(macro.nameTokDef); else { @@ -1151,6 +1152,10 @@ namespace simplecpp { } } + bool valueDefinedInCode() const { + return valueDefinedInCode_; + } + /** * Expand macro. This will recursively expand inner macros. * @param output destination tokenlist @@ -1821,6 +1826,9 @@ namespace simplecpp { /** usage of this macro */ mutable std::list usageList; + + /** was the value of this macro actually defined in the code? */ + bool valueDefinedInCode_; }; } @@ -2612,7 +2620,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const Macro ¯o = macroIt->second; const std::list &usage = macro.usage(); for (std::list::const_iterator usageIt = usage.begin(); usageIt != usage.end(); ++usageIt) { - MacroUsage mu(usageIt->files); + MacroUsage mu(usageIt->files, macro.valueDefinedInCode()); mu.macroName = macro.name(); mu.macroLocation = macro.defineLocation(); mu.useLocation = *usageIt; diff --git a/simplecpp.h b/simplecpp.h index d58fda07..b4bce4c1 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -271,10 +271,11 @@ namespace simplecpp { /** Tracking how macros are used */ struct SIMPLECPP_LIB MacroUsage { - explicit MacroUsage(const std::vector &f) : macroLocation(f), useLocation(f) {} + explicit MacroUsage(const std::vector &f, bool macroValueKnown_) : macroLocation(f), useLocation(f), macroValueKnown(macroValueKnown_) {} std::string macroName; Location macroLocation; Location useLocation; + bool macroValueKnown; }; /** From a765b88dc6aa6d4562acfc13aeee8e09adc9b733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 2 Nov 2018 16:27:45 +0100 Subject: [PATCH 060/381] Handle '# %num% %str%' locations generated by gcc preprocessor --- simplecpp.cpp | 5 ++++- test.cpp | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5d92006e..2f4552d5 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -439,7 +439,6 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (oldLastToken != cback()) { oldLastToken = cback(); const std::string lastline(lastLine()); - if (lastline == "# file %str%") { loc.push(location); location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); @@ -451,6 +450,10 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen loc.push(location); location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = std::atol(cback()->previous->str().c_str()); + } else if (lastline == "# %num% %str%") { + loc.push(location); + location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); + location.line = std::atol(cback()->previous->str().c_str()); } // #endfile else if (lastline == "# endfile" && !loc.empty()) { diff --git a/test.cpp b/test.cpp index d039d76e..6380f5d0 100644 --- a/test.cpp +++ b/test.cpp @@ -894,6 +894,15 @@ static void ifalt() // using "and", "or", etc ASSERT_EQUALS("\n1", preprocess(code)); } +static void location1() +{ + const char *code; + + code = "# 1 \"main.c\"\n\n\n" + "x"; + ASSERT_EQUALS("\n#line 3 \"main.c\"\nx", preprocess(code)); +} + static void missingHeader1() { const simplecpp::DUI dui; @@ -1626,6 +1635,8 @@ int main(int argc, char **argv) TEST_CASE(ifdiv0); TEST_CASE(ifalt); // using "and", "or", etc + TEST_CASE(location1); + TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); TEST_CASE(missingHeader3); From fa221c4fcf335a869e289f8ddeb065fa00164659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 25 Nov 2018 14:26:58 +0100 Subject: [PATCH 061/381] Some more details about invalid macro --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2f4552d5..b66474ff 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1136,7 +1136,7 @@ namespace simplecpp { std::istringstream istr(def); tokenListDefine.readfile(istr); if (!parseDefine(tokenListDefine.cfront())) - throw std::runtime_error("bad macro syntax"); + throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); } Macro(const Macro ¯o) : nameTokDef(NULL), files(macro.files), tokenListDefine(macro.files), valueDefinedInCode_(macro.valueDefinedInCode_) { From 25bdc35acc1a89b15851b649709d7903cd2f71dc Mon Sep 17 00:00:00 2001 From: Diorcet Yann Date: Mon, 26 Nov 2018 20:26:44 +0100 Subject: [PATCH 062/381] Fix null directive after define (#139) --- simplecpp.cpp | 2 +- test.cpp | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index b66474ff..64a9baa6 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1505,7 +1505,7 @@ namespace simplecpp { for (const Token *tok = valueToken; tok != endToken;) { if (tok->op != '#') { // A##B => AB - if (tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { + if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { if (!sameline(tok, tok->next->next->next)) throw invalidHashHash(tok->location, name()); output->push_back(newMacroToken(expandArgStr(tok, parametertokens2), loc, isReplaced(expandedmacros))); diff --git a/test.cpp b/test.cpp index 6380f5d0..76384912 100644 --- a/test.cpp +++ b/test.cpp @@ -1086,6 +1086,18 @@ static void nullDirective2() ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); } +static void nullDirective3() +{ + const char code[] = "#if 1\n" + "#define a 1\n" + "#\n" + "#endif\n" + "x = a;\n"; + + const simplecpp::DUI dui; + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); +} + static void include1() { const char code[] = "#include \"A.h\"\n"; @@ -1644,6 +1656,7 @@ int main(int argc, char **argv) TEST_CASE(nullDirective1); TEST_CASE(nullDirective2); + TEST_CASE(nullDirective3); TEST_CASE(include1); TEST_CASE(include2); From 11c6d6f34a9bcc6c62c74a8df73b4488d7ff2711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 24 Dec 2018 08:23:46 +0100 Subject: [PATCH 063/381] Handle multiline #error directives --- simplecpp.cpp | 5 ++++- test.cpp | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 64a9baa6..52fa2fcf 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -473,12 +473,15 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen TokenString currentToken; if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && (lastLine() == "# error" || lastLine() == "# warning")) { - while (istr.good() && ch != '\r' && ch != '\n') { + char prev = ' '; + while (istr.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; + prev = ch; ch = readChar(istr, bom); } istr.unget(); push_back(new Token(currentToken, location)); + location.adjust(currentToken); continue; } diff --git a/test.cpp b/test.cpp index 76384912..eef1ef78 100644 --- a/test.cpp +++ b/test.cpp @@ -506,6 +506,16 @@ static void error2() ASSERT_EQUALS("file0,1,#error,#error it's an error\n", toString(outputList)); } +static void error3() +{ + std::istringstream istr("#error \"bla bla\\\n" + " bla bla.\"\n"); + std::vector files; + simplecpp::OutputList outputList; + simplecpp::TokenList rawtokens(istr, files, "test.c", &outputList); + ASSERT_EQUALS("", toString(outputList)); +} + static void garbage() { const simplecpp::DUI dui; @@ -1612,6 +1622,7 @@ int main(int argc, char **argv) TEST_CASE(error1); TEST_CASE(error2); + TEST_CASE(error3); TEST_CASE(garbage); TEST_CASE(garbage_endif); From 1f603f57c92fe523bba92d25648dd2ff654fd9a6 Mon Sep 17 00:00:00 2001 From: mitsujin Date: Wed, 6 Feb 2019 23:17:53 +0800 Subject: [PATCH 064/381] Cache results of FindFirstFileExA to improve performance on Windows (#143) * Cache results of FindFirstFileExA to improve performance on Windows * Review actions * Review actions - const string& --- simplecpp.cpp | 82 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 6 deletions(-) mode change 100644 => 100755 simplecpp.cpp diff --git a/simplecpp.cpp b/simplecpp.cpp old mode 100644 new mode 100755 index 52fa2fcf..b2dfd4e5 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1839,6 +1839,72 @@ namespace simplecpp { } #ifdef SIMPLECPP_WINDOWS + +class ScopedLock +{ +public: + explicit ScopedLock(HANDLE mutex) + : m_mutex(mutex) + { + if (WaitForSingleObject(m_mutex, INFINITE) == WAIT_FAILED) + throw std::runtime_error("cannot lock the mutex"); + } + + ~ScopedLock() + { + ReleaseMutex(m_mutex); + } + +private: + ScopedLock& operator=(const ScopedLock&); + ScopedLock(const ScopedLock&); + + HANDLE m_mutex; +}; + +class RealFileNameMap +{ +public: + RealFileNameMap() + { + m_mutex = CreateMutex(NULL, FALSE, NULL); + + if (!m_mutex) + { + throw std::runtime_error("cannot create the mutex handle"); + } + } + + ~RealFileNameMap() + { + CloseHandle(m_mutex); + } + + bool getRealPathFromCache(const std::string& path, std::string* returnPath) + { + ScopedLock lock(m_mutex); + + std::map::iterator it = m_fileMap.find(path); + if (it != m_fileMap.end()) + { + *returnPath = it->second; + return true; + } + return false; + } + + void addToCache(const std::string& path, const std::string& actualPath) + { + ScopedLock lock(m_mutex); + m_fileMap[path] = actualPath; + } + + std::map m_fileMap; + HANDLE m_mutex; +}; + +static RealFileNameMap realFileNameMap; + static bool realFileName(const std::string &f, std::string *result) { // are there alpha characters in last subpath? @@ -1858,13 +1924,17 @@ static bool realFileName(const std::string &f, std::string *result) return false; // Lookup filename or foldername on file system - WIN32_FIND_DATAA FindFileData; - HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); + if (!realFileNameMap.getRealPathFromCache(f, result)) + { + WIN32_FIND_DATAA FindFileData; + HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); - if (INVALID_HANDLE_VALUE == hFind) - return false; - *result = FindFileData.cFileName; - FindClose(hFind); + if (INVALID_HANDLE_VALUE == hFind) + return false; + *result = FindFileData.cFileName; + realFileNameMap.addToCache(f, *result); + FindClose(hFind); + } return true; } From b9a7fc45cc0daf8bae32743b36279c68a3f5fb13 Mon Sep 17 00:00:00 2001 From: HolgiHo <47777278+HolgiHo@users.noreply.github.com> Date: Wed, 20 Feb 2019 15:25:50 +0100 Subject: [PATCH 065/381] use critical section in ScopedLock (faster than mutex) (#145) --- simplecpp.cpp | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index b2dfd4e5..5764e267 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1843,23 +1843,22 @@ namespace simplecpp { class ScopedLock { public: - explicit ScopedLock(HANDLE mutex) - : m_mutex(mutex) + explicit ScopedLock(CRITICAL_SECTION& criticalSection) + : m_criticalSection(criticalSection) { - if (WaitForSingleObject(m_mutex, INFINITE) == WAIT_FAILED) - throw std::runtime_error("cannot lock the mutex"); + EnterCriticalSection(&m_criticalSection); } ~ScopedLock() { - ReleaseMutex(m_mutex); + LeaveCriticalSection(&m_criticalSection); } private: ScopedLock& operator=(const ScopedLock&); ScopedLock(const ScopedLock&); - HANDLE m_mutex; + CRITICAL_SECTION& m_criticalSection; }; class RealFileNameMap @@ -1867,22 +1866,17 @@ class RealFileNameMap public: RealFileNameMap() { - m_mutex = CreateMutex(NULL, FALSE, NULL); - - if (!m_mutex) - { - throw std::runtime_error("cannot create the mutex handle"); - } + InitializeCriticalSection(&m_criticalSection); } ~RealFileNameMap() { - CloseHandle(m_mutex); + DeleteCriticalSection(&m_criticalSection); } bool getRealPathFromCache(const std::string& path, std::string* returnPath) { - ScopedLock lock(m_mutex); + ScopedLock lock(m_criticalSection); std::map::iterator it = m_fileMap.find(path); if (it != m_fileMap.end()) @@ -1895,12 +1889,13 @@ class RealFileNameMap void addToCache(const std::string& path, const std::string& actualPath) { - ScopedLock lock(m_mutex); + ScopedLock lock(m_criticalSection); m_fileMap[path] = actualPath; } - + +private: std::map m_fileMap; - HANDLE m_mutex; + CRITICAL_SECTION m_criticalSection; }; static RealFileNameMap realFileNameMap; From d332fa1ba402c6266cfe436b34e43fdf78a377a2 Mon Sep 17 00:00:00 2001 From: rikardfalkeborn Date: Thu, 21 Feb 2019 06:42:27 +0100 Subject: [PATCH 066/381] Fix ## of assignement operators. Fixes #141 (#147) Previously, simplecpp would throw an error for invalid usage of ## for the following code: #define ADD_OPERATOR(OP) void operator OP ## = (void) { x = x OP 1; } ADD_OPERATOR(+) This commit allows the above macro work with all allowed c and c++ compound assignment operators. Note that #define A < ## < ## = A will still throw an error. --- simplecpp.cpp | 10 ++++++++-- test.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5764e267..6f5902ef 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1744,11 +1744,17 @@ namespace simplecpp { throw invalidHashHash(tok->location, name()); if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) throw invalidHashHash(tok->location, name()); - if (!A->name && !A->number && A->op != ',' && !A->str().empty()) + + bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; + if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual) throw invalidHashHash(tok->location, name()); Token *B = tok->next->next; - if (!B->name && !B->number && B->op && B->op != '#') + if (!B->name && !B->number && B->op && !B->isOneOf("#=")) + throw invalidHashHash(tok->location, name()); + + if ((canBeConcatenatedWithEqual && B->op != '=') || + (!canBeConcatenatedWithEqual && B->op == '=')) throw invalidHashHash(tok->location, name()); std::string strAB; diff --git a/test.cpp b/test.cpp index eef1ef78..8dc2ca0e 100644 --- a/test.cpp +++ b/test.cpp @@ -644,6 +644,54 @@ static void hashhash8() ASSERT_EQUALS("\nxy = 123 ;", preprocess(code)); } +static void hashhash9() +{ + const char * code = "#define ADD_OPERATOR(OP) void operator OP ## = (void) { x = x OP 1; }\n" + "ADD_OPERATOR(+);\n" + "ADD_OPERATOR(-);\n" + "ADD_OPERATOR(*);\n" + "ADD_OPERATOR(/);\n" + "ADD_OPERATOR(%);\n" + "ADD_OPERATOR(&);\n" + "ADD_OPERATOR(|);\n" + "ADD_OPERATOR(^);\n" + "ADD_OPERATOR(<<);\n" + "ADD_OPERATOR(>>);\n"; + const char expected[] = "\n" + "void operator += ( void ) { x = x + 1 ; } ;\n" + "void operator -= ( void ) { x = x - 1 ; } ;\n" + "void operator *= ( void ) { x = x * 1 ; } ;\n" + "void operator /= ( void ) { x = x / 1 ; } ;\n" + "void operator %= ( void ) { x = x % 1 ; } ;\n" + "void operator &= ( void ) { x = x & 1 ; } ;\n" + "void operator |= ( void ) { x = x | 1 ; } ;\n" + "void operator ^= ( void ) { x = x ^ 1 ; } ;\n" + "void operator <<= ( void ) { x = x << 1 ; } ;\n" + "void operator >>= ( void ) { x = x >> 1 ; } ;"; + ASSERT_EQUALS(expected, preprocess(code)); + + const simplecpp::DUI dui; + simplecpp::OutputList outputList; + + code = "#define A +##x\n" + "A"; + outputList.clear(); + preprocess(code, dui, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); + + code = "#define A 2##=\n" + "A"; + outputList.clear(); + preprocess(code, dui, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); + + code = "#define A <<##x\n" + "A"; + outputList.clear(); + preprocess(code, dui, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); +} + static void hashhash_invalid_1() { std::istringstream istr("#define f(a) (##x)\nf(1)"); @@ -1636,6 +1684,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash6); TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) TEST_CASE(hashhash8); + TEST_CASE(hashhash9); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); From e670f7afb62e3f7df8dd3d13349ec5313cf70e35 Mon Sep 17 00:00:00 2001 From: rikardfalkeborn Date: Thu, 21 Feb 2019 06:46:16 +0100 Subject: [PATCH 067/381] Minor fixes to getAndSkipBOM() (#148) Add a missing call to istream.unget() in case the file starts with 0xfe or 0xff but does not continue with 0xff. This makes simplecpp abort parsing immediately on such files, instead of silently ignoring the first character and continue parsing. Also, save return value of istream.peek() as int. If the file is empty, istream.peek() returns EOF (a negative int value, typically -1). It was silently converted to unsigned char, which made it take a somewhat unintentional path before returning. Note that there was no erroneous parsing, or undefined behaviour. Fixes #133. --- simplecpp.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6f5902ef..bad1f7bb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -329,13 +329,14 @@ static void ungetChar(std::istream &istr, unsigned int bom) static unsigned short getAndSkipBOM(std::istream &istr) { - const unsigned char ch1 = istr.peek(); + const int ch1 = istr.peek(); // The UTF-16 BOM is 0xfffe or 0xfeff. if (ch1 >= 0xfe) { unsigned short bom = ((unsigned char)istr.get() << 8); if (istr.peek() >= 0xfe) return bom | (unsigned char)istr.get(); + istr.unget(); return 0; } From bc2880b30cdba693ec148fa4f2b4388e2d0a51d2 Mon Sep 17 00:00:00 2001 From: Holger Hoffmann <47777278+HolgiHo@users.noreply.github.com> Date: Sat, 23 Feb 2019 08:05:37 +0100 Subject: [PATCH 068/381] Use realFileNameMap cache also in outer realFilename function (#150) --- simplecpp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index bad1f7bb..2fb9fe41 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1945,6 +1945,8 @@ static std::string realFilename(const std::string &f) { std::string ret; ret.reserve(f.size()); // this will be the final size + if (realFileNameMap.getRealPathFromCache(f, &ret)) + return ret; // Current subpath std::string subpath; @@ -1984,6 +1986,7 @@ static std::string realFilename(const std::string &f) ret += subpath; } + realFileNameMap.addToCache(f, ret); return ret; } From f7b2438de0842117ad6582b01e9fba7d6e6e4a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 23 Feb 2019 08:34:20 +0100 Subject: [PATCH 069/381] astyle formatting --- simplecpp.cpp | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2fb9fe41..2613e098 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1847,17 +1847,14 @@ namespace simplecpp { #ifdef SIMPLECPP_WINDOWS -class ScopedLock -{ +class ScopedLock { public: explicit ScopedLock(CRITICAL_SECTION& criticalSection) - : m_criticalSection(criticalSection) - { + : m_criticalSection(criticalSection) { EnterCriticalSection(&m_criticalSection); } - ~ScopedLock() - { + ~ScopedLock() { LeaveCriticalSection(&m_criticalSection); } @@ -1868,38 +1865,32 @@ class ScopedLock CRITICAL_SECTION& m_criticalSection; }; -class RealFileNameMap -{ +class RealFileNameMap { public: - RealFileNameMap() - { + RealFileNameMap() { InitializeCriticalSection(&m_criticalSection); } - ~RealFileNameMap() - { + ~RealFileNameMap() { DeleteCriticalSection(&m_criticalSection); } - bool getRealPathFromCache(const std::string& path, std::string* returnPath) - { + bool getRealPathFromCache(const std::string& path, std::string* returnPath) { ScopedLock lock(m_criticalSection); std::map::iterator it = m_fileMap.find(path); - if (it != m_fileMap.end()) - { + if (it != m_fileMap.end()) { *returnPath = it->second; return true; } return false; } - void addToCache(const std::string& path, const std::string& actualPath) - { + void addToCache(const std::string& path, const std::string& actualPath) { ScopedLock lock(m_criticalSection); m_fileMap[path] = actualPath; } - + private: std::map m_fileMap; CRITICAL_SECTION m_criticalSection; @@ -1926,8 +1917,7 @@ static bool realFileName(const std::string &f, std::string *result) return false; // Lookup filename or foldername on file system - if (!realFileNameMap.getRealPathFromCache(f, result)) - { + if (!realFileNameMap.getRealPathFromCache(f, result)) { WIN32_FIND_DATAA FindFileData; HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); @@ -1946,7 +1936,7 @@ static std::string realFilename(const std::string &f) std::string ret; ret.reserve(f.size()); // this will be the final size if (realFileNameMap.getRealPathFromCache(f, &ret)) - return ret; + return ret; // Current subpath std::string subpath; From a1c5d83c157e3a873085ca45cc77d68a3bbf1d3c Mon Sep 17 00:00:00 2001 From: Holger Hoffmann <47777278+HolgiHo@users.noreply.github.com> Date: Sun, 3 Mar 2019 18:49:18 +0100 Subject: [PATCH 070/381] Cache non-existing header files to skip expensive file open call (#152) * Cache non-existing header files to skip expensive file open call (Windows only) * run astyle --- simplecpp.cpp | 73 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2613e098..ed1e4887 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2164,23 +2164,73 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) return tok; } +#ifdef SIMPLECPP_WINDOWS + +class NonExistingFilesCache { +public: + NonExistingFilesCache() { + InitializeCriticalSection(&m_criticalSection); + } + + ~NonExistingFilesCache() { + DeleteCriticalSection(&m_criticalSection); + } + + bool contains(const std::string& path) { + ScopedLock lock(m_criticalSection); + return (m_pathSet.find(path) != m_pathSet.end()); + } + + void add(const std::string& path) { + ScopedLock lock(m_criticalSection); + m_pathSet.insert(path); + } + +private: + std::set m_pathSet; + CRITICAL_SECTION m_criticalSection; +}; + +static NonExistingFilesCache nonExistingFilesCache; + +#endif + +static std::string _openHeader(std::ifstream &f, const std::string &path) +{ +#ifdef SIMPLECPP_WINDOWS + std::string simplePath = simplecpp::simplifyPath(path); + if (nonExistingFilesCache.contains(simplePath)) + return ""; // file is known not to exist, skip expensive file open call + + f.open(simplePath.c_str()); + if (f.is_open()) + return simplePath; + else { + nonExistingFilesCache.add(simplePath); + return ""; + } +#else + f.open(path.c_str()); + return f.is_open() ? simplecpp::simplifyPath(path) : ""; +#endif +} + static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { if (isAbsolutePath(header)) { - f.open(header.c_str()); - return f.is_open() ? simplecpp::simplifyPath(header) : ""; + return _openHeader(f, header); } if (!systemheader) { if (sourcefile.find_first_of("\\/") != std::string::npos) { const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - f.open(s.c_str()); - if (f.is_open()) - return simplecpp::simplifyPath(s); + std::string simplePath = _openHeader(f, s); + if (!simplePath.empty()) + return simplePath; } else { - f.open(header.c_str()); - if (f.is_open()) - return simplecpp::simplifyPath(header); + std::string simplePath = _openHeader(f, header); + if (!simplePath.empty()) + return simplePath; } } @@ -2189,9 +2239,10 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') s += '/'; s += header; - f.open(s.c_str()); - if (f.is_open()) - return simplecpp::simplifyPath(s); + + std::string simplePath = _openHeader(f, s); + if (!simplePath.empty()) + return simplePath; } return ""; From a5977dfb1688be2bd059b61d98bb581c794ceb66 Mon Sep 17 00:00:00 2001 From: Holger Hoffmann <47777278+HolgiHo@users.noreply.github.com> Date: Wed, 6 Mar 2019 12:51:45 +0100 Subject: [PATCH 071/381] 3 fixes for realFileName/realFilename (#153) * fix for loop condition in realFileName: scan also f[0] use separate caches for realFilename (stores entire paths) and realFileName (stores file names) convert Cygwin paths to Windows paths in call to FindFirstFileExA added testcase for convertCygwinToWindowsPath * realFilename: do not call FindFirstFileExA for Windows drive specification only (like "C:") * put test case convertCygwinPath in alphabetical order --- simplecpp.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++++------ simplecpp.h | 3 +++ test.cpp | 20 +++++++++++++++++ 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ed1e4887..cbe6bdf1 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1845,6 +1845,39 @@ namespace simplecpp { }; } +namespace simplecpp { + + std::string convertCygwinToWindowsPath(const std::string &cygwinPath) + { + std::string windowsPath; + + std::string::size_type pos = 0; + if (cygwinPath.size() >= 11 && startsWith(cygwinPath, "/cygdrive/")) { + unsigned char driveLetter = cygwinPath[10]; + if (std::isalpha(driveLetter)) { + if (cygwinPath.size() == 11) { + windowsPath = toupper(driveLetter); + windowsPath += ":\\"; // volume root directory + pos = 11; + } else if (cygwinPath[11] == '/') { + windowsPath = toupper(driveLetter); + windowsPath += ":"; + pos = 11; + } + } + } + + for (; pos < cygwinPath.size(); ++pos) { + unsigned char c = cygwinPath[pos]; + if (c == '/') + c = '\\'; + windowsPath += c; + } + + return windowsPath; + } +} + #ifdef SIMPLECPP_WINDOWS class ScopedLock { @@ -1875,7 +1908,7 @@ class RealFileNameMap { DeleteCriticalSection(&m_criticalSection); } - bool getRealPathFromCache(const std::string& path, std::string* returnPath) { + bool getCacheEntry(const std::string& path, std::string* returnPath) { ScopedLock lock(m_criticalSection); std::map::iterator it = m_fileMap.find(path); @@ -1902,9 +1935,9 @@ static bool realFileName(const std::string &f, std::string *result) { // are there alpha characters in last subpath? bool alpha = false; - for (std::string::size_type pos = 1; pos < f.size(); ++pos) { + for (std::string::size_type pos = 1; pos <= f.size(); ++pos) { unsigned char c = f[f.size() - pos]; - if (c=='/' || c=='\\') + if (c == '/' || c == '\\') break; if (std::isalpha(c)) { alpha = true; @@ -1917,9 +1950,16 @@ static bool realFileName(const std::string &f, std::string *result) return false; // Lookup filename or foldername on file system - if (!realFileNameMap.getRealPathFromCache(f, result)) { + if (!realFileNameMap.getCacheEntry(f, result)) { + WIN32_FIND_DATAA FindFileData; + +#ifdef __CYGWIN__ + std::string fConverted = simplecpp::convertCygwinToWindowsPath(f); + HANDLE hFind = FindFirstFileExA(fConverted.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); +#else HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); +#endif if (INVALID_HANDLE_VALUE == hFind) return false; @@ -1930,12 +1970,14 @@ static bool realFileName(const std::string &f, std::string *result) return true; } +static RealFileNameMap realFilePathMap; + /** Change case in given path to match filesystem */ static std::string realFilename(const std::string &f) { std::string ret; ret.reserve(f.size()); // this will be the final size - if (realFileNameMap.getRealPathFromCache(f, &ret)) + if (realFilePathMap.getCacheEntry(f, &ret)) return ret; // Current subpath @@ -1952,9 +1994,12 @@ static std::string realFilename(const std::string &f) continue; } + bool isDriveSpecification = + (pos == 2 && subpath.size() == 2 && std::isalpha(subpath[0]) && subpath[1] == ':'); + // Append real filename (proper case) std::string f2; - if (realFileName(f.substr(0,pos),&f2)) + if (!isDriveSpecification && realFileName(f.substr(0, pos), &f2)) ret += f2; else ret += subpath; @@ -1976,7 +2021,7 @@ static std::string realFilename(const std::string &f) ret += subpath; } - realFileNameMap.addToCache(f, ret); + realFilePathMap.addToCache(f, ret); return ret; } diff --git a/simplecpp.h b/simplecpp.h index b4bce4c1..d7ce4f4a 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -312,6 +312,9 @@ namespace simplecpp { /** Simplify path */ SIMPLECPP_LIB std::string simplifyPath(std::string path); + + /** Convert Cygwin path to Windows path */ + SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); } #endif diff --git a/test.cpp b/test.cpp index 8dc2ca0e..ffa4c8ae 100644 --- a/test.cpp +++ b/test.cpp @@ -208,6 +208,24 @@ static void constFold() ASSERT_EQUALS("exception", testConstFold("?2:3")); } +static void convertCygwinPath() +{ + // absolute paths + ASSERT_EQUALS("X:\\", simplecpp::convertCygwinToWindowsPath("/cygdrive/x")); // initial backslash + ASSERT_EQUALS("X:\\", simplecpp::convertCygwinToWindowsPath("/cygdrive/x/")); + ASSERT_EQUALS("X:\\dir", simplecpp::convertCygwinToWindowsPath("/cygdrive/x/dir")); + ASSERT_EQUALS("X:\\dir\\file", simplecpp::convertCygwinToWindowsPath("/cygdrive/x/dir/file")); + + // relative paths + ASSERT_EQUALS("file", simplecpp::convertCygwinToWindowsPath("file")); + ASSERT_EQUALS("dir\\file", simplecpp::convertCygwinToWindowsPath("dir/file")); + ASSERT_EQUALS("..\\dir\\file", simplecpp::convertCygwinToWindowsPath("../dir/file")); + + // incorrect Cygwin paths + ASSERT_EQUALS("\\cygdrive", simplecpp::convertCygwinToWindowsPath("/cygdrive")); + ASSERT_EQUALS("\\cygdrive\\", simplecpp::convertCygwinToWindowsPath("/cygdrive/")); +} + static void define1() { const char code[] = "#define A 1+2\n" @@ -1632,6 +1650,8 @@ int main(int argc, char **argv) TEST_CASE(constFold); + TEST_CASE(convertCygwinPath); + TEST_CASE(define1); TEST_CASE(define2); TEST_CASE(define3); From c8f79da02910458dc88c7af9507258246a844690 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 10 Mar 2019 08:37:33 +0100 Subject: [PATCH 072/381] Use ungetChar and readChar to handle reading of utf16 files (#154) --- simplecpp.cpp | 14 +++++++------- simplecpp.h | 2 +- test.cpp | 30 ++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index cbe6bdf1..201363e8 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -480,7 +480,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen prev = ch; ch = readChar(istr, bom); } - istr.unget(); + ungetChar(istr, bom); push_back(new Token(currentToken, location)); location.adjust(currentToken); continue; @@ -512,7 +512,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen ++multiline; currentToken.erase(currentToken.size() - 1U); } else { - istr.unget(); + ungetChar(istr, bom); } } @@ -578,7 +578,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen continue; } - currentToken = readUntil(istr,location,ch,ch,outputList); + currentToken = readUntil(istr,location,ch,ch,outputList,bom); if (currentToken.size() < 2U) // TODO report return; @@ -607,7 +607,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen } if (currentToken == "<" && lastLine() == "# include") { - currentToken = readUntil(istr, location, '<', '>', outputList); + currentToken = readUntil(istr, location, '<', '>', outputList, bom); if (currentToken.size() < 2U) return; } @@ -1045,7 +1045,7 @@ void simplecpp::TokenList::removeComments() } } -std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList) +std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList, unsigned int bom) { std::string ret; ret += start; @@ -1053,7 +1053,7 @@ std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location & bool backslash = false; char ch = 0; while (ch != end && ch != '\r' && ch != '\n' && istr.good()) { - ch = (unsigned char)istr.get(); + ch = readChar(istr, bom); if (backslash && ch == '\n') { ch = 0; backslash = false; @@ -1062,7 +1062,7 @@ std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location & backslash = false; ret += ch; if (ch == '\\') { - const char next = (unsigned char)istr.get(); + const char next = readChar(istr, bom); if (next == '\r' || next == '\n') { ret.erase(ret.size()-1U); backslash = (next == '\r'); diff --git a/simplecpp.h b/simplecpp.h index d7ce4f4a..f3ce9e7f 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -258,7 +258,7 @@ namespace simplecpp { void constFoldLogicalOp(Token *tok); void constFoldQuestionOp(Token **tok1); - std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList); + std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList, unsigned int bom); std::string lastLine(int maxsize=100000) const; diff --git a/test.cpp b/test.cpp index ffa4c8ae..b2e3b915 100644 --- a/test.cpp +++ b/test.cpp @@ -534,6 +534,30 @@ static void error3() ASSERT_EQUALS("", toString(outputList)); } +static void error4() +{ + // "#error x\n1" + std::istringstream istr(std::string("\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31", 22)); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); +} + +static void error5() +{ + // "#error x\n1" + std::istringstream istr(std::string("\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00", 22)); + std::vector files; + std::map filedata; + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); +} + static void garbage() { const simplecpp::DUI dui; @@ -1515,6 +1539,10 @@ static void unicode() { ASSERT_EQUALS("12", readfile("\xFE\xFF\x00\x31\x00\x32", 6)); ASSERT_EQUALS("12", readfile("\xFF\xFE\x31\x00\x32\x00", 6)); + ASSERT_EQUALS("//\n1", readfile("\xFE\xFF\x00\x2f\x00\x2f\x00\x0a\x00\x31", 10)); + ASSERT_EQUALS("//\n1", readfile("\xFF\xFE\x2f\x00\x2f\x00\x0a\x00\x31\x00", 10)); + ASSERT_EQUALS("\"a\"", readfile("\xFE\xFF\x00\x22\x00\x61\x00\x22", 8)); + ASSERT_EQUALS("\"a\"", readfile("\xFF\xFE\x22\x00\x61\x00\x22\x00", 8)); ASSERT_EQUALS("\n//1", readfile("\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00",16)); } @@ -1691,6 +1719,8 @@ int main(int argc, char **argv) TEST_CASE(error1); TEST_CASE(error2); TEST_CASE(error3); + TEST_CASE(error4); + TEST_CASE(error5); TEST_CASE(garbage); TEST_CASE(garbage_endif); From 4e022f0a8d7c2a669260e1a0f93dc27d4bad702d Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 10 Mar 2019 08:40:18 +0100 Subject: [PATCH 073/381] Merge string and character literal with prefix (#155) Previously, simplecpp would output the character L'a' as two tokens "L" and "'a'". A user of the tokens has no way of knowing if they were originally "L'a'" och "L 'a'". --- simplecpp.cpp | 51 ++++++++++++++++++++++++++++++++------------------- test.cpp | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 201363e8..af6cda39 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -327,6 +327,15 @@ static void ungetChar(std::istream &istr, unsigned int bom) istr.unget(); } +static unsigned char prevChar(std::istream &istr, unsigned int bom) +{ + ungetChar(istr, bom); + ungetChar(istr, bom); + unsigned char c = readChar(istr, bom); + readChar(istr, bom); + return c; +} + static unsigned short getAndSkipBOM(std::istream &istr) { const int ch1 = istr.peek(); @@ -384,9 +393,10 @@ static void portabilityBackslash(simplecpp::OutputList *outputList, const std::v outputList->push_back(err); } -static bool isRawStringId(const std::string &str) +static bool isStringLiteralPrefix(const std::string &str) { - return str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; + return str == "u" || str == "U" || str == "L" || str == "u8" || + str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; } void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filename, OutputList *outputList) @@ -545,31 +555,34 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { - // C++11 raw string literal - if (ch == '\"' && cback() && cback()->name && isRawStringId(cback()->str())) { + if (cback() && cback()->name && !std::isspace(prevChar(istr, bom)) && (isStringLiteralPrefix(cback()->str()))) { std::string delim; - ch = readChar(istr,bom); - while (istr.good() && ch != '(' && ch != '\n') { - delim += ch; + currentToken = ch; + bool hasR = *cback()->str().rbegin() == 'R'; + std::string prefix = cback()->str(); + if (hasR) { + prefix.resize(prefix.size() - 1); + delim = ")"; ch = readChar(istr,bom); + while (istr.good() && ch != '(' && ch != '\n') { + delim += ch; + ch = readChar(istr,bom); + } + if (!istr.good() || ch == '\n') + // TODO report + return; } - if (!istr.good() || ch == '\n') - // TODO report - return; - currentToken = '\"'; - const std::string endOfRawString(')' + delim + '\"'); - while (istr.good() && !endsWith(currentToken, endOfRawString)) + const std::string endOfRawString(delim + currentToken); + while (istr.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) currentToken += readChar(istr,bom); if (!endsWith(currentToken, endOfRawString)) // TODO report return; currentToken.erase(currentToken.size() - endOfRawString.size(), endOfRawString.size() - 1U); - if (cback()->op == 'R') - back()->setstr(escapeString(currentToken)); - else { - back()->setstr(cback()->str().substr(0, cback()->str().size() - 1)); - push_back(new Token(currentToken, location)); // push string without newlines - } + if (hasR) + currentToken = escapeString(currentToken); + currentToken.insert(0, prefix); + back()->setstr(currentToken); location.adjust(currentToken); if (currentToken.find_first_of("\r\n") == std::string::npos) location.col += 2 + 2 * delim.size(); diff --git a/test.cpp b/test.cpp index b2e3b915..1a7923f1 100644 --- a/test.cpp +++ b/test.cpp @@ -1354,17 +1354,36 @@ static void readfile_nullbyte() ASSERT_EQUALS(true, outputList.empty()); // should warning be written? } +static void readfile_char() +{ + ASSERT_EQUALS("'c'", readfile("'c'")); + ASSERT_EQUALS("L'c'", readfile("L'c'")); + ASSERT_EQUALS("L 'c'", readfile("L 'c'")); + ASSERT_EQUALS("A = 'c'", readfile("A = 'c'")); + ASSERT_EQUALS("A = L'c'", readfile("A = L'c'")); + ASSERT_EQUALS("A = u'c'", readfile("A = u'c'")); + ASSERT_EQUALS("A = U'c'", readfile("A = U'c'")); + ASSERT_EQUALS("A = u8'c'", readfile("A = u8'c'")); + ASSERT_EQUALS("A = u8 'c'", readfile("A = u8 'c'")); +} + static void readfile_string() { - const char code[] = "A = \"abc\'def\""; - ASSERT_EQUALS("A = \"abc\'def\"", readfile(code)); + ASSERT_EQUALS("\"\"", readfile("\"\"")); + ASSERT_EQUALS("\"a\"", readfile("\"a\"")); + ASSERT_EQUALS("u\"a\"", readfile("u\"a\"")); + ASSERT_EQUALS("u \"a\"", readfile("u \"a\"")); + ASSERT_EQUALS("A = \"\"", readfile("A = \"\"")); + ASSERT_EQUALS("A = \"abc\'def\"", readfile("A = \"abc\'def\"")); ASSERT_EQUALS("( \"\\\\\\\\\" )", readfile("(\"\\\\\\\\\")")); ASSERT_EQUALS("x = \"a b\"\n;", readfile("x=\"a\\\n b\";")); ASSERT_EQUALS("x = \"a b\"\n;", readfile("x=\"a\\\r\n b\";")); -} -static void readfile_rawstring() -{ + ASSERT_EQUALS("A = u8\"a\"", readfile("A = u8\"a\"")); + ASSERT_EQUALS("A = u\"a\"", readfile("A = u\"a\"")); + ASSERT_EQUALS("A = U\"a\"", readfile("A = U\"a\"")); + ASSERT_EQUALS("A = L\"a\"", readfile("A = L\"a\"")); + ASSERT_EQUALS("A = L \"a\"", readfile("A = L \"a\"")); ASSERT_EQUALS("A = \"abc\\\\\\\\def\"", readfile("A = R\"(abc\\\\def)\"")); ASSERT_EQUALS("A = \"abc\\\\\\\\def\"", readfile("A = R\"x(abc\\\\def)x\"")); ASSERT_EQUALS("A = \"\"", readfile("A = R\"()\"")); @@ -1372,10 +1391,10 @@ static void readfile_rawstring() ASSERT_EQUALS("A = \"\\\"\"", readfile("A = R\"(\")\"")); ASSERT_EQUALS("A = \"abc\"", readfile("A = R\"\"\"(abc)\"\"\"")); ASSERT_EQUALS("A = \"a\nb\nc\";", readfile("A = R\"foo(a\nb\nc)foo\";")); - ASSERT_EQUALS("A = L \"abc\"", readfile("A = LR\"(abc)\"")); - ASSERT_EQUALS("A = u \"abc\"", readfile("A = uR\"(abc)\"")); - ASSERT_EQUALS("A = U \"abc\"", readfile("A = UR\"(abc)\"")); - ASSERT_EQUALS("A = u8 \"abc\"", readfile("A = u8R\"(abc)\"")); + ASSERT_EQUALS("A = L\"abc\"", readfile("A = LR\"(abc)\"")); + ASSERT_EQUALS("A = u\"abc\"", readfile("A = uR\"(abc)\"")); + ASSERT_EQUALS("A = U\"abc\"", readfile("A = UR\"(abc)\"")); + ASSERT_EQUALS("A = u8\"abc\"", readfile("A = u8R\"(abc)\"")); } static void readfile_cpp14_number() @@ -1786,8 +1805,8 @@ int main(int argc, char **argv) TEST_CASE(multiline7); // multiline string in macro TEST_CASE(readfile_nullbyte); + TEST_CASE(readfile_char); TEST_CASE(readfile_string); - TEST_CASE(readfile_rawstring); TEST_CASE(readfile_cpp14_number); TEST_CASE(readfile_unhandled_chars); TEST_CASE(readfile_error); From 1eb30dcea852431f3e355383534ea527e48c0b5d Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Mon, 11 Mar 2019 06:13:31 +0100 Subject: [PATCH 074/381] Fix multiline prefixed string literals in macros (#156) Commit 4e022f0a8d7c2a669260e1a0f93dc27d4bad702d broke parsing of multiline prefixed strings in defines. The commit changed so all prefixed strings went through the same code path that previously only raw string literals did. That didn't handle newlines in defines well. Change this to strip out the prefix first, and then check if it is a raw string or not. --- simplecpp.cpp | 35 +++++++++++++++++++---------------- test.cpp | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index af6cda39..d16df0c0 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -555,32 +555,31 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { + std::string prefix; if (cback() && cback()->name && !std::isspace(prevChar(istr, bom)) && (isStringLiteralPrefix(cback()->str()))) { + prefix = cback()->str(); + } + // C++11 raw string literal + if (ch == '\"' && !prefix.empty() && *cback()->str().rbegin() == 'R') { std::string delim; currentToken = ch; - bool hasR = *cback()->str().rbegin() == 'R'; - std::string prefix = cback()->str(); - if (hasR) { - prefix.resize(prefix.size() - 1); - delim = ")"; + prefix.resize(prefix.size() - 1); + ch = readChar(istr,bom); + while (istr.good() && ch != '(' && ch != '\n') { + delim += ch; ch = readChar(istr,bom); - while (istr.good() && ch != '(' && ch != '\n') { - delim += ch; - ch = readChar(istr,bom); - } - if (!istr.good() || ch == '\n') - // TODO report - return; } - const std::string endOfRawString(delim + currentToken); + if (!istr.good() || ch == '\n') + // TODO report + return; + const std::string endOfRawString(')' + delim + currentToken); while (istr.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) currentToken += readChar(istr,bom); if (!endsWith(currentToken, endOfRawString)) // TODO report return; currentToken.erase(currentToken.size() - endOfRawString.size(), endOfRawString.size() - 1U); - if (hasR) - currentToken = escapeString(currentToken); + currentToken = escapeString(currentToken); currentToken.insert(0, prefix); back()->setstr(currentToken); location.adjust(currentToken); @@ -588,6 +587,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen location.col += 2 + 2 * delim.size(); else location.col += 1 + delim.size(); + continue; } @@ -604,7 +604,10 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen newlines++; } - push_back(new Token(s, location)); // push string without newlines + if (prefix.empty()) + push_back(new Token(s, location)); // push string without newlines + else + back()->setstr(prefix + s); if (newlines > 0 && lastLine().compare(0,9,"# define ") == 0) { multiline += newlines; diff --git a/test.cpp b/test.cpp index 1a7923f1..a0a317dd 100644 --- a/test.cpp +++ b/test.cpp @@ -1162,6 +1162,22 @@ static void multiline7() // multiline string in macro "A ( 1 )", rawtokens.stringify()); } +static void multiline8() // multiline prefix string in macro +{ + const char code[] = "#define A L\"a\\\n" + " b\"\n" + "A;"; + ASSERT_EQUALS("\n\nL\"a b\" ;", preprocess(code)); +} + +static void multiline9() // multiline prefix string in macro +{ + const char code[] = "#define A u8\"a\\\n" + " b\"\n" + "A;"; + ASSERT_EQUALS("\n\nu8\"a b\" ;", preprocess(code)); +} + static void nullDirective1() { const char code[] = "#\n" @@ -1803,6 +1819,8 @@ int main(int argc, char **argv) TEST_CASE(multiline5); // column TEST_CASE(multiline6); // multiline string in macro TEST_CASE(multiline7); // multiline string in macro + TEST_CASE(multiline8); // multiline prefix string in macro + TEST_CASE(multiline9); // multiline prefix string in macro TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_char); From 2b22008c7ac67a39f82d8b86121695f531057fdf Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Mon, 11 Mar 2019 18:16:54 +0100 Subject: [PATCH 075/381] Add error message to ill-formed string and character literals (#157) Add error messages to ill formed raw string literals. There was a TODO for adding an error message in case reading a string failed, (i.e., it was not terminated properly. There was already an error message from readUntil() so remove the TODO and add a few test cases. --- simplecpp.cpp | 24 +++++++++++++++++++----- test.cpp | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d16df0c0..617710b5 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -569,15 +569,29 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen delim += ch; ch = readChar(istr,bom); } - if (!istr.good() || ch == '\n') - // TODO report + if (!istr.good() || ch == '\n') { + if (outputList) { + Output err(files); + err.type = Output::SYNTAX_ERROR; + err.location = location; + err.msg = "Invalid newline in raw string delimiter."; + outputList->push_back(err); + } return; + } const std::string endOfRawString(')' + delim + currentToken); while (istr.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) currentToken += readChar(istr,bom); - if (!endsWith(currentToken, endOfRawString)) - // TODO report + if (!endsWith(currentToken, endOfRawString)) { + if (outputList) { + Output err(files); + err.type = Output::SYNTAX_ERROR; + err.location = location; + err.msg = "Raw string missing terminating delimiter."; + outputList->push_back(err); + } return; + } currentToken.erase(currentToken.size() - endOfRawString.size(), endOfRawString.size() - 1U); currentToken = escapeString(currentToken); currentToken.insert(0, prefix); @@ -593,7 +607,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen currentToken = readUntil(istr,location,ch,ch,outputList,bom); if (currentToken.size() < 2U) - // TODO report + // Error is reported by readUntil() return; std::string s = currentToken; diff --git a/test.cpp b/test.cpp index a0a317dd..7ecb1618 100644 --- a/test.cpp +++ b/test.cpp @@ -1383,6 +1383,18 @@ static void readfile_char() ASSERT_EQUALS("A = u8 'c'", readfile("A = u8 'c'")); } +static void readfile_char_error() +{ + simplecpp::OutputList outputList; + + readfile("A = L's", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + readfile("A = 's\n'", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); +} + static void readfile_string() { ASSERT_EQUALS("\"\"", readfile("\"\"")); @@ -1413,6 +1425,43 @@ static void readfile_string() ASSERT_EQUALS("A = u8\"abc\"", readfile("A = u8R\"(abc)\"")); } +static void readfile_string_error() +{ + simplecpp::OutputList outputList; + + readfile("A = \"abs", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + readfile("A = u8\"abs\n\"", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + readfile("A = R\"as\n(abc)as\"", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,Invalid newline in raw string delimiter.\n", toString(outputList)); + outputList.clear(); + + readfile("A = u8R\"as\n(abc)as\"", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,Invalid newline in raw string delimiter.\n", toString(outputList)); + outputList.clear(); + + readfile("A = R\"as(abc)a\"", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,Raw string missing terminating delimiter.\n", toString(outputList)); + outputList.clear(); + + readfile("A = LR\"as(abc)a\"", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,Raw string missing terminating delimiter.\n", toString(outputList)); + outputList.clear(); + + readfile("#define A \"abs", -1, &outputList); + ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); + outputList.clear(); + + // Don't warn for a multiline define + readfile("#define A \"abs\\\n\"", -1, &outputList); + ASSERT_EQUALS("", toString(outputList)); +} + static void readfile_cpp14_number() { ASSERT_EQUALS("A = 12345 ;", readfile("A = 12\'345;")); @@ -1824,7 +1873,9 @@ int main(int argc, char **argv) TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_char); + TEST_CASE(readfile_char_error); TEST_CASE(readfile_string); + TEST_CASE(readfile_string_error); TEST_CASE(readfile_cpp14_number); TEST_CASE(readfile_unhandled_chars); TEST_CASE(readfile_error); From c42c2d180e06c2400d896f25308148b41366ac66 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Tue, 12 Mar 2019 06:10:14 +0100 Subject: [PATCH 076/381] Add regression tests for string literals with " inside (#158) Commit 4e022f0a8d7c2a669260e1a0f93dc27d4bad702d broke (among other things), parsing of prefixed strings with \" and prefixed characters containing \'. This (and other things) was fixed in 1eb30dcea852431f3e355383534ea527e48c0b5d. Add tests for this. --- test.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test.cpp b/test.cpp index 7ecb1618..3a1aa03b 100644 --- a/test.cpp +++ b/test.cpp @@ -1381,6 +1381,13 @@ static void readfile_char() ASSERT_EQUALS("A = U'c'", readfile("A = U'c'")); ASSERT_EQUALS("A = u8'c'", readfile("A = u8'c'")); ASSERT_EQUALS("A = u8 'c'", readfile("A = u8 'c'")); + + // The character \' + ASSERT_EQUALS("'\\''", readfile("'\\\''")); + ASSERT_EQUALS("L'\\''", readfile("L'\\\''")); + ASSERT_EQUALS("u'\\''", readfile("u'\\\''")); + ASSERT_EQUALS("U'\\''", readfile("U'\\\''")); + ASSERT_EQUALS("u8'\\''", readfile("u8'\\\''")); } static void readfile_char_error() @@ -1423,6 +1430,18 @@ static void readfile_string() ASSERT_EQUALS("A = u\"abc\"", readfile("A = uR\"(abc)\"")); ASSERT_EQUALS("A = U\"abc\"", readfile("A = UR\"(abc)\"")); ASSERT_EQUALS("A = u8\"abc\"", readfile("A = u8R\"(abc)\"")); + + // Strings containing \" + ASSERT_EQUALS("\"\\\"\"", readfile("\"\\\"\"")); + ASSERT_EQUALS("L\"\\\"\"", readfile("L\"\\\"\"")); + ASSERT_EQUALS("u\"\\\"\"", readfile("u\"\\\"\"")); + ASSERT_EQUALS("U\"\\\"\"", readfile("U\"\\\"\"")); + ASSERT_EQUALS("u8\"\\\"\"", readfile("u8\"\\\"\"")); + ASSERT_EQUALS("\"\\\"a\\\"\"", readfile("\"\\\"a\\\"\"")); + ASSERT_EQUALS("L\"a\\\"\\\"\"", readfile("L\"a\\\"\\\"\"")); + ASSERT_EQUALS("u\"a\\\"\\\"\"", readfile("u\"a\\\"\\\"\"")); + ASSERT_EQUALS("U\"a\\\"\\\"\"", readfile("U\"a\\\"\\\"\"")); + ASSERT_EQUALS("u8\"a\\\"\\\"\"", readfile("u8\"a\\\"\\\"\"")); } static void readfile_string_error() From 6dc9fb9258d2f9200f7cc4b39bf0f08fbad71690 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 31 Mar 2019 07:43:24 +0200 Subject: [PATCH 077/381] Fix check if previous read character was space (#160) The prevChar was broken in the sense that there is no guarantee that unget can be called more than once on a stream. This lead to problems in some cases on some operative systems. Fix this be checking the location of the previous token instead. --- simplecpp.cpp | 12 ++---------- test.cpp | 8 ++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 617710b5..2da0c443 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -327,15 +327,6 @@ static void ungetChar(std::istream &istr, unsigned int bom) istr.unget(); } -static unsigned char prevChar(std::istream &istr, unsigned int bom) -{ - ungetChar(istr, bom); - ungetChar(istr, bom); - unsigned char c = readChar(istr, bom); - readChar(istr, bom); - return c; -} - static unsigned short getAndSkipBOM(std::istream &istr) { const int ch1 = istr.peek(); @@ -556,7 +547,8 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // string / char literal else if (ch == '\"' || ch == '\'') { std::string prefix; - if (cback() && cback()->name && !std::isspace(prevChar(istr, bom)) && (isStringLiteralPrefix(cback()->str()))) { + if (cback() && cback()->name && isStringLiteralPrefix(cback()->str()) && + ((cback()->location.col + cback()->str().size()) == location.col)) { prefix = cback()->str(); } // C++11 raw string literal diff --git a/test.cpp b/test.cpp index 3a1aa03b..25771f89 100644 --- a/test.cpp +++ b/test.cpp @@ -309,6 +309,13 @@ static void define9() ASSERT_EQUALS("\nab . AB . CD", preprocess(code)); } +static void define10() // don't combine prefix with space in macro +{ + const char code[] = "#define A u8 \"a b\"\n" + "A;"; + ASSERT_EQUALS("\nu8 \"a b\" ;", preprocess(code)); +} + static void define_invalid_1() { std::istringstream istr("#define A(\nB\n"); @@ -1792,6 +1799,7 @@ int main(int argc, char **argv) TEST_CASE(define7); TEST_CASE(define8); TEST_CASE(define9); + TEST_CASE(define10); TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_define_1); From ec4c5e2f28c967080688f96967714b5c9051eab1 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 31 Mar 2019 11:52:51 +0200 Subject: [PATCH 078/381] Don't join prefix and string on separate lines (#161) --- simplecpp.cpp | 3 ++- test.cpp | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2da0c443..7e35b97c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -548,7 +548,8 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen else if (ch == '\"' || ch == '\'') { std::string prefix; if (cback() && cback()->name && isStringLiteralPrefix(cback()->str()) && - ((cback()->location.col + cback()->str().size()) == location.col)) { + ((cback()->location.col + cback()->str().size()) == location.col) && + (cback()->location.line == location.line)) { prefix = cback()->str(); } // C++11 raw string literal diff --git a/test.cpp b/test.cpp index 25771f89..09eed4ef 100644 --- a/test.cpp +++ b/test.cpp @@ -1449,6 +1449,10 @@ static void readfile_string() ASSERT_EQUALS("u\"a\\\"\\\"\"", readfile("u\"a\\\"\\\"\"")); ASSERT_EQUALS("U\"a\\\"\\\"\"", readfile("U\"a\\\"\\\"\"")); ASSERT_EQUALS("u8\"a\\\"\\\"\"", readfile("u8\"a\\\"\\\"\"")); + + // Do not concatenate when prefix is not directly adjacent to " + ASSERT_EQUALS("u8 \"a b\"", readfile("u8 \"a b\"")); + ASSERT_EQUALS("u8\n\"a b\"", readfile("u8\n \"a b\"")); } static void readfile_string_error() From 0a0cce4159c369f9251150a4fc65ac5f3b293d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 3 Apr 2019 19:41:02 +0200 Subject: [PATCH 079/381] a few optimizations (#162) * optimized simplecpp::TokenList::lastLine() reduces Ir from 2722 to 1296 according to callgrind * optimized simplecpp::Token::flags() reduces Ir from 198 to 43 according to callgrind --- simplecpp.cpp | 6 +++--- simplecpp.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7e35b97c..dfe8559a 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1117,9 +1117,9 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (tok->comment) continue; if (!ret.empty()) - ret = ' ' + ret; - ret = (tok->str()[0] == '\"' ? std::string("%str%") - : tok->number ? std::string("%num%") : tok->str()) + ret; + ret.insert(0, 1, ' '); + ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") + : tok->number ? std::string("%num%") : tok->str()); if (++count > maxsize) return ""; } diff --git a/simplecpp.h b/simplecpp.h index f3ce9e7f..5463fa14 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -108,7 +108,7 @@ namespace simplecpp { void flags() { name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$'); - comment = (string.compare(0, 2, "//") == 0 || string.compare(0, 2, "/*") == 0); + comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); op = (string.size() == 1U) ? string[0] : '\0'; } From b37671ffad5c764e31282d1e145dfb78e724c8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 27 Apr 2019 19:40:55 +0200 Subject: [PATCH 080/381] Better but not perfect handling of #line, location will not be decreased (#115) --- simplecpp.cpp | 13 ++++++++++--- test.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index dfe8559a..86139644 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -446,16 +446,23 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = 1U; } else if (lastline == "# line %num%") { - loc.push(location); + const Location loc1 = location; location.line = std::atol(cback()->str().c_str()); + if (location.line < loc1.line) + location.line = loc1.line; } else if (lastline == "# line %num% %str%") { - loc.push(location); + const Location loc1 = location; location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = std::atol(cback()->previous->str().c_str()); + if (loc1.fileIndex == location.fileIndex && location.line < loc1.line) + location.line = loc1.line; } else if (lastline == "# %num% %str%") { + const Location loc1 = location; loc.push(location); location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = std::atol(cback()->previous->str().c_str()); + if (loc1.fileIndex == location.fileIndex && location.line < loc1.line) + location.line = loc1.line; } // #endfile else if (lastline == "# endfile" && !loc.empty()) { @@ -2017,7 +2024,7 @@ static std::string realFilename(const std::string &f) continue; } - bool isDriveSpecification = + bool isDriveSpecification = (pos == 2 && subpath.size() == 2 && std::isalpha(subpath[0]) && subpath[1] == ':'); // Append real filename (proper case) diff --git a/test.cpp b/test.cpp index 09eed4ef..06c66dcf 100644 --- a/test.cpp +++ b/test.cpp @@ -1010,6 +1010,29 @@ static void location1() ASSERT_EQUALS("\n#line 3 \"main.c\"\nx", preprocess(code)); } +static void location2() +{ + const char *code; + + code = "{ {\n" + "#line 40 \"abc.y\"\n" + "{\n" + "}\n" + "#line 42 \"abc.y\"\n" + "{\n" + "}\n" + "} }"; + + ASSERT_EQUALS("{ {\n" + "#line 40 \"abc.y\"\n" + "{\n" + "}\n" + "\n" + "{\n" + "}\n" + "} }", preprocess(code)); +} + static void missingHeader1() { const simplecpp::DUI dui; @@ -1873,6 +1896,7 @@ int main(int argc, char **argv) TEST_CASE(ifalt); // using "and", "or", etc TEST_CASE(location1); + TEST_CASE(location2); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); From d91a9dc464d0fcc47ce256c662ff328046e8946a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 1 May 2019 20:35:06 +0200 Subject: [PATCH 081/381] Fixed handling when including system headers --- simplecpp.cpp | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 86139644..32182fda 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1126,7 +1126,7 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (!ret.empty()) ret.insert(0, 1, ' '); ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") - : tok->number ? std::string("%num%") : tok->str()); + : tok->number ? std::string("%num%") : tok->str()); if (++count > maxsize) return ""; } @@ -2290,23 +2290,21 @@ static std::string _openHeader(std::ifstream &f, const std::string &path) #endif } -static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) +static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header) { if (isAbsolutePath(header)) { return _openHeader(f, header); } - if (!systemheader) { - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - std::string simplePath = _openHeader(f, s); - if (!simplePath.empty()) - return simplePath; - } else { - std::string simplePath = _openHeader(f, header); - if (!simplePath.empty()) - return simplePath; - } + if (sourcefile.find_first_of("\\/") != std::string::npos) { + const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + const std::string simplePath = _openHeader(f, s); + if (!simplePath.empty()) + return simplePath; + } else { + const std::string simplePath = _openHeader(f, header); + if (!simplePath.empty()) + return simplePath; } for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { @@ -2415,7 +2413,7 @@ std::map simplecpp::load(const simplecpp::To continue; std::ifstream f; - const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); + const std::string header2 = openHeader(f,dui,sourcefile,header); if (!f.is_open()) continue; @@ -2636,7 +2634,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (header2.empty()) { // try to load file.. std::ifstream f; - header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); + header2 = openHeader(f, dui, rawtok->location.file(), header); if (f.is_open()) { TokenList *tokens = new TokenList(f, files, header2, outputList); filedata[header2] = tokens; From 03f9ac8cfa6c4c25e6844c4030ea551d8d06f996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 2 May 2019 06:56:32 +0200 Subject: [PATCH 082/381] added move constructor and assignment for TokenList - gets rid of copy of result in Preprocessor::preprocess() (#164) reduces Ir from 14.98% to 12.70% with package pion according to callgrind --- simplecpp.cpp | 22 ++++++++++++++++++++++ simplecpp.h | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 32182fda..f3253305 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -200,6 +200,13 @@ simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(NULL), back *this = other; } +#if __cplusplus >= 201103L +simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(NULL), backToken(NULL), files(other.files) +{ + *this = std::move(other); +} +#endif + simplecpp::TokenList::~TokenList() { clear(); @@ -216,6 +223,21 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(const TokenList &other) return *this; } +#if __cplusplus >= 201103L +simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) +{ + if (this != &other) { + clear(); + backToken = other.backToken; + other.backToken = NULL; + frontToken = other.frontToken; + other.frontToken = NULL; + sizeOfType = std::move(other.sizeOfType); + } + return *this; +} +#endif + void simplecpp::TokenList::clear() { backToken = NULL; diff --git a/simplecpp.h b/simplecpp.h index 5463fa14..096fa324 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -181,8 +181,14 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = 0); TokenList(const TokenList &other); +#if __cplusplus >= 201103L + TokenList(TokenList &&other); +#endif ~TokenList(); TokenList &operator=(const TokenList &other); +#if __cplusplus >= 201103L + TokenList &operator=(TokenList &&other); +#endif void clear(); bool empty() const { From 9cd5547a1f623c83f2b4d71831d818b8661098bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 3 May 2019 18:47:47 +0200 Subject: [PATCH 083/381] Better handling of line directive --- simplecpp.cpp | 38 +++++++++++++++++++++----------------- simplecpp.h | 1 + test.cpp | 12 +++++++++++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f3253305..3ce667f6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -412,6 +412,23 @@ static bool isStringLiteralPrefix(const std::string &str) str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; } +void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int line, Location *location) +{ + if (fileIndex != location->fileIndex || line >= location->line) { + location->fileIndex = fileIndex; + location->line = line; + return; + } + + if (line + 2 >= location->line) { + location->line = line; + while (cback()->op != '#') + deleteToken(back()); + deleteToken(back()); + return; + } +} + void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filename, OutputList *outputList) { std::stack loc; @@ -468,23 +485,10 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); location.line = 1U; } else if (lastline == "# line %num%") { - const Location loc1 = location; - location.line = std::atol(cback()->str().c_str()); - if (location.line < loc1.line) - location.line = loc1.line; - } else if (lastline == "# line %num% %str%") { - const Location loc1 = location; - location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); - location.line = std::atol(cback()->previous->str().c_str()); - if (loc1.fileIndex == location.fileIndex && location.line < loc1.line) - location.line = loc1.line; - } else if (lastline == "# %num% %str%") { - const Location loc1 = location; - loc.push(location); - location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); - location.line = std::atol(cback()->previous->str().c_str()); - if (loc1.fileIndex == location.fileIndex && location.line < loc1.line) - location.line = loc1.line; + lineDirective(location.fileIndex, std::atol(cback()->str().c_str()), &location); + } else if (lastline == "# %num% %str%" || lastline == "# line %num% %str%") { + lineDirective(fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)), + std::atol(cback()->previous->str().c_str()), &location); } // #endfile else if (lastline == "# endfile" && !loc.empty()) { diff --git a/simplecpp.h b/simplecpp.h index 096fa324..16754a5c 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -265,6 +265,7 @@ namespace simplecpp { void constFoldQuestionOp(Token **tok1); std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList, unsigned int bom); + void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); std::string lastLine(int maxsize=100000) const; diff --git a/test.cpp b/test.cpp index 06c66dcf..46e0b44e 100644 --- a/test.cpp +++ b/test.cpp @@ -1027,12 +1027,21 @@ static void location2() "#line 40 \"abc.y\"\n" "{\n" "}\n" - "\n" "{\n" "}\n" "} }", preprocess(code)); } +static void location3() +{ + const char *code; + code = "#line 1 \"x\" \n" + "a\n" + "#line 1 \"x\" \n" + "b\n"; + ASSERT_EQUALS("\n#line 1 \"x\"\na b", preprocess(code)); +} + static void missingHeader1() { const simplecpp::DUI dui; @@ -1897,6 +1906,7 @@ int main(int argc, char **argv) TEST_CASE(location1); TEST_CASE(location2); + TEST_CASE(location3); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); From ad8599f78ab33b3ceb865846ae1fbef92885d5a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 5 May 2019 09:52:58 +0200 Subject: [PATCH 084/381] fixed some clang-tidy and CLion inspection warnings (#165) --- main.cpp | 2 +- simplecpp.cpp | 8 ++++---- simplecpp.h | 10 +++++----- test.cpp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/main.cpp b/main.cpp index 2f513083..31ca9b6f 100644 --- a/main.cpp +++ b/main.cpp @@ -32,7 +32,7 @@ int main(int argc, char **argv) if (std::strncmp(arg, "-include=",9)==0) dui.includes.push_back(arg+9); break; - }; + } } else { filename = arg; } diff --git a/simplecpp.cpp b/simplecpp.cpp index 3ce667f6..7dfd3d96 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -149,17 +149,17 @@ void simplecpp::Location::adjust(const std::string &str) bool simplecpp::Token::isOneOf(const char ops[]) const { - return (op != '\0') && (std::strchr(ops, op) != 0); + return (op != '\0') && (std::strchr(ops, op) != NULL); } bool simplecpp::Token::startsWithOneOf(const char c[]) const { - return std::strchr(c, string[0]) != 0; + return std::strchr(c, string[0]) != NULL; } bool simplecpp::Token::endsWithOneOf(const char c[]) const { - return std::strchr(c, string[string.size() - 1U]) != 0; + return std::strchr(c, string[string.size() - 1U]) != NULL; } void simplecpp::Token::printAll() const @@ -2685,7 +2685,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (pragmaOnce.find(header2) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); const TokenList *includetokens = filedata.find(header2)->second; - rawtok = includetokens ? includetokens->cfront() : 0; + rawtok = includetokens ? includetokens->cfront() : NULL; continue; } } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { diff --git a/simplecpp.h b/simplecpp.h index 16754a5c..7f44d308 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -179,7 +179,7 @@ namespace simplecpp { class SIMPLECPP_LIB TokenList { public: explicit TokenList(std::vector &filenames); - TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = 0); + TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = NULL); TokenList(const TokenList &other); #if __cplusplus >= 201103L TokenList(TokenList &&other); @@ -199,7 +199,7 @@ namespace simplecpp { void dump() const; std::string stringify() const; - void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = 0); + void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = NULL); void constFold(); void removeComments(); @@ -264,7 +264,7 @@ namespace simplecpp { void constFoldLogicalOp(Token *tok); void constFoldQuestionOp(Token **tok1); - std::string readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList, unsigned int bom); + std::string readUntil(std::istream &istr, const Location &location, char start, char end, OutputList *outputList, unsigned int bom); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); std::string lastLine(int maxsize=100000) const; @@ -297,7 +297,7 @@ namespace simplecpp { std::list includes; }; - SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = 0); + SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = NULL); /** * Preprocess @@ -310,7 +310,7 @@ namespace simplecpp { * @param outputList output: list that will receive output messages * @param macroUsage output: macro usage */ - SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = 0, std::list *macroUsage = 0); + SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = NULL, std::list *macroUsage = NULL); /** * Deallocate data diff --git a/test.cpp b/test.cpp index 46e0b44e..591f4393 100644 --- a/test.cpp +++ b/test.cpp @@ -1060,7 +1060,7 @@ static void missingHeader2() std::istringstream istr("#include \"foo.h\"\n"); // this file exists std::vector files; std::map filedata; - filedata["foo.h"] = 0; + filedata["foo.h"] = NULL; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); From 5dedcb74f0e50d1b9d76e0564442e892040f2b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 May 2019 10:43:32 +0200 Subject: [PATCH 085/381] updated CMake to include warnings and syntax check (#166) * updated CMake to include warnings and syntax check also removed -Wabi since it will no longer warn by itself will report the following for each file: cc1plus: warning: -Wabi won't warn about anything [-Wabi] cc1plus: note: -Wabi warns about differences from the most up-to-date ABI, which is also used by default cc1plus: note: use e.g. -Wabi=11 to warn about changes from GCC 7 add -Wundef since it is a useful warning * fixed compiler warning * Makefile: restored accidentally removed -std=c++0x * CMakeLists.txt: some cleanups / only add syntax check target for compilers which actually support to set the standard to the desired version --- CMakeLists.txt | 29 ++++++++++++++++++++++++++++- Makefile | 2 +- test.cpp | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0e0f50f..63c81581 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,35 @@ cmake_minimum_required (VERSION 3.5) -project (simplecpp) +project (simplecpp LANGUAGES CXX) + +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef) +endif() + +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Weverything) + # no need for c++98 compatibility + add_compile_options(-Wno-c++98-compat-pedantic) + # these are not really fixable + add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) + # TODO: fix these? + add_compile_options(-Wno-zero-as-null-pointer-constant -Wno-padded -Wno-sign-conversion -Wno-conversion -Wno-old-style-cast) +endif() add_executable(simplecpp simplecpp.cpp main.cpp) set_property(TARGET simplecpp PROPERTY CXX_STANDARD 11) +# it is not possible to set a standard older than C++14 with Visual Studio +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # we need to create a dummy library as -fsyntax-only will not produce any output files causing the build to fail + add_library(simplecpp-03-syntax STATIC simplecpp.cpp) + target_compile_options(simplecpp-03-syntax PRIVATE -std=c++03) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + target_compile_options(simplecpp-03-syntax PRIVATE -Wno-long-long) + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long) + endif() + add_dependencies(simplecpp simplecpp-03-syntax) +endif() + add_executable(testrunner simplecpp.cpp test.cpp) set_property(TARGET testrunner PROPERTY CXX_STANDARD 11) diff --git a/Makefile b/Makefile index 93da951d..b6865ef1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -Wabi -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -pedantic -g -std=c++0x +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -std=c++0x -g LDFLAGS = -g %.o: %.cpp simplecpp.h diff --git a/test.cpp b/test.cpp index 591f4393..c4a403f0 100644 --- a/test.cpp +++ b/test.cpp @@ -4,7 +4,7 @@ #include #include "simplecpp.h" -int numberOfFailedAssertions = 0; +static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) From b5e385a48db2ec23bcc25018c891c4c8aee95f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 7 May 2019 19:12:05 +0200 Subject: [PATCH 086/381] Revert "Fixed handling when including system headers" This reverts commit d91a9dc464d0fcc47ce256c662ff328046e8946a. This fix caused endless loops and OOM. --- simplecpp.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7dfd3d96..ed30a6bc 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1152,7 +1152,7 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (!ret.empty()) ret.insert(0, 1, ' '); ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") - : tok->number ? std::string("%num%") : tok->str()); + : tok->number ? std::string("%num%") : tok->str()); if (++count > maxsize) return ""; } @@ -2316,21 +2316,23 @@ static std::string _openHeader(std::ifstream &f, const std::string &path) #endif } -static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header) +static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { if (isAbsolutePath(header)) { return _openHeader(f, header); } - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - const std::string simplePath = _openHeader(f, s); - if (!simplePath.empty()) - return simplePath; - } else { - const std::string simplePath = _openHeader(f, header); - if (!simplePath.empty()) - return simplePath; + if (!systemheader) { + if (sourcefile.find_first_of("\\/") != std::string::npos) { + const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + std::string simplePath = _openHeader(f, s); + if (!simplePath.empty()) + return simplePath; + } else { + std::string simplePath = _openHeader(f, header); + if (!simplePath.empty()) + return simplePath; + } } for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { @@ -2439,7 +2441,7 @@ std::map simplecpp::load(const simplecpp::To continue; std::ifstream f; - const std::string header2 = openHeader(f,dui,sourcefile,header); + const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); if (!f.is_open()) continue; @@ -2660,7 +2662,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (header2.empty()) { // try to load file.. std::ifstream f; - header2 = openHeader(f, dui, rawtok->location.file(), header); + header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); if (f.is_open()) { TokenList *tokens = new TokenList(f, files, header2, outputList); filedata[header2] = tokens; From 329ba8584a850c681a0d06fef5f93c9dc009c789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 7 May 2019 19:13:02 +0200 Subject: [PATCH 087/381] astyle formatting --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ed30a6bc..51dfbc69 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1152,7 +1152,7 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (!ret.empty()) ret.insert(0, 1, ' '); ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") - : tok->number ? std::string("%num%") : tok->str()); + : tok->number ? std::string("%num%") : tok->str()); if (++count > maxsize) return ""; } From 91e85f8b46c0ca0ae58cef6735da3247502db5e4 Mon Sep 17 00:00:00 2001 From: Katayama Hirofumi MZ Date: Mon, 1 Jul 2019 04:41:43 +0900 Subject: [PATCH 088/381] Use std::make_pair 2 (#171) --- simplecpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 51dfbc69..0c38f6fe 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2519,9 +2519,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::pair(macro.name(), macro)); } - macros.insert(std::pair("__FILE__", Macro("__FILE__", "__FILE__", files))); - macros.insert(std::pair("__LINE__", Macro("__LINE__", "__LINE__", files))); - macros.insert(std::pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files))); + macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files))); + macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", files))); + macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files))); // TRUE => code in current #if block should be kept // ELSE_IS_TRUE => code in current #if block should be dropped. the code in the #else should be kept. From 44434ee27a2581fae09ab659c423de72a9be6154 Mon Sep 17 00:00:00 2001 From: Katayama Hirofumi MZ Date: Mon, 1 Jul 2019 04:42:19 +0900 Subject: [PATCH 089/381] Use std::make_pair for simplicity (#170) --- simplecpp.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0c38f6fe..ab91efc6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2484,26 +2484,26 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage) { std::map sizeOfType(rawtokens.sizeOfType); - sizeOfType.insert(std::pair("char", sizeof(char))); - sizeOfType.insert(std::pair("short", sizeof(short))); - sizeOfType.insert(std::pair("short int", sizeOfType["short"])); - sizeOfType.insert(std::pair("int", sizeof(int))); - sizeOfType.insert(std::pair("long", sizeof(long))); - sizeOfType.insert(std::pair("long int", sizeOfType["long"])); - sizeOfType.insert(std::pair("long long", sizeof(long long))); - sizeOfType.insert(std::pair("float", sizeof(float))); - sizeOfType.insert(std::pair("double", sizeof(double))); - sizeOfType.insert(std::pair("long double", sizeof(long double))); - sizeOfType.insert(std::pair("char *", sizeof(char *))); - sizeOfType.insert(std::pair("short *", sizeof(short *))); - sizeOfType.insert(std::pair("short int *", sizeOfType["short *"])); - sizeOfType.insert(std::pair("int *", sizeof(int *))); - sizeOfType.insert(std::pair("long *", sizeof(long *))); - sizeOfType.insert(std::pair("long int *", sizeOfType["long *"])); - sizeOfType.insert(std::pair("long long *", sizeof(long long *))); - sizeOfType.insert(std::pair("float *", sizeof(float *))); - sizeOfType.insert(std::pair("double *", sizeof(double *))); - sizeOfType.insert(std::pair("long double *", sizeof(long double *))); + sizeOfType.insert(std::make_pair("char", sizeof(char))); + sizeOfType.insert(std::make_pair("short", sizeof(short))); + sizeOfType.insert(std::make_pair("short int", sizeOfType["short"])); + sizeOfType.insert(std::make_pair("int", sizeof(int))); + sizeOfType.insert(std::make_pair("long", sizeof(long))); + sizeOfType.insert(std::make_pair("long int", sizeOfType["long"])); + sizeOfType.insert(std::make_pair("long long", sizeof(long long))); + sizeOfType.insert(std::make_pair("float", sizeof(float))); + sizeOfType.insert(std::make_pair("double", sizeof(double))); + sizeOfType.insert(std::make_pair("long double", sizeof(long double))); + sizeOfType.insert(std::make_pair("char *", sizeof(char *))); + sizeOfType.insert(std::make_pair("short *", sizeof(short *))); + sizeOfType.insert(std::make_pair("short int *", sizeOfType["short *"])); + sizeOfType.insert(std::make_pair("int *", sizeof(int *))); + sizeOfType.insert(std::make_pair("long *", sizeof(long *))); + sizeOfType.insert(std::make_pair("long int *", sizeOfType["long *"])); + sizeOfType.insert(std::make_pair("long long *", sizeof(long long *))); + sizeOfType.insert(std::make_pair("float *", sizeof(float *))); + sizeOfType.insert(std::make_pair("double *", sizeof(double *))); + sizeOfType.insert(std::make_pair("long double *", sizeof(long double *))); std::map macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { From 229d41f8f625632440d580fd6b1938ac53d72156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 3 Jul 2019 16:56:26 +0200 Subject: [PATCH 090/381] Fixed MACRO() expansion in #if condition --- simplecpp.cpp | 3 +++ test.cpp | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index ab91efc6..2493a0a0 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2750,6 +2750,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL output.clear(); return; } + if (!tmp) + break; + tok = tmp->previous; } try { conditionIsTrue = (evaluate(expr, sizeOfType) != 0); diff --git a/test.cpp b/test.cpp index c4a403f0..28979013 100644 --- a/test.cpp +++ b/test.cpp @@ -1001,6 +1001,15 @@ static void ifalt() // using "and", "or", etc ASSERT_EQUALS("\n1", preprocess(code)); } +static void ifexpr() +{ + const char *code = "#define MACRO() (1)\n" + "#if ~MACRO() & 8\n" + "1\n" + "#endif"; + ASSERT_EQUALS("\n\n1", preprocess(code)); +} + static void location1() { const char *code; @@ -1903,6 +1912,7 @@ int main(int argc, char **argv) TEST_CASE(ifoverflow); TEST_CASE(ifdiv0); TEST_CASE(ifalt); // using "and", "or", etc + TEST_CASE(ifexpr); TEST_CASE(location1); TEST_CASE(location2); From b00c8e7fcf668d03339bc88f9c7ad50f36d40127 Mon Sep 17 00:00:00 2001 From: Oleg Bolshakov <53259526+o-bolshakov@users.noreply.github.com> Date: Wed, 24 Jul 2019 22:46:22 +0300 Subject: [PATCH 091/381] Added octal numbers recognition (#172) * Added octal numbers recognition * More strict octal numbers recognition. constFold() is extended to check octals. --- simplecpp.cpp | 17 ++++++++++++++--- test.cpp | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2493a0a0..d8beca1b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -45,6 +45,11 @@ static bool isHex(const std::string &s) return s.size()>2 && (s.compare(0,2,"0x")==0 || s.compare(0,2,"0X")==0); } +static bool isOct(const std::string &s) +{ + return s.size()>1 && (s[0]=='0') && (s[1] >= '0') && (s[1] < '8'); +} + static const simplecpp::TokenString DEFINE("define"); static const simplecpp::TokenString UNDEF("undef"); @@ -76,9 +81,12 @@ static long long stringToLL(const std::string &s) { long long ret; const bool hex = isHex(s); - std::istringstream istr(hex ? s.substr(2) : s); + const bool oct = isOct(s); + std::istringstream istr(hex ? s.substr(2) : oct ? s.substr(1) : s); if (hex) istr >> std::hex; + else if (oct) + istr >> std::oct; istr >> ret; return ret; } @@ -87,9 +95,12 @@ static unsigned long long stringToULL(const std::string &s) { unsigned long long ret; const bool hex = isHex(s); - std::istringstream istr(hex ? s.substr(2) : s); + const bool oct = isOct(s); + std::istringstream istr(hex ? s.substr(2) : oct ? s.substr(1) : s); if (hex) istr >> std::hex; + else if (oct) + istr >> std::oct; istr >> ret; return ret; } @@ -765,7 +776,7 @@ void simplecpp::TokenList::combineOperators() } // match: [0-9.]+E [+-] [0-9]+ const char lastChar = tok->str()[tok->str().size() - 1]; - if (tok->number && !isHex(tok->str()) && (lastChar == 'E' || lastChar == 'e') && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { + if (tok->number && !isHex(tok->str()) && !isOct(tok->str()) && (lastChar == 'E' || lastChar == 'e') && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { tok->setstr(tok->str() + tok->next->op + tok->next->next->str()); deleteToken(tok->next); deleteToken(tok->next); diff --git a/test.cpp b/test.cpp index 28979013..0af8d24e 100644 --- a/test.cpp +++ b/test.cpp @@ -204,6 +204,8 @@ static void constFold() ASSERT_EQUALS("29", testConstFold("13 ^ 16")); ASSERT_EQUALS("25", testConstFold("24 | 1")); ASSERT_EQUALS("2", testConstFold("1?2:3")); + ASSERT_EQUALS("24", testConstFold("010+020")); + ASSERT_EQUALS("1", testConstFold("010==8")); ASSERT_EQUALS("exception", testConstFold("!1 ? 2 :")); ASSERT_EQUALS("exception", testConstFold("?2:3")); } From 28f2dee07281e2dcf1f31159eb6358da5ee2c7c1 Mon Sep 17 00:00:00 2001 From: IOBYTE Date: Wed, 4 Sep 2019 03:21:14 -0400 Subject: [PATCH 092/381] make ellipsis ... a single token (#176) Using cppcheck -E to preprocess code with ellipsis produces output that can't be compiled because ... is split into 3 tokens. --- simplecpp.cpp | 18 ++++++++++-------- test.cpp | 14 +++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d8beca1b..3120ec87 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -756,10 +756,14 @@ void simplecpp::TokenList::combineOperators() } if (tok->op == '.') { - if (tok->previous && tok->previous->op == '.') - continue; - if (tok->next && tok->next->op == '.') + // ellipsis ... + if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) && + tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) { + tok->setstr("..."); + deleteToken(tok->next); + deleteToken(tok->next); continue; + } // float literals.. if (tok->previous && tok->previous->number) { tok->setstr(tok->previous->str() + '.'); @@ -1387,14 +1391,12 @@ namespace simplecpp { args.clear(); const Token *argtok = nameTokDef->next->next; while (sameline(nametoken, argtok) && argtok->op != ')') { - if (argtok->op == '.' && - argtok->next && argtok->next->op == '.' && - argtok->next->next && argtok->next->next->op == '.' && - argtok->next->next->next && argtok->next->next->next->op == ')') { + if (argtok->str() == "..." && + argtok->next && argtok->next->op == ')') { variadic = true; if (!argtok->previous->name) args.push_back("__VA_ARGS__"); - argtok = argtok->next->next->next; // goto ')' + argtok = argtok->next; // goto ')' break; } if (argtok->op != ',') diff --git a/test.cpp b/test.cpp index 0af8d24e..7246c023 100644 --- a/test.cpp +++ b/test.cpp @@ -175,6 +175,12 @@ static void combineOperators_andequal() ASSERT_EQUALS("f ( x &= 2 ) ;", preprocess("f(x &= 2);")); } +static void combineOperators_ellipsis() +{ + ASSERT_EQUALS("void f ( int , ... ) ;", preprocess("void f(int, ...);")); + ASSERT_EQUALS("void f ( ) { switch ( x ) { case 1 ... 4 : } }", preprocess("void f() { switch(x) { case 1 ... 4: } }")); +} + static void comment() { ASSERT_EQUALS("// abc", readfile("// abc")); @@ -506,11 +512,6 @@ static void dollar() ASSERT_EQUALS("a$b", readfile("a$b")); } -static void dotDotDot() -{ - ASSERT_EQUALS("1 . . . 2", readfile("1 ... 2")); -} - static void error1() { std::istringstream istr("#error hello world!\n"); @@ -1829,6 +1830,7 @@ int main(int argc, char **argv) TEST_CASE(combineOperators_increment); TEST_CASE(combineOperators_coloncolon); TEST_CASE(combineOperators_andequal); + TEST_CASE(combineOperators_ellipsis); TEST_CASE(comment); TEST_CASE(comment_multiline); @@ -1872,8 +1874,6 @@ int main(int argc, char **argv) TEST_CASE(dollar); - TEST_CASE(dotDotDot); // ... - TEST_CASE(error1); TEST_CASE(error2); TEST_CASE(error3); From 4fb271d9bc15a162611893f03e860f8486d209ec Mon Sep 17 00:00:00 2001 From: IOBYTE Date: Wed, 25 Sep 2019 09:26:33 -0400 Subject: [PATCH 093/381] Fix tokenization of ">> ==" (#179) ">> ==" was tokenized to ">>= =". It is now tokenized to ">> ==". --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3120ec87..f94710b3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -841,7 +841,7 @@ void simplecpp::TokenList::combineOperators() } else if ((tok->op == '<' || tok->op == '>') && tok->op == tok->next->op) { tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); - if (tok->next && tok->next->op == '=') { + if (tok->next && tok->next->op == '=' && tok->next->next && tok->next->next->op != '=') { tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } From ef7712e3e456219806b5c03222e2b7a3c62ed7cd Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 6 Oct 2019 07:24:40 +0200 Subject: [PATCH 094/381] Fix #120 (handle multiline character literals) (#180) Make simplecpp accept the following code: const char*ptr="\\ \n"; Output from simplecpp: const char * ptr = "\\n" ; Output from gcc -E: const char* ptr = "\\n"; Do this by extending the special casing for backslashes to read all consecutive backslashes before continuing. --- simplecpp.cpp | 20 ++++++++++++++------ test.cpp | 8 ++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f94710b3..af48ca27 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1133,12 +1133,20 @@ std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location & backslash = false; ret += ch; if (ch == '\\') { - const char next = readChar(istr, bom); - if (next == '\r' || next == '\n') { - ret.erase(ret.size()-1U); - backslash = (next == '\r'); - } - ret += next; + bool update_ch = false; + char next = 0; + do { + next = readChar(istr, bom); + if (next == '\r' || next == '\n') { + ret.erase(ret.size()-1U); + backslash = (next == '\r'); + update_ch = false; + } else if (next == '\\') + update_ch = !update_ch; + ret += next; + } while (next == '\\'); + if (update_ch) + ch = next; } } diff --git a/test.cpp b/test.cpp index 7246c023..e0ca1be1 100644 --- a/test.cpp +++ b/test.cpp @@ -1229,6 +1229,13 @@ static void multiline9() // multiline prefix string in macro ASSERT_EQUALS("\n\nu8\"a b\" ;", preprocess(code)); } +static void multiline10() // multiline string literal +{ + const char code[] = "const char *ptr = \"\\\\\n" + "\\n\";"; + ASSERT_EQUALS("const char * ptr = \"\\\\n\"\n;", preprocess(code)); +} + static void nullDirective1() { const char code[] = "#\n" @@ -1947,6 +1954,7 @@ int main(int argc, char **argv) TEST_CASE(multiline7); // multiline string in macro TEST_CASE(multiline8); // multiline prefix string in macro TEST_CASE(multiline9); // multiline prefix string in macro + TEST_CASE(multiline10); TEST_CASE(readfile_nullbyte); TEST_CASE(readfile_char); From 8e260b208540d0f1772e3be4dab3d46093681e6c Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Wed, 23 Oct 2019 15:47:00 +0200 Subject: [PATCH 095/381] Improve support for hexadecimal floating point literal (#181) Handle plus and minus sign in the exponent. Hexadecimal floating point numbers were added in C99 and C++17. References: https://en.cppreference.com/w/cpp/language/floating_literal https://en.cppreference.com/w/c/language/floating_constant --- simplecpp.cpp | 5 ++++- test.cpp | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index af48ca27..2eabaf4a 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -780,7 +780,10 @@ void simplecpp::TokenList::combineOperators() } // match: [0-9.]+E [+-] [0-9]+ const char lastChar = tok->str()[tok->str().size() - 1]; - if (tok->number && !isHex(tok->str()) && !isOct(tok->str()) && (lastChar == 'E' || lastChar == 'e') && tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { + if (tok->number && !isOct(tok->str()) && + ((!isHex(tok->str()) && (lastChar == 'E' || lastChar == 'e')) || + (isHex(tok->str()) && (lastChar == 'P' || lastChar == 'p'))) && + tok->next && tok->next->isOneOf("+-") && tok->next->next && tok->next->next->number) { tok->setstr(tok->str() + tok->next->op + tok->next->next->str()); deleteToken(tok->next); deleteToken(tok->next); diff --git a/test.cpp b/test.cpp index e0ca1be1..6e05c026 100644 --- a/test.cpp +++ b/test.cpp @@ -154,6 +154,11 @@ static void combineOperators_floatliteral() ASSERT_EQUALS("1E-7", preprocess("1E-7")); ASSERT_EQUALS("1E+7", preprocess("1E+7")); ASSERT_EQUALS("0x1E + 7", preprocess("0x1E+7")); + ASSERT_EQUALS("0x1.2p3", preprocess("0x1.2p3")); + ASSERT_EQUALS("0x1p+3", preprocess("0x1p+3")); + ASSERT_EQUALS("0x1p+3f", preprocess("0x1p+3f")); + ASSERT_EQUALS("0x1p+3L", preprocess("0x1p+3L")); + ASSERT_EQUALS("1p + 3", preprocess("1p+3")); } static void combineOperators_increment() From 71adfce5898bad0a059938948b4d9e1ab767189f Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sat, 16 Nov 2019 08:04:05 +0100 Subject: [PATCH 096/381] Fix hexadecimal float parsing when dot is followed by letter (#182) --- simplecpp.cpp | 2 +- test.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2eabaf4a..9fd88d28 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -768,7 +768,7 @@ void simplecpp::TokenList::combineOperators() if (tok->previous && tok->previous->number) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); - if (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("Ee"))) { + if (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp"))) { tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } diff --git a/test.cpp b/test.cpp index 6e05c026..27c7be06 100644 --- a/test.cpp +++ b/test.cpp @@ -153,8 +153,15 @@ static void combineOperators_floatliteral() ASSERT_EQUALS("1E7", preprocess("1E7")); ASSERT_EQUALS("1E-7", preprocess("1E-7")); ASSERT_EQUALS("1E+7", preprocess("1E+7")); + ASSERT_EQUALS("1.e+7", preprocess("1.e+7")); ASSERT_EQUALS("0x1E + 7", preprocess("0x1E+7")); + ASSERT_EQUALS("0x1ffp10", preprocess("0x1ffp10")); + ASSERT_EQUALS("0x0p-1", preprocess("0x0p-1")); + ASSERT_EQUALS("0x1.p0", preprocess("0x1.p0")); + ASSERT_EQUALS("0xf.p-1", preprocess("0xf.p-1")); ASSERT_EQUALS("0x1.2p3", preprocess("0x1.2p3")); + ASSERT_EQUALS("0x1.ap3", preprocess("0x1.ap3")); + ASSERT_EQUALS("0x1.2ap3", preprocess("0x1.2ap3")); ASSERT_EQUALS("0x1p+3", preprocess("0x1p+3")); ASSERT_EQUALS("0x1p+3f", preprocess("0x1p+3f")); ASSERT_EQUALS("0x1p+3L", preprocess("0x1p+3L")); From c27e5d46dced1f8ead1f3a755ce539cdd0ee42b2 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 7 Dec 2019 21:25:11 +0100 Subject: [PATCH 097/381] simplecpp.cpp: Issue error when an explicitly included file is not found (#184) * simplecpp.cpp: Issue error when an explicitly included file is not found If a file is explicitly included for example via command line option `-include=` but can not be opened this error message is issued now. Example: $ ./simplecpp -include=blah.h foo.c int main ( ) { return 0 ; } foo.c:1: missing header: Can not open include file 'blah.h' that is explicitly included for all files. Fixes #183 * Change `error.type` from `MISSING_HEADER` header to `ERROR` `MISSING_HEADER` is only meant as an information to the user, nothing really relevant while this issue reveals a problem with the command line options that should get fixed. * simplecpp.cpp: Fix error message * Add new output error type EXPLICIT_INCLUDE_NOT_FOUND --- main.cpp | 3 +++ simplecpp.cpp | 10 +++++++++- simplecpp.h | 3 ++- test.cpp | 3 +++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/main.cpp b/main.cpp index 31ca9b6f..54124a0f 100644 --- a/main.cpp +++ b/main.cpp @@ -86,6 +86,9 @@ int main(int argc, char **argv) case simplecpp::Output::UNHANDLED_CHAR_ERROR: std::cerr << "unhandled char error: "; break; + case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: + std::cerr << "explicit include not found: "; + break; } std::cerr << output.msg << std::endl; } diff --git a/simplecpp.cpp b/simplecpp.cpp index 9fd88d28..2c38cd67 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2426,8 +2426,16 @@ std::map simplecpp::load(const simplecpp::To continue; std::ifstream fin(filename.c_str()); - if (!fin.is_open()) + if (!fin.is_open()) { + if (outputList) { + simplecpp::Output err(fileNumbers); + err.type = simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND; + err.location = Location(fileNumbers); + err.msg = "Can not open include file '" + filename + "' that is explicitly included."; + outputList->push_back(err); + } continue; + } TokenList *tokenlist = new TokenList(fin, fileNumbers, filename, outputList); if (!tokenlist->front()) { diff --git a/simplecpp.h b/simplecpp.h index 7f44d308..577711af 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -167,7 +167,8 @@ namespace simplecpp { INCLUDE_NESTED_TOO_DEEPLY, SYNTAX_ERROR, PORTABILITY_BACKSLASH, - UNHANDLED_CHAR_ERROR + UNHANDLED_CHAR_ERROR, + EXPLICIT_INCLUDE_NOT_FOUND } type; Location location; std::string msg; diff --git a/test.cpp b/test.cpp index 27c7be06..d647b980 100644 --- a/test.cpp +++ b/test.cpp @@ -92,6 +92,9 @@ static std::string toString(const simplecpp::OutputList &outputList) break; case simplecpp::Output::Type::UNHANDLED_CHAR_ERROR: ostr << "unhandled_char_error,"; + break; + case simplecpp::Output::Type::EXPLICIT_INCLUDE_NOT_FOUND: + ostr << "explicit_include_not_found,"; } ostr << output.msg << '\n'; From fc32948e713ae0ed75bf7d436a6ab418db08275c Mon Sep 17 00:00:00 2001 From: jannick0 Date: Sat, 28 Dec 2019 21:45:56 +0100 Subject: [PATCH 098/381] bug fix - de-escape double backslashes in file paths of #line directives (#187) * build(cmake): add test running TESTRUNNER to CMakeLists.txt * tests: add test handling escaped backslashes in #line directive file paths In #line directives backslashes in original file paths are escaped by double backslashes. * test.cpp: add test function LOCATION4 checking that in #line directive escaped backslashes (i.e. double backslashes) in file paths are de-escaped, i.e. converted back to a single backslash. * bug fix - handle escaped backslashes in #line dir file paths * simplecpp.cpp: - add static function REPLACEALL which replaces a string FROM to the string TO at every occurance in the input string. - simplecpp::TokenList::readfile: replace any double backslashes in any file path of a #line directive. --- CMakeLists.txt | 3 +++ simplecpp.cpp | 8 +++++++- test.cpp | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 63c81581..b1baef3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,3 +33,6 @@ endif() add_executable(testrunner simplecpp.cpp test.cpp) set_property(TARGET testrunner PROPERTY CXX_STANDARD 11) + +enable_testing() +add_test(NAME testrunner COMMAND testrunner) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2c38cd67..77e04bb9 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -137,6 +137,12 @@ static bool isAlternativeUnaryOp(const simplecpp::Token *tok, const std::string (tok->next && (tok->next->name || tok->next->number))); } +static std::string replaceAll(std::string s, const std::string& from, const std::string& to) +{ + for (size_t pos = s.find(from); pos != std::string::npos; pos = s.find(from, pos + to.size())) + s.replace(pos, from.size(), to); + return s; +} const std::string simplecpp::Location::emptyFileName; @@ -498,7 +504,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen } else if (lastline == "# line %num%") { lineDirective(location.fileIndex, std::atol(cback()->str().c_str()), &location); } else if (lastline == "# %num% %str%" || lastline == "# line %num% %str%") { - lineDirective(fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)), + lineDirective(fileIndex(replaceAll(cback()->str().substr(1U, cback()->str().size() - 2U),"\\\\","\\")), std::atol(cback()->previous->str().c_str()), &location); } // #endfile diff --git a/test.cpp b/test.cpp index d647b980..44a96616 100644 --- a/test.cpp +++ b/test.cpp @@ -1069,6 +1069,14 @@ static void location3() ASSERT_EQUALS("\n#line 1 \"x\"\na b", preprocess(code)); } +static void location4() +{ + const char *code; + code = "#line 1 \"abc\\\\def.g\" \n" + "a\n"; + ASSERT_EQUALS("\n#line 1 \"abc\\def.g\"\na", preprocess(code)); +} + static void missingHeader1() { const simplecpp::DUI dui; @@ -1941,6 +1949,7 @@ int main(int argc, char **argv) TEST_CASE(location1); TEST_CASE(location2); TEST_CASE(location3); + TEST_CASE(location4); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); From 24db46d94a34161b2ed9b0415d2ada82adb56731 Mon Sep 17 00:00:00 2001 From: jannick0 Date: Sun, 29 Dec 2019 14:00:39 +0100 Subject: [PATCH 099/381] simplecpp.cpp: astyling (#188) Goal is to keep file in sync with corresponding file in cppcheck repo. --- simplecpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 77e04bb9..6de39505 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -139,9 +139,9 @@ static bool isAlternativeUnaryOp(const simplecpp::Token *tok, const std::string static std::string replaceAll(std::string s, const std::string& from, const std::string& to) { - for (size_t pos = s.find(from); pos != std::string::npos; pos = s.find(from, pos + to.size())) - s.replace(pos, from.size(), to); - return s; + for (size_t pos = s.find(from); pos != std::string::npos; pos = s.find(from, pos + to.size())) + s.replace(pos, from.size(), to); + return s; } const std::string simplecpp::Location::emptyFileName; From 1bc600bc3cf5f7399b75c2cf5dce34fe157a1474 Mon Sep 17 00:00:00 2001 From: orbitcowboy Date: Fri, 3 Jan 2020 19:44:25 +0100 Subject: [PATCH 100/381] Fixed -Wshadow warning. (#186) * Fixed -Wshadow warning. * Updated local variable name to 'alternativeOp' --- simplecpp.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6de39505..0224c5f0 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1026,15 +1026,15 @@ void simplecpp::TokenList::constFoldBitwise(Token *tok) { Token * const tok1 = tok; for (const char *op = "&^|"; *op; op++) { - const std::string* altop; + const std::string* alternativeOp; if (*op == '&') - altop = &BITAND; + alternativeOp = &BITAND; else if (*op == '|') - altop = &BITOR; + alternativeOp = &BITOR; else - altop = &XOR; + alternativeOp = &XOR; for (tok = tok1; tok && tok->op != ')'; tok = tok->next) { - if (tok->op != *op && !isAlternativeBinaryOp(tok, *altop)) + if (tok->op != *op && !isAlternativeBinaryOp(tok, *alternativeOp)) continue; if (!tok->previous || !tok->previous->number) continue; From 4720ba798c6bb232d85662ff88d144cc064601a3 Mon Sep 17 00:00:00 2001 From: amai2012 Date: Sun, 10 May 2020 15:34:31 +0200 Subject: [PATCH 101/381] Add github action: unixish CI (#189) --- .github/workflows/CI-unixish.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/CI-unixish.yml diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml new file mode 100644 index 00000000..f513f2a0 --- /dev/null +++ b/.github/workflows/CI-unixish.yml @@ -0,0 +1,20 @@ +name: CI Unixish + +on: [push, pull_request] + +jobs: + build: + + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + fail-fast: false # not worthwhile... + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: make simplecpp + run: make + - name: make test + run: make test From 66b0762f7607f4d61ba3549d16da4aa5305f9da0 Mon Sep 17 00:00:00 2001 From: amai2012 Date: Sun, 17 May 2020 07:48:22 +0200 Subject: [PATCH 102/381] Add github action: CI windows (#190) --- .github/workflows/CI-windows.yml | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/CI-windows.yml diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml new file mode 100644 index 00000000..9ab288dc --- /dev/null +++ b/.github/workflows/CI-windows.yml @@ -0,0 +1,42 @@ +# Some convenient links: +# - https://github.com/actions/virtual-environments/blob/master/images/win/Windows2019-Readme.md +# + +name: CI-windows + +on: [push,pull_request] + +defaults: + run: + shell: cmd + +jobs: + + build: + strategy: + matrix: + # windows 2016 should default to VS 2017. Not supported by setup-msbuild + os: [windows-2019] + fail-fast: true + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Setup msbuild.exe + uses: microsoft/setup-msbuild@v1.0.0 + + - name: Run cmake + run: | + cmake -G "Visual Studio 16" . -A x64 + dir + + - name: Build + run: | + msbuild -m simplecpp.sln /p:Configuration=Release /p:Platform=x64 + + - name: Test + run: | + .\Release\testrunner.exe + From 546d0854008f146d6cabeccfed6c440533eb959d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 30 Jun 2020 21:07:59 +0200 Subject: [PATCH 103/381] Better handling of standalone # in macros (#108) --- simplecpp.cpp | 6 ++++++ test.cpp | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0224c5f0..327962e3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1619,6 +1619,12 @@ namespace simplecpp { continue; } + if (numberOfHash == 2 && tok->location.col + 1 < tok->next->location.col) { + output->push_back(new Token(*tok)); + tok = tok->next; + continue; + } + tok = tok->next; if (tok == endToken) { output->push_back(new Token(*tok->previous)); diff --git a/test.cpp b/test.cpp index 44a96616..4cdd3bae 100644 --- a/test.cpp +++ b/test.cpp @@ -759,6 +759,13 @@ static void hashhash9() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); } +static void hashhash10() +{ + const char code[] = "#define x # #\n" + "x"; + ASSERT_EQUALS("# #", preprocess(code)); +} + static void hashhash_invalid_1() { std::istringstream istr("#define f(a) (##x)\nf(1)"); @@ -1923,6 +1930,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) TEST_CASE(hashhash8); TEST_CASE(hashhash9); + TEST_CASE(hashhash10); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); From e6d9457f4ccba666e55e0d5eaedfdf6b8cee6231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 30 Jun 2020 21:18:20 +0200 Subject: [PATCH 104/381] Better handling of # in define (#60) --- simplecpp.cpp | 4 ++-- test.cpp | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 327962e3..4669fbcc 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1612,14 +1612,14 @@ namespace simplecpp { hashToken = hashToken->next; ++numberOfHash; } - if (numberOfHash == 4) { + if (numberOfHash == 4 && tok->next->location.col + 1 == tok->next->next->location.col) { // # ## # => ## output->push_back(newMacroToken("##", loc, isReplaced(expandedmacros))); tok = hashToken; continue; } - if (numberOfHash == 2 && tok->location.col + 1 < tok->next->location.col) { + if (numberOfHash >= 2 && tok->location.col + 1 < tok->next->location.col) { output->push_back(new Token(*tok)); tok = tok->next; continue; diff --git a/test.cpp b/test.cpp index 4cdd3bae..0279f0fe 100644 --- a/test.cpp +++ b/test.cpp @@ -766,6 +766,13 @@ static void hashhash10() ASSERT_EQUALS("# #", preprocess(code)); } +static void hashhash11() +{ + const char code[] = "#define x # # #\n" + "x"; + ASSERT_EQUALS("# # #", preprocess(code)); +} + static void hashhash_invalid_1() { std::istringstream istr("#define f(a) (##x)\nf(1)"); @@ -1930,7 +1937,8 @@ int main(int argc, char **argv) TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) TEST_CASE(hashhash8); TEST_CASE(hashhash9); - TEST_CASE(hashhash10); + TEST_CASE(hashhash10); // #108 : #define x # # + TEST_CASE(hashhash11); // #60: #define x # # # TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); From 83fc3cbe2be42a7047000d793f46c315b86cb949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 1 Jul 2020 14:55:22 +0200 Subject: [PATCH 105/381] Fixed #142 (__LINE__ is not resolved properly) --- simplecpp.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4669fbcc..9d739315 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1472,6 +1472,7 @@ namespace simplecpp { } const Token *appendTokens(TokenList *tokens, + const Location &rawloc, const Token *lpar, const std::map ¯os, const std::set &expandedmacros, @@ -1483,17 +1484,17 @@ namespace simplecpp { while (sameline(lpar, tok)) { if (tok->op == '#' && sameline(tok,tok->next) && tok->next->op == '#' && sameline(tok,tok->next->next)) { // A##B => AB - tok = expandHashHash(tokens, tok->location, tok, macros, expandedmacros, parametertokens); + tok = expandHashHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens); } else if (tok->op == '#' && sameline(tok, tok->next) && tok->next->op != '#') { - tok = expandHash(tokens, tok->location, tok, macros, expandedmacros, parametertokens); + tok = expandHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens); } else { - if (!expandArg(tokens, tok, tok->location, macros, expandedmacros, parametertokens)) { + if (!expandArg(tokens, tok, rawloc, macros, expandedmacros, parametertokens)) { bool expanded = false; const std::map::const_iterator it = macros.find(tok->str()); if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { - m.expand(tokens, tok->location, tok, macros, expandedmacros); + m.expand(tokens, rawloc, tok, macros, expandedmacros); expanded = true; } } @@ -1687,7 +1688,7 @@ namespace simplecpp { TokenList temp2(files); temp2.push_back(new Token(temp.cback()->str(), tok->location)); - const Token *tok2 = appendTokens(&temp2, tok->next, macros, expandedmacros, parametertokens); + const Token *tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) return tok->next; @@ -1711,7 +1712,7 @@ namespace simplecpp { } TokenList tokens(files); tokens.push_back(new Token(*tok)); - const Token *tok2 = appendTokens(&tokens, tok->next, macros, expandedmacros, parametertokens); + const Token *tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) { output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; @@ -1885,7 +1886,7 @@ namespace simplecpp { if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { const std::map::const_iterator it = macros.find(strAB); if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { - const Token *tok2 = appendTokens(&tokens, B->next, macros, expandedmacros, parametertokens); + const Token *tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); if (tok2) nextTok = tok2->next; } From 2a0c07f584206d1a6b8ccc0eeda3bffb91e957b8 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Thu, 2 Jul 2020 08:40:18 +0200 Subject: [PATCH 106/381] run-tests: Make it run with both python 2 and 3 (#193) --- run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.py b/run-tests.py index 785fd89c..b19aeddf 100644 --- a/run-tests.py +++ b/run-tests.py @@ -5,7 +5,7 @@ def cleanup(out): ret = '' - for s in out.split('\n'): + for s in out.decode('utf-8').split('\n'): if len(s) > 1 and s[0] == '#': continue s = "".join(s.split()) From f0a9cee8741c766c309d07452e76b51335edf139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 2 Jul 2020 18:05:54 +0200 Subject: [PATCH 107/381] Fixed #79 (Incorrectly preprocessed cpp file) --- simplecpp.cpp | 88 ++++++++++++++++++++++++++++----------------------- test.cpp | 20 ++++++++++++ 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9d739315..900e07c6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1652,6 +1652,41 @@ namespace simplecpp { return functionLike() ? parametertokens2.back()->next : nameTokInst->next; } + const Token *recursiveExpandToken(TokenList *output, TokenList &temp, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + if (!(temp.cback() && temp.cback()->name && tok->next && tok->next->op == '(')) { + output->takeTokens(temp); + return tok->next; + } + + if (!sameline(tok, tok->next)) { + output->takeTokens(temp); + return tok->next; + } + + const std::map::const_iterator it = macros.find(temp.cback()->str()); + if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) { + output->takeTokens(temp); + return tok->next; + } + + const Macro &calledMacro = it->second; + if (!calledMacro.functionLike()) { + output->takeTokens(temp); + return tok->next; + } + + TokenList temp2(files); + temp2.push_back(new Token(temp.cback()->str(), tok->location)); + + const Token *tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); + if (!tok2) + return tok->next; + output->takeTokens(temp); + output->deleteToken(output->back()); + calledMacro.expand(output, loc, temp2.cfront(), macros, expandedmacros); + return tok2->next; + } + const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { @@ -1662,50 +1697,22 @@ namespace simplecpp { // Macro parameter.. { TokenList temp(files); - if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) { - if (!(temp.cback() && temp.cback()->name && tok->next && tok->next->op == '(')) { - output->takeTokens(temp); - return tok->next; - } - - if (!sameline(tok, tok->next)) { - output->takeTokens(temp); - return tok->next; - } - - const std::map::const_iterator it = macros.find(temp.cback()->str()); - if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) { - output->takeTokens(temp); - return tok->next; - } - - const Macro &calledMacro = it->second; - if (!calledMacro.functionLike()) { - output->takeTokens(temp); - return tok->next; - } - - TokenList temp2(files); - temp2.push_back(new Token(temp.cback()->str(), tok->location)); - - const Token *tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); - if (!tok2) - return tok->next; - - output->takeTokens(temp); - output->deleteToken(output->back()); - calledMacro.expand(output, loc, temp2.cfront(), macros, expandedmacros); - - return tok2->next; - } + if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) + return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); } // Macro.. const std::map::const_iterator it = macros.find(tok->str()); if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { + std::set expandedmacros2(expandedmacros); + expandedmacros2.insert(tok->str()); + const Macro &calledMacro = it->second; - if (!calledMacro.functionLike()) - return calledMacro.expand(output, loc, tok, macros, expandedmacros); + if (!calledMacro.functionLike()) { + TokenList temp(files); + calledMacro.expand(&temp, loc, tok, macros, expandedmacros); + return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); + } if (!sameline(tok, tok->next) || tok->next->op != '(') { output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; @@ -1717,8 +1724,9 @@ namespace simplecpp { output->push_back(newMacroToken(tok->str(), loc, true)); return tok->next; } - calledMacro.expand(output, loc, tokens.cfront(), macros, expandedmacros); - return tok2->next; + TokenList temp(files); + calledMacro.expand(&temp, loc, tokens.cfront(), macros, expandedmacros); + return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros2, parametertokens); } else if (tok->str() == DEFINED) { diff --git a/test.cpp b/test.cpp index 0279f0fe..1e7115b7 100644 --- a/test.cpp +++ b/test.cpp @@ -485,6 +485,24 @@ static void define_define_14() // issue #58 - endless recursion ASSERT_EQUALS("\n\n\nf ( f ( w", preprocess(code)); // Don't crash } +static void define_define_15() // issue #72 without __VA_ARGS__ +{ + const char code[] = "#define a f\n" + "#define foo(x,y) a(x,y)\n" + "#define f(x, y) x y\n" + "foo(1,2)"; + ASSERT_EQUALS("\n\n\n1 2", preprocess(code)); +} + +static void define_define_16() // issue #72 with __VA_ARGS__ +{ + const char code[] = "#define ab(a, b) a##b\n" + "#define foo(...) ab(f, 2) (__VA_ARGS__)\n" + "#define f2(x, y) x y\n" + "foo(1,2)"; + ASSERT_EQUALS("\n\n\n1 2", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -1909,6 +1927,8 @@ int main(int argc, char **argv) TEST_CASE(define_define_12); // expand result of ## TEST_CASE(define_define_13); TEST_CASE(define_define_14); + TEST_CASE(define_define_15); + TEST_CASE(define_define_16); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From fe7f1c320e8cf86e02ee4ad782c30e075a8340a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 3 Jul 2020 11:38:10 +0200 Subject: [PATCH 108/381] Fixed wrong locations in expanded macros --- simplecpp.cpp | 4 +++- test.cpp | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 900e07c6..c78bc874 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1473,7 +1473,7 @@ namespace simplecpp { const Token *appendTokens(TokenList *tokens, const Location &rawloc, - const Token *lpar, + const Token * const lpar, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { @@ -1512,6 +1512,8 @@ namespace simplecpp { tok = tok->next; } } + for (Token *tok2 = tokens->front(); tok2; tok2 = tok2->next) + tok2->location = lpar->location; return sameline(lpar,tok) ? tok : NULL; } diff --git a/test.cpp b/test.cpp index 1e7115b7..dcfe2c92 100644 --- a/test.cpp +++ b/test.cpp @@ -339,6 +339,19 @@ static void define10() // don't combine prefix with space in macro ASSERT_EQUALS("\nu8 \"a b\" ;", preprocess(code)); } +static void define11() // location of expanded argument +{ + const char code[] = "#line 4 \"version.h\"\n" + "#define A(x) B(x)\n" + "#define B(x) x\n" + "#define VER A(1)\n" + "\n" + "#line 10 \"cppcheck.cpp\"\n" + "VER;"; + ASSERT_EQUALS("\n#line 10 \"cppcheck.cpp\"\n1 ;", preprocess(code)); +} + + static void define_invalid_1() { std::istringstream istr("#define A(\nB\n"); @@ -1911,6 +1924,7 @@ int main(int argc, char **argv) TEST_CASE(define8); TEST_CASE(define9); TEST_CASE(define10); + TEST_CASE(define11); TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_define_1); From 391de4cd597a1357317fb0edcc3a77c5b8dd7bfd Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sat, 4 Jul 2020 08:02:02 +0200 Subject: [PATCH 109/381] Remove fixed test case from TODO (#194) It was fixed in f0a9cee8741c. --- run-tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run-tests.py b/run-tests.py index b19aeddf..0211ed04 100644 --- a/run-tests.py +++ b/run-tests.py @@ -59,7 +59,6 @@ def cleanup(out): 'macro_backslash.c', 'macro_fn_comma_swallow.c', 'macro_fn_comma_swallow2.c', - 'macro_fn_lparen_scan.c', 'macro_expand.c', 'macro_fn_disable_expand.c', 'macro_paste_commaext.c', From 11a9809db939218064d8836fcd4f85d016e8dd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 11 Sep 2020 18:27:37 +0200 Subject: [PATCH 110/381] if system header can't be found in a -I path, try to find header locally --- simplecpp.cpp | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c78bc874..61cc1d38 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2363,25 +2363,23 @@ static std::string _openHeader(std::ifstream &f, const std::string &path) #endif } -static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) +static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) { - if (isAbsolutePath(header)) { - return _openHeader(f, header); - } - - if (!systemheader) { - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - std::string simplePath = _openHeader(f, s); - if (!simplePath.empty()) - return simplePath; - } else { - std::string simplePath = _openHeader(f, header); - if (!simplePath.empty()) - return simplePath; - } + if (sourcefile.find_first_of("\\/") != std::string::npos) { + const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + std::string simplePath = _openHeader(f, s); + if (!simplePath.empty()) + return simplePath; + } else { + std::string simplePath = _openHeader(f, header); + if (!simplePath.empty()) + return simplePath; } + return ""; +} +static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) +{ for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { std::string s = *it; if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') @@ -2392,10 +2390,25 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (!simplePath.empty()) return simplePath; } - return ""; } +static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) +{ + if (isAbsolutePath(header)) + return _openHeader(f, header); + + std::string ret; + + if (systemheader) { + ret = openHeaderIncludePath(f, dui, header); + return ret.empty() ? openHeaderRelative(f, sourcefile, header) : ret; + } + + ret = openHeaderRelative(f, sourcefile, header); + return ret.empty() ? openHeaderIncludePath(f, dui, header) : ret; +} + static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { if (filedata.empty()) { From 75feeb5c6a344ec97355a9728e5c8fce966aaf7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 14 Sep 2020 15:42:23 +0200 Subject: [PATCH 111/381] fix hang when header includes itself --- simplecpp.cpp | 57 ++++++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 61cc1d38..52ba3ea6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2363,30 +2363,30 @@ static std::string _openHeader(std::ifstream &f, const std::string &path) #endif } +static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) +{ + if (sourcefile.find_first_of("\\/") != std::string::npos) + return simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header); + return simplecpp::simplifyPath(header); +} + static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) { - if (sourcefile.find_first_of("\\/") != std::string::npos) { - const std::string s = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - std::string simplePath = _openHeader(f, s); - if (!simplePath.empty()) - return simplePath; - } else { - std::string simplePath = _openHeader(f, header); - if (!simplePath.empty()) - return simplePath; - } - return ""; + return _openHeader(f, getRelativeFileName(sourcefile, header)); +} + +static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) +{ + std::string path = includePath; + if (!path.empty() && path[path.size()-1U]!='/' && path[path.size()-1U]!='\\') + path += '/'; + return path + header; } static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) { for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = *it; - if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') - s += '/'; - s += header; - - std::string simplePath = _openHeader(f, s); + std::string simplePath = _openHeader(f, getIncludePathFileName(*it, header)); if (!simplePath.empty()) return simplePath; } @@ -2418,28 +2418,19 @@ static std::string getFileName(const std::map::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = *it; - if (!s.empty() && s[s.size()-1U]!='/' && s[s.size()-1U]!='\\') - s += '/'; - s += header; - s = simplecpp::simplifyPath(s); + std::string s = simplecpp::simplifyPath(getIncludePathFileName(*it, header)); if (filedata.find(s) != filedata.end()) return s; } + if (filedata.find(relativeFilename) != filedata.end()) + return relativeFilename; + return ""; } From 7b25866ca97c41f54911e278672319feb52c5b0c Mon Sep 17 00:00:00 2001 From: amai2012 Date: Tue, 1 Dec 2020 08:16:27 +0100 Subject: [PATCH 112/381] Update to microsoft/setup-msbuild@v1.0.2 (#198) --- .github/workflows/CI-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 9ab288dc..f64e34f9 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup msbuild.exe - uses: microsoft/setup-msbuild@v1.0.0 + uses: microsoft/setup-msbuild@v1.0.2 - name: Run cmake run: | From f783d60bacaf01a7ef1b6de843df8939eb68022b Mon Sep 17 00:00:00 2001 From: amai2012 Date: Tue, 1 Dec 2020 18:04:28 +0100 Subject: [PATCH 113/381] Add github build status (#199) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4d5c1328..e4bdb2f1 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ Simplecpp has better fidelity than normal C/C++ preprocessors. This information is normally lost during preprocessing but it can be necessary for proper static analysis. +## Status +![CI-windows](https://github.com/danmar/simplecpp/workflows/CI-windows/badge.svg) +![CI Unixish](https://github.com/danmar/simplecpp/workflows/CI%20Unixish/badge.svg) + + ## Compiling Compiling standalone simplecpp preprocessor: From d940320031260e69e1e087056f3896e0ede4342c Mon Sep 17 00:00:00 2001 From: amai2012 Date: Wed, 2 Dec 2020 07:29:22 +0100 Subject: [PATCH 114/381] Add Valgrind to CI build (#197) --- .github/workflows/CI-unixish.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index f513f2a0..7bf4c3e7 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -14,7 +14,21 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install missing software on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install valgrind + - name: make simplecpp - run: make + run: make -j$(nproc) + - name: make test - run: make test + run: make -j$(nproc) test + + - name: Run valgrind + if: matrix.os == 'ubuntu-latest' + run: | + valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all ./testrunner + From 78db47388be25a2f8d37d98797132585c8b9a710 Mon Sep 17 00:00:00 2001 From: amai2012 Date: Thu, 24 Dec 2020 20:11:45 +0100 Subject: [PATCH 115/381] Extend set of virtual environments (#200) --- .github/workflows/CI-unixish.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 7bf4c3e7..e1fb327d 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -7,8 +7,9 @@ jobs: strategy: matrix: - os: [ubuntu-latest, macos-latest] - fail-fast: false # not worthwhile... + compiler: [clang++, g++] + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11.0] + fail-fast: true runs-on: ${{ matrix.os }} @@ -16,19 +17,19 @@ jobs: - uses: actions/checkout@v2 - name: Install missing software on ubuntu - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update sudo apt-get install valgrind - name: make simplecpp - run: make -j$(nproc) + run: make -j$(nproc) CXX=${{ matrix.compiler }} - name: make test - run: make -j$(nproc) test + run: make -j$(nproc) test CXX=${{ matrix.compiler }} - name: Run valgrind - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-20.04' run: | - valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all ./testrunner + valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner From ecb1225a41bdf2cb545a5065030d9506f029bcea Mon Sep 17 00:00:00 2001 From: amai2012 Date: Fri, 15 Jan 2021 08:29:19 +0100 Subject: [PATCH 116/381] Skip macos 11 for now (#206) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index e1fb327d..ed1914f5 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11.0] + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-10.15] fail-fast: true runs-on: ${{ matrix.os }} From 027bba1ae5a95fafa84dd427eb184ee4be0bae18 Mon Sep 17 00:00:00 2001 From: Ken-Patrick Lehrmann Date: Fri, 15 Jan 2021 08:30:18 +0100 Subject: [PATCH 117/381] Fix preprocessing of several ## (#205) ``` #define MAX_FOO (2 * 1) #define MAX_FOO_AA (3 * 1) #define M(UpperCaseName, b) \ do { \ int MaxValue = MAX_##UpperCaseName; \ if (b) { \ MaxValue = MAX_##UpperCaseName##_AA; \ } \ } while (0) static void f(bool b) { M(FOO, b); } ``` was a preprocessor error `a.cpp:4:0: error: failed to expand 'M', Invalid ## usage when expanding 'M'. [preprocessorErrorDirective]` because `MAX_##UpperCaseName##_AA` was expanded to (2 * 1)##_AA, while gcc or clang happily expand it to `(3 * 1)`. --- simplecpp.cpp | 7 ++++++- test.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 52ba3ea6..afe5b8ea 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1880,7 +1880,6 @@ namespace simplecpp { } const Token *nextTok = B->next; - if (varargs && tokensB.empty() && tok->previous->str() == ",") output->deleteToken(A); else if (strAB != "," && macros.find(strAB) == macros.end()) { @@ -1888,6 +1887,12 @@ namespace simplecpp { for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; output->takeTokens(tokensB); + } else if (nextTok->op == '#' && nextTok->next->op == '#') { + TokenList output2(files); + output2.push_back(new Token(strAB, tok->location)); + nextTok = expandHashHash(&output2, loc, nextTok, macros, expandedmacros, parametertokens); + output->deleteToken(A); + output->takeTokens(output2); } else { output->deleteToken(A); TokenList tokens(files); diff --git a/test.cpp b/test.cpp index dcfe2c92..6c4b30c9 100644 --- a/test.cpp +++ b/test.cpp @@ -804,6 +804,43 @@ static void hashhash11() ASSERT_EQUALS("# # #", preprocess(code)); } +static void hashhash12() +{ + { + const char code[] = "#define MAX_FOO 1\n" + "#define MAX_FOO_AA 2\n" + "\n" + "#define M(UpperCaseName, b) " + " do { " + " int MaxValue = MAX_##UpperCaseName; " + " if (b) { " + " MaxValue = MAX_##UpperCaseName##_AA; " + " } " + " } while (0)" + "\n" + "static void f(bool b) { M(FOO, b); }\n"; + + ASSERT_EQUALS("\n\n\n\nstatic void f ( bool b ) { do { int MaxValue = 1 ; if ( b ) { MaxValue = 2 ; } } while ( 0 ) ; }", preprocess(code)); + } + + { + const char code[] = "#define MAX_FOO (1 * 1)\n" + "#define MAX_FOO_AA (2 * 1)\n" + "\n" + "#define M(UpperCaseName, b) " + " do { " + " int MaxValue = MAX_##UpperCaseName; " + " if (b) { " + " MaxValue = MAX_##UpperCaseName##_AA; " + " } " + " } while (0)" + "\n" + "static void f(bool b) { M(FOO, b); }\n"; + + ASSERT_EQUALS("\n\n\n\nstatic void f ( bool b ) { do { int MaxValue = ( 1 * 1 ) ; if ( b ) { MaxValue = ( 2 * 1 ) ; } } while ( 0 ) ; }", preprocess(code)); + } +} + static void hashhash_invalid_1() { std::istringstream istr("#define f(a) (##x)\nf(1)"); @@ -1973,6 +2010,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash9); TEST_CASE(hashhash10); // #108 : #define x # # TEST_CASE(hashhash11); // #60: #define x # # # + TEST_CASE(hashhash12); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); From 3faade2eccf9292c525aa8ccc224b669a17e6be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 21 Feb 2021 19:11:38 +0100 Subject: [PATCH 118/381] Better handling of 'defined A##B'. Fixes #192 --- simplecpp.cpp | 13 ++++++++++++- test.cpp | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index afe5b8ea..7d48a715 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1744,7 +1744,18 @@ namespace simplecpp { defToken = lastToken = tok2; } if (defToken) { - const bool def = (macros.find(defToken->str()) != macros.end()); + std::string macroName = defToken->str(); + if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { + TokenList temp(files); + if (expandArg(&temp, defToken, parametertokens)) + macroName = temp.cback()->str(); + if (expandArg(&temp, defToken->next->next->next, parametertokens)) + macroName += temp.cback()->str(); + else + macroName += defToken->next->next->next->str(); + lastToken = defToken->next->next->next; + } + const bool def = (macros.find(macroName) != macros.end()); output->push_back(newMacroToken(def ? "1" : "0", loc, true)); return lastToken->next; } diff --git a/test.cpp b/test.cpp index 6c4b30c9..284f5bce 100644 --- a/test.cpp +++ b/test.cpp @@ -990,6 +990,25 @@ static void ifDefinedInvalid2() ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } +static void ifDefinedHashHash() +{ + const char code[] = "#define ENABLE(FEATURE) defined ENABLE_##FEATURE\n" + "#define ENABLE_FOO 1\n" + "#if ENABLE(FOO)\n" + "#error FOO is enabled\n" // <-- expected result + "#else\n" + "#error FOO is not enabled\n" + "#endif\n"; + simplecpp::DUI dui; + simplecpp::OutputList outputList; + std::vector files; + simplecpp::TokenList tokens2(files); + std::istringstream istr(code); + std::map filedata; + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("file0,4,#error,#error FOO is enabled\n", toString(outputList)); +} + static void ifLogical() { const char code[] = "#if defined(A) || defined(B)\n" @@ -2025,6 +2044,7 @@ int main(int argc, char **argv) TEST_CASE(ifDefinedNestedNoPar); TEST_CASE(ifDefinedInvalid1); TEST_CASE(ifDefinedInvalid2); + TEST_CASE(ifDefinedHashHash); TEST_CASE(ifLogical); TEST_CASE(ifSizeof); TEST_CASE(elif); From 8bfd5488f5ca962c26edfa863536ee08b11a56e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 11 Mar 2021 08:40:51 +0100 Subject: [PATCH 119/381] Stop using Travis --- .github/workflows/CI-unixish.yml | 3 +++ .travis.yml | 9 --------- 2 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index ed1914f5..5205d85e 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -28,6 +28,9 @@ jobs: - name: make test run: make -j$(nproc) test CXX=${{ matrix.compiler }} + - name: ensure that simplecpp.cpp uses c++03 + run: CXX=${{ matrix.compiler }} ; $CXX -fsyntax-only -std=c++98 simplecpp.cpp + - name: Run valgrind if: matrix.os == 'ubuntu-20.04' run: | diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ffaa11cc..00000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ - -language: cpp - -compiler: gcc - -script: - - make test -# it should be possible to import simplecpp.cpp into software that is compiled with old compilers - - g++ -fsyntax-only -std=c++98 simplecpp.cpp From f1086d2c8d2123b149d511f6db42a5795fc459ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 4 Apr 2021 12:39:38 +0200 Subject: [PATCH 120/381] Fixed macro expansion --- simplecpp.cpp | 11 ++++++++--- test.cpp | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7d48a715..ef2c1fa0 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1287,7 +1287,10 @@ namespace simplecpp { rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); rawtok = rawtok->next; } - if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros)) + bool first = true; + if (valueToken && valueToken->str() == rawtok1->str()) + first = false; + if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros, first)) rawtok = rawtok1->next; } else { rawtok = expand(&output2, rawtok->location, rawtok, macros, expandedmacros); @@ -1517,8 +1520,10 @@ namespace simplecpp { return sameline(lpar,tok) ? tok : NULL; } - const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const std::map ¯os, std::set expandedmacros) const { - expandedmacros.insert(nameTokInst->str()); + const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const std::map ¯os, std::set expandedmacros, bool first=false) const { + + if (!first) + expandedmacros.insert(nameTokInst->str()); usageList.push_back(loc); diff --git a/test.cpp b/test.cpp index 284f5bce..bdcc1765 100644 --- a/test.cpp +++ b/test.cpp @@ -516,6 +516,14 @@ static void define_define_16() // issue #72 with __VA_ARGS__ ASSERT_EQUALS("\n\n\n1 2", preprocess(code)); } +static void define_define_17() +{ + const char code[] = "#define Bar(x) x\n" + "#define Foo Bar(1)\n" + "Bar( Foo ) ;"; + ASSERT_EQUALS("\n\n1 ;", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -1999,6 +2007,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_14); TEST_CASE(define_define_15); TEST_CASE(define_define_16); + TEST_CASE(define_define_17); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From 7076ade40a170ae84fea3b5401fd9cf11854de9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 25 Apr 2021 20:59:32 +0200 Subject: [PATCH 121/381] Handle C++17 __has_include if __cplusplus is defined and greater than 201703L --- simplecpp.cpp | 41 ++++++++++++++++++++++++++++++++++++++--- test.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ef2c1fa0..8f609a7d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -70,6 +70,8 @@ static const simplecpp::TokenString ENDIF("endif"); static const simplecpp::TokenString PRAGMA("pragma"); static const simplecpp::TokenString ONCE("once"); +static const simplecpp::TokenString HAS_INCLUDE("__has_include"); + template static std::string toString(T t) { std::ostringstream ostr; @@ -2515,7 +2517,6 @@ std::map simplecpp::load(const simplecpp::To continue; bool systemheader = (htok->str()[0] == '<'); - const std::string header(realFilename(htok->str().substr(1U, htok->str().size() - 2U))); if (hasFile(ret, sourcefile, header, dui, systemheader)) continue; @@ -2585,6 +2586,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL sizeOfType.insert(std::make_pair("double *", sizeof(double *))); sizeOfType.insert(std::make_pair("long double *", sizeof(long double *))); + bool hasInclude = false; std::map macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; @@ -2597,6 +2599,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const std::string rhs(eq==std::string::npos ? std::string("1") : macrostr.substr(eq+1)); const Macro macro(lhs, rhs, files); macros.insert(std::pair(macro.name(), macro)); + if (lhs == "__cplusplus" && stringToLL(rhs) >= 201703) + hasInclude = true; } macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files))); @@ -2787,9 +2791,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (ifstates.top() == ALWAYS_FALSE || (ifstates.top() == ELSE_IS_TRUE && rawtok->str() != ELIF)) conditionIsTrue = false; else if (rawtok->str() == IFDEF) - conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end()); + conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end() || (hasInclude && rawtok->next->str() == HAS_INCLUDE)); else if (rawtok->str() == IFNDEF) - conditionIsTrue = (macros.find(rawtok->next->str()) == macros.end()); + conditionIsTrue = (macros.find(rawtok->next->str()) == macros.end() && !(hasInclude && rawtok->next->str() == HAS_INCLUDE)); else { /*if (rawtok->str() == IF || rawtok->str() == ELIF)*/ TokenList expr(files); for (const Token *tok = rawtok->next; tok && tok->location.sameline(rawtok->location); tok = tok->next) { @@ -2806,6 +2810,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (tok) { if (macros.find(tok->str()) != macros.end()) expr.push_back(new Token("1", tok->location)); + else if (hasInclude && tok->str() == HAS_INCLUDE) + expr.push_back(new Token("1", tok->location)); else expr.push_back(new Token("0", tok->location)); } @@ -2825,6 +2831,35 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } + if (hasInclude && tok->str() == HAS_INCLUDE) { + tok = tok->next; + const bool par = (tok && tok->op == '('); + if (par) + tok = tok->next; + if (tok) { + const std::string &sourcefile = rawtok->location.file(); + const bool systemheader = (tok->str()[0] == '<'); + const std::string header(realFilename(tok->str().substr(1U, tok->str().size() - 2U))); + std::ifstream f; + const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); + expr.push_back(new Token(header2.empty() ? "0" : "1", tok->location)); + } + if (par) + tok = tok ? tok->next : NULL; + if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { + if (outputList) { + Output out(rawtok->location.files); + out.type = Output::SYNTAX_ERROR; + out.location = rawtok->location; + out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; + outputList->push_back(out); + } + output.clear(); + return; + } + continue; + } + const Token *tmp = tok; if (!preprocessToken(expr, &tmp, macros, files, outputList)) { output.clear(); diff --git a/test.cpp b/test.cpp index bdcc1765..1536beb8 100644 --- a/test.cpp +++ b/test.cpp @@ -871,6 +871,36 @@ static void hashhash_invalid_2() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } +static void has_include_1() +{ + const char code[] = "#ifdef __has_include\n" + " #ifdef __has_include(\"simplecpp.h\")\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.defines.push_back("__cplusplus=201703L"); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + +static void has_include_2() +{ + const char code[] = "#if defined( __has_include)\n" + " #ifdef __has_include(\"simplecpp.h\")\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.defines.push_back("__cplusplus=201703L"); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + static void ifdef1() { const char code[] = "#ifdef A\n" @@ -2042,6 +2072,10 @@ int main(int argc, char **argv) TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); + // c++17 __has_include + TEST_CASE(has_include_1); + TEST_CASE(has_include_2); + TEST_CASE(ifdef1); TEST_CASE(ifdef2); TEST_CASE(ifndef); From e4d58abeaea16ae776d1e334e880534471c20da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 26 Apr 2021 16:24:38 +0200 Subject: [PATCH 122/381] Add -std command line option --- main.cpp | 6 +++++- simplecpp.cpp | 13 ++++++++++--- simplecpp.h | 3 ++- test.cpp | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/main.cpp b/main.cpp index 54124a0f..823ed9f7 100644 --- a/main.cpp +++ b/main.cpp @@ -15,7 +15,7 @@ int main(int argc, char **argv) const char *arg = argv[i]; if (*arg == '-') { char c = arg[1]; - if (c != 'D' && c != 'U' && c != 'I' && c != 'i') + if (c != 'D' && c != 'U' && c != 'I' && c != 'i' && c != 's') continue; // Ignored const char *value = arg[2] ? (argv[i] + 2) : argv[++i]; switch (c) { @@ -32,6 +32,10 @@ int main(int argc, char **argv) if (std::strncmp(arg, "-include=",9)==0) dui.includes.push_back(arg+9); break; + case 's': + if (std::strncmp(arg, "-std=",5)==0) + dui.std = arg + 5; + break; } } else { filename = arg; diff --git a/simplecpp.cpp b/simplecpp.cpp index 8f609a7d..d174cbac 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2586,7 +2586,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL sizeOfType.insert(std::make_pair("double *", sizeof(double *))); sizeOfType.insert(std::make_pair("long double *", sizeof(long double *))); - bool hasInclude = false; + const bool hasInclude = (dui.std.size() == 5 && dui.std.compare(0,3,"c++") == 0 && dui.std >= "c++17"); std::map macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; @@ -2599,14 +2599,21 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const std::string rhs(eq==std::string::npos ? std::string("1") : macrostr.substr(eq+1)); const Macro macro(lhs, rhs, files); macros.insert(std::pair(macro.name(), macro)); - if (lhs == "__cplusplus" && stringToLL(rhs) >= 201703) - hasInclude = true; } macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files))); macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", files))); macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files))); + if (dui.std == "c++11") + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201103L", files))); + else if (dui.std == "c++14") + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201402L", files))); + else if (dui.std == "c++17") + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201703L", files))); + else if (dui.std == "c++20") + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "202002L", files))); + // TRUE => code in current #if block should be kept // ELSE_IS_TRUE => code in current #if block should be dropped. the code in the #else should be kept. // ALWAYS_FALSE => drop all code in #if and #else diff --git a/simplecpp.h b/simplecpp.h index 577711af..5ca74a17 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -288,7 +288,7 @@ namespace simplecpp { /** * Command line preprocessor settings. - * On the command line these are configured by -D, -U, -I, --include + * On the command line these are configured by -D, -U, -I, --include, -std */ struct SIMPLECPP_LIB DUI { DUI() {} @@ -296,6 +296,7 @@ namespace simplecpp { std::set undefined; std::list includePaths; std::list includes; + std::string std; }; SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = NULL); diff --git a/test.cpp b/test.cpp index 1536beb8..e453ad79 100644 --- a/test.cpp +++ b/test.cpp @@ -881,7 +881,7 @@ static void has_include_1() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.defines.push_back("__cplusplus=201703L"); + dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } @@ -896,7 +896,7 @@ static void has_include_2() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.defines.push_back("__cplusplus=201703L"); + dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } From 39f369e36d8a1e3a98a436ce5405cc0008bab68e Mon Sep 17 00:00:00 2001 From: keinflue <80230456+keinflue@users.noreply.github.com> Date: Fri, 30 Apr 2021 15:40:10 +0000 Subject: [PATCH 123/381] Partial fix for character values with escape sequences - Issue 214 (#216) --- simplecpp.cpp | 190 +++++++++++++++++++++++++++++++++++++++++++++++++- simplecpp.h | 5 +- test.cpp | 62 ++++++++++++++++ 3 files changed, 254 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d174cbac..a854a435 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -23,6 +23,7 @@ #include "simplecpp.h" #include +#include #include #include #include @@ -2304,6 +2305,191 @@ static void simplifyName(simplecpp::TokenList &expr) } } +/* + * Reads at least minlen and at most maxlen digits (inc. prefix) in base base + * from s starting at position pos and converts them to a + * unsigned long long value, updating pos to point to the first + * unused element of s. + * Returns ULLONG_MAX if the result is not representable and + * throws if the above requirements were not possible to satisfy. + */ +static unsigned long long stringToULLbounded( + const std::string& s, + std::size_t& pos, + int base = 0, + std::ptrdiff_t minlen = 1, + std::size_t maxlen = std::string::npos +) { + std::string sub = s.substr(pos, maxlen); + const char* start = sub.c_str(); + char* end; + unsigned long long value = std::strtoull(start, &end, base); + pos += end - start; + if(end - start < minlen) + throw std::runtime_error("expected digit"); + return value; +} + +/* Converts character literal (including prefix, but not ud-suffix) + * to long long value. + * + * Assumes ASCII-compatible single-byte encoded str. + * + * For target assumes + * - UTF-8 execution character set encoding or encoding matching str + * - UTF-32 execution wide-character set encoding + * - requirements for __STDC_UTF_16__, __STDC_UTF_32__ and __STDC_ISO_10646__ satisfied + * - char16_t is 16bit wide + * - char32_t is 32bit wide + * - wchar_t is 32bit wide and unsigned + * - matching char signedness to host + * - matching sizeof(int) to host + * + * For host assumes + * - ASCII-compatible execution character set + * + * For host and target assumes + * - CHAR_BIT == 8 + * - two's complement + * + * Implements multi-character narrow literals according to GCC's behavior, + * except multi code unit universal character names are not supported. + * Multi-character wide literals are not supported. + * Limited support of universal character names for non-UTF-8 execution character set encodings. + */ +long long simplecpp::characterLiteralToLL(const std::string& str) +{ + // default is wide/utf32 + bool narrow = false; + bool utf8 = false; + bool utf16 = false; + + std::size_t pos; + + if(str.size() >= 1 && str[0] == '\'') { + narrow = true; + pos = 1; + } else if(str.size() >= 2 && str[0] == 'u' && str[1] == '\'') { + utf16 = true; + pos = 2; + } else if(str.size() >= 3 && str[0] == 'u' && str[1] == '8' && str[2] == '\'') { + utf8 = true; + pos = 3; + } else if(str.size() >= 2 && (str[0] == 'L' || str[0] == 'U') && str[1] == '\'') { + pos = 2; + } else + throw std::runtime_error("expected a character literal"); + + unsigned long long multivalue = 0; + + std::size_t nbytes = 0; + + while(pos + 1 < str.size()) { + if(str[pos] == '\'' || str[pos] == '\n') + throw std::runtime_error("raw single quotes and newlines not allowed in character literals"); + + if(nbytes >= 1 && !narrow) + throw std::runtime_error("multiple characters only supported in narrow character literals"); + + unsigned long long value; + + if(str[pos] == '\\') { + pos++; + char escape = str[pos++]; + + if(pos >= str.size()) + throw std::runtime_error("unexpected end of character literal"); + + switch(escape) { + case '\'': + case '"': + case '?': + case '\\': + value = static_cast(escape); + break; + + case 'a': value = static_cast('\a'); break; + case 'b': value = static_cast('\b'); break; + case 'f': value = static_cast('\f'); break; + case 'n': value = static_cast('\n'); break; + case 'r': value = static_cast('\r'); break; + case 't': value = static_cast('\t'); break; + case 'v': value = static_cast('\v'); break; + + // ESC extension + case 'e': value = static_cast('\x1b'); break; + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + // octal escape sequences consist of 1 to 3 digits + value = stringToULLbounded(str, --pos, 8, 1, 3); + break; + + case 'x': + // hexadecimal escape sequences consist of at least 1 digit + value = stringToULLbounded(str, pos, 16); + break; + + case 'u': + case 'U': { + // universal character names have exactly 4 or 8 digits + std::size_t ndigits = (escape == 'u' ? 4 : 8); + value = stringToULLbounded(str, pos, 16, ndigits, ndigits); + + // UTF-8 encodes code points above 0x7f in multiple code units + // code points above 0x10ffff are not allowed + if(((narrow || utf8) && value > 0x7f) || (utf16 && value > 0xffff) || value > 0x10ffff) + throw std::runtime_error("code point too large"); + + if(value >= 0xd800 && value <= 0xdfff) + throw std::runtime_error("surrogate code points not allowed in universal character names"); + + break; + } + + default: + throw std::runtime_error("invalid escape sequence"); + } + } else { + value = static_cast(str[pos++]); + + if(!narrow && value > 0x7f) + throw std::runtime_error("non-ASCII source characters supported only in narrow character literals"); + } + + if(((narrow || utf8) && value > std::numeric_limits::max()) || (utf16 && value >> 16) || value >> 32) + throw std::runtime_error("numeric escape sequence too large"); + + multivalue <<= CHAR_BIT; + multivalue |= value; + nbytes++; + } + + if(pos + 1 != str.size() || str[pos] != '\'') + throw std::runtime_error("missing closing quote in character literal"); + + if(!nbytes) + throw std::runtime_error("empty character literal"); + + // ordinary narrow character literal's value is determined by (possibly signed) char + if(narrow && nbytes == 1) + return static_cast(multivalue); + + // while multi-character literal's value is determined by (signed) int + if(narrow) + return static_cast(multivalue); + + // All other cases are unsigned. Since long long is at least 64bit wide, + // while the literals at most 32bit wide, the conversion preserves all values. + return multivalue; +} + static void simplifyNumbers(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { @@ -2311,8 +2497,8 @@ static void simplifyNumbers(simplecpp::TokenList &expr) continue; if (tok->str().compare(0,2,"0x") == 0) tok->setstr(toString(stringToULL(tok->str()))); - else if (tok->str()[0] == '\'') - tok->setstr(toString(tok->str()[1] & 0xffU)); + else if (!tok->number && tok->str().find('\'') != tok->str().npos) + tok->setstr(toString(simplecpp::characterLiteralToLL(tok->str()))); } } diff --git a/simplecpp.h b/simplecpp.h index 5ca74a17..8ddb92ee 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -107,7 +107,8 @@ namespace simplecpp { } void flags() { - name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$'); + name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$') + && (string.find('\'') == string.npos); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); op = (string.size() == 1U) ? string[0] : '\0'; @@ -298,6 +299,8 @@ namespace simplecpp { std::list includes; std::string std; }; + + SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = NULL); diff --git a/test.cpp b/test.cpp index e453ad79..c645ff57 100644 --- a/test.cpp +++ b/test.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include "simplecpp.h" @@ -146,6 +147,65 @@ static std::string testConstFold(const char code[]) return expr.stringify(); } +static void characterLiteral() { + ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("'A'")); + + ASSERT_EQUALS('\'', simplecpp::characterLiteralToLL("'\\''")); + ASSERT_EQUALS('\"', simplecpp::characterLiteralToLL("'\\\"'")); + ASSERT_EQUALS('\?', simplecpp::characterLiteralToLL("'\\?'")); + ASSERT_EQUALS('\\', simplecpp::characterLiteralToLL("'\\\\'")); + ASSERT_EQUALS('\a', simplecpp::characterLiteralToLL("'\\a'")); + ASSERT_EQUALS('\b', simplecpp::characterLiteralToLL("'\\b'")); + ASSERT_EQUALS('\f', simplecpp::characterLiteralToLL("'\\f'")); + ASSERT_EQUALS('\n', simplecpp::characterLiteralToLL("'\\n'")); + ASSERT_EQUALS('\r', simplecpp::characterLiteralToLL("'\\r'")); + ASSERT_EQUALS('\t', simplecpp::characterLiteralToLL("'\\t'")); + ASSERT_EQUALS('\v', simplecpp::characterLiteralToLL("'\\v'")); + + ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\e'")); + + ASSERT_EQUALS('\0', simplecpp::characterLiteralToLL("'\\0'")); + ASSERT_EQUALS('\1', simplecpp::characterLiteralToLL("'\\1'")); + ASSERT_EQUALS('\10', simplecpp::characterLiteralToLL("'\\10'")); + ASSERT_EQUALS('\010', simplecpp::characterLiteralToLL("'\\010'")); + ASSERT_EQUALS('\377', simplecpp::characterLiteralToLL("'\\377'")); + + ASSERT_EQUALS('\x0', simplecpp::characterLiteralToLL("'\\x0'")); + ASSERT_EQUALS('\x10', simplecpp::characterLiteralToLL("'\\x10'")); + ASSERT_EQUALS('\xff', simplecpp::characterLiteralToLL("'\\xff'")); + + ASSERT_EQUALS('\u0012', simplecpp::characterLiteralToLL("'\\u0012'")); + ASSERT_EQUALS('\U00000012', simplecpp::characterLiteralToLL("'\\U00000012'")); + + ASSERT_EQUALS(((unsigned int)(unsigned char)'b' << 8) | (unsigned char)'c', simplecpp::characterLiteralToLL("'bc'")); + ASSERT_EQUALS(((unsigned int)(unsigned char)'\x23' << 8) | (unsigned char)'\x45', simplecpp::characterLiteralToLL("'\\x23\\x45'")); + ASSERT_EQUALS(((unsigned int)(unsigned char)'\11' << 8) | (unsigned char)'\222', simplecpp::characterLiteralToLL("'\\11\\222'")); + ASSERT_EQUALS(((unsigned int)(unsigned char)'\a' << 8) | (unsigned char)'\b', simplecpp::characterLiteralToLL("'\\a\\b'")); + if(sizeof(int) <= 4) + ASSERT_EQUALS(-1, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); + else + ASSERT_EQUALS(0xffffffff, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); + + ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("u8'A'")); + ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("u'A'")); + ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("L'A'")); + ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("U'A'")); + + ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("u8'\\xff'")); + ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("u'\\xff'")); + ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("L'\\xff'")); + ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("U'\\xff'")); + + ASSERT_EQUALS(0xfedc, simplecpp::characterLiteralToLL("u'\\xfedc'")); + ASSERT_EQUALS(0xfedcba98, simplecpp::characterLiteralToLL("L'\\xfedcba98'")); + ASSERT_EQUALS(0xfedcba98, simplecpp::characterLiteralToLL("U'\\xfedcba98'")); + + ASSERT_EQUALS(0x12, simplecpp::characterLiteralToLL("u8'\\u0012'")); + ASSERT_EQUALS(0x1234, simplecpp::characterLiteralToLL("u'\\u1234'")); + ASSERT_EQUALS(0x00012345, simplecpp::characterLiteralToLL("L'\\U00012345'")); + ASSERT_EQUALS(0x00012345, simplecpp::characterLiteralToLL("U'\\U00012345'")); +} + static void combineOperators_floatliteral() { ASSERT_EQUALS("1.", preprocess("1.")); @@ -1995,6 +2055,8 @@ int main(int argc, char **argv) TEST_CASE(builtin); + TEST_CASE(characterLiteral); + TEST_CASE(combineOperators_floatliteral); TEST_CASE(combineOperators_increment); TEST_CASE(combineOperators_coloncolon); From 88e6551b96608231eb18216105499163f15d1f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 30 Apr 2021 17:41:30 +0200 Subject: [PATCH 124/381] astyle formatting --- simplecpp.cpp | 187 +++++++++++++++++++++++++++----------------------- simplecpp.h | 4 +- test.cpp | 19 ++--- 3 files changed, 114 insertions(+), 96 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a854a435..111ba4ba 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2307,25 +2307,26 @@ static void simplifyName(simplecpp::TokenList &expr) /* * Reads at least minlen and at most maxlen digits (inc. prefix) in base base - * from s starting at position pos and converts them to a + * from s starting at position pos and converts them to a * unsigned long long value, updating pos to point to the first * unused element of s. * Returns ULLONG_MAX if the result is not representable and * throws if the above requirements were not possible to satisfy. */ static unsigned long long stringToULLbounded( - const std::string& s, - std::size_t& pos, - int base = 0, - std::ptrdiff_t minlen = 1, - std::size_t maxlen = std::string::npos -) { + const std::string& s, + std::size_t& pos, + int base = 0, + std::ptrdiff_t minlen = 1, + std::size_t maxlen = std::string::npos +) +{ std::string sub = s.substr(pos, maxlen); const char* start = sub.c_str(); char* end; unsigned long long value = std::strtoull(start, &end, base); pos += end - start; - if(end - start < minlen) + if (end - start < minlen) throw std::runtime_error("expected digit"); return value; } @@ -2334,7 +2335,7 @@ static unsigned long long stringToULLbounded( * to long long value. * * Assumes ASCII-compatible single-byte encoded str. - * + * * For target assumes * - UTF-8 execution character set encoding or encoding matching str * - UTF-32 execution wide-character set encoding @@ -2363,19 +2364,19 @@ long long simplecpp::characterLiteralToLL(const std::string& str) bool narrow = false; bool utf8 = false; bool utf16 = false; - + std::size_t pos; - if(str.size() >= 1 && str[0] == '\'') { + if (str.size() >= 1 && str[0] == '\'') { narrow = true; pos = 1; - } else if(str.size() >= 2 && str[0] == 'u' && str[1] == '\'') { + } else if (str.size() >= 2 && str[0] == 'u' && str[1] == '\'') { utf16 = true; pos = 2; - } else if(str.size() >= 3 && str[0] == 'u' && str[1] == '8' && str[2] == '\'') { + } else if (str.size() >= 3 && str[0] == 'u' && str[1] == '8' && str[2] == '\'') { utf8 = true; pos = 3; - } else if(str.size() >= 2 && (str[0] == 'L' || str[0] == 'U') && str[1] == '\'') { + } else if (str.size() >= 2 && (str[0] == 'L' || str[0] == 'U') && str[1] == '\'') { pos = 2; } else throw std::runtime_error("expected a character literal"); @@ -2384,86 +2385,102 @@ long long simplecpp::characterLiteralToLL(const std::string& str) std::size_t nbytes = 0; - while(pos + 1 < str.size()) { - if(str[pos] == '\'' || str[pos] == '\n') + while (pos + 1 < str.size()) { + if (str[pos] == '\'' || str[pos] == '\n') throw std::runtime_error("raw single quotes and newlines not allowed in character literals"); - if(nbytes >= 1 && !narrow) + if (nbytes >= 1 && !narrow) throw std::runtime_error("multiple characters only supported in narrow character literals"); unsigned long long value; - - if(str[pos] == '\\') { + + if (str[pos] == '\\') { pos++; char escape = str[pos++]; - - if(pos >= str.size()) + + if (pos >= str.size()) throw std::runtime_error("unexpected end of character literal"); - switch(escape) { - case '\'': - case '"': - case '?': - case '\\': - value = static_cast(escape); - break; + switch (escape) { + case '\'': + case '"': + case '?': + case '\\': + value = static_cast(escape); + break; + + case 'a': + value = static_cast('\a'); + break; + case 'b': + value = static_cast('\b'); + break; + case 'f': + value = static_cast('\f'); + break; + case 'n': + value = static_cast('\n'); + break; + case 'r': + value = static_cast('\r'); + break; + case 't': + value = static_cast('\t'); + break; + case 'v': + value = static_cast('\v'); + break; - case 'a': value = static_cast('\a'); break; - case 'b': value = static_cast('\b'); break; - case 'f': value = static_cast('\f'); break; - case 'n': value = static_cast('\n'); break; - case 'r': value = static_cast('\r'); break; - case 't': value = static_cast('\t'); break; - case 'v': value = static_cast('\v'); break; - - // ESC extension - case 'e': value = static_cast('\x1b'); break; - - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - // octal escape sequences consist of 1 to 3 digits - value = stringToULLbounded(str, --pos, 8, 1, 3); - break; - - case 'x': - // hexadecimal escape sequences consist of at least 1 digit - value = stringToULLbounded(str, pos, 16); - break; - - case 'u': - case 'U': { - // universal character names have exactly 4 or 8 digits - std::size_t ndigits = (escape == 'u' ? 4 : 8); - value = stringToULLbounded(str, pos, 16, ndigits, ndigits); - - // UTF-8 encodes code points above 0x7f in multiple code units - // code points above 0x10ffff are not allowed - if(((narrow || utf8) && value > 0x7f) || (utf16 && value > 0xffff) || value > 0x10ffff) - throw std::runtime_error("code point too large"); - - if(value >= 0xd800 && value <= 0xdfff) - throw std::runtime_error("surrogate code points not allowed in universal character names"); - - break; - } - - default: - throw std::runtime_error("invalid escape sequence"); + // ESC extension + case 'e': + value = static_cast('\x1b'); + break; + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + // octal escape sequences consist of 1 to 3 digits + value = stringToULLbounded(str, --pos, 8, 1, 3); + break; + + case 'x': + // hexadecimal escape sequences consist of at least 1 digit + value = stringToULLbounded(str, pos, 16); + break; + + case 'u': + case 'U': { + // universal character names have exactly 4 or 8 digits + std::size_t ndigits = (escape == 'u' ? 4 : 8); + value = stringToULLbounded(str, pos, 16, ndigits, ndigits); + + // UTF-8 encodes code points above 0x7f in multiple code units + // code points above 0x10ffff are not allowed + if (((narrow || utf8) && value > 0x7f) || (utf16 && value > 0xffff) || value > 0x10ffff) + throw std::runtime_error("code point too large"); + + if (value >= 0xd800 && value <= 0xdfff) + throw std::runtime_error("surrogate code points not allowed in universal character names"); + + break; + } + + default: + throw std::runtime_error("invalid escape sequence"); } } else { value = static_cast(str[pos++]); - if(!narrow && value > 0x7f) + if (!narrow && value > 0x7f) throw std::runtime_error("non-ASCII source characters supported only in narrow character literals"); } - if(((narrow || utf8) && value > std::numeric_limits::max()) || (utf16 && value >> 16) || value >> 32) + if (((narrow || utf8) && value > std::numeric_limits::max()) || (utf16 && value >> 16) || value >> 32) throw std::runtime_error("numeric escape sequence too large"); multivalue <<= CHAR_BIT; @@ -2471,20 +2488,20 @@ long long simplecpp::characterLiteralToLL(const std::string& str) nbytes++; } - if(pos + 1 != str.size() || str[pos] != '\'') + if (pos + 1 != str.size() || str[pos] != '\'') throw std::runtime_error("missing closing quote in character literal"); - - if(!nbytes) + + if (!nbytes) throw std::runtime_error("empty character literal"); - + // ordinary narrow character literal's value is determined by (possibly signed) char - if(narrow && nbytes == 1) + if (narrow && nbytes == 1) return static_cast(multivalue); - + // while multi-character literal's value is determined by (signed) int - if(narrow) + if (narrow) return static_cast(multivalue); - + // All other cases are unsigned. Since long long is at least 64bit wide, // while the literals at most 32bit wide, the conversion preserves all values. return multivalue; diff --git a/simplecpp.h b/simplecpp.h index 8ddb92ee..80d025d6 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -108,7 +108,7 @@ namespace simplecpp { void flags() { name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$') - && (string.find('\'') == string.npos); + && (string.find('\'') == string.npos); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); op = (string.size() == 1U) ? string[0] : '\0'; @@ -299,7 +299,7 @@ namespace simplecpp { std::list includes; std::string std; }; - + SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = NULL); diff --git a/test.cpp b/test.cpp index c645ff57..86cf8a6c 100644 --- a/test.cpp +++ b/test.cpp @@ -147,9 +147,10 @@ static std::string testConstFold(const char code[]) return expr.stringify(); } -static void characterLiteral() { +static void characterLiteral() +{ ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("'A'")); - + ASSERT_EQUALS('\'', simplecpp::characterLiteralToLL("'\\''")); ASSERT_EQUALS('\"', simplecpp::characterLiteralToLL("'\\\"'")); ASSERT_EQUALS('\?', simplecpp::characterLiteralToLL("'\\?'")); @@ -161,9 +162,9 @@ static void characterLiteral() { ASSERT_EQUALS('\r', simplecpp::characterLiteralToLL("'\\r'")); ASSERT_EQUALS('\t', simplecpp::characterLiteralToLL("'\\t'")); ASSERT_EQUALS('\v', simplecpp::characterLiteralToLL("'\\v'")); - + ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\e'")); - + ASSERT_EQUALS('\0', simplecpp::characterLiteralToLL("'\\0'")); ASSERT_EQUALS('\1', simplecpp::characterLiteralToLL("'\\1'")); ASSERT_EQUALS('\10', simplecpp::characterLiteralToLL("'\\10'")); @@ -173,7 +174,7 @@ static void characterLiteral() { ASSERT_EQUALS('\x0', simplecpp::characterLiteralToLL("'\\x0'")); ASSERT_EQUALS('\x10', simplecpp::characterLiteralToLL("'\\x10'")); ASSERT_EQUALS('\xff', simplecpp::characterLiteralToLL("'\\xff'")); - + ASSERT_EQUALS('\u0012', simplecpp::characterLiteralToLL("'\\u0012'")); ASSERT_EQUALS('\U00000012', simplecpp::characterLiteralToLL("'\\U00000012'")); @@ -181,7 +182,7 @@ static void characterLiteral() { ASSERT_EQUALS(((unsigned int)(unsigned char)'\x23' << 8) | (unsigned char)'\x45', simplecpp::characterLiteralToLL("'\\x23\\x45'")); ASSERT_EQUALS(((unsigned int)(unsigned char)'\11' << 8) | (unsigned char)'\222', simplecpp::characterLiteralToLL("'\\11\\222'")); ASSERT_EQUALS(((unsigned int)(unsigned char)'\a' << 8) | (unsigned char)'\b', simplecpp::characterLiteralToLL("'\\a\\b'")); - if(sizeof(int) <= 4) + if (sizeof(int) <= 4) ASSERT_EQUALS(-1, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); else ASSERT_EQUALS(0xffffffff, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); @@ -190,16 +191,16 @@ static void characterLiteral() { ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("u'A'")); ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("L'A'")); ASSERT_EQUALS('A', simplecpp::characterLiteralToLL("U'A'")); - + ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("u8'\\xff'")); ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("u'\\xff'")); ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("L'\\xff'")); ASSERT_EQUALS(0xff, simplecpp::characterLiteralToLL("U'\\xff'")); - + ASSERT_EQUALS(0xfedc, simplecpp::characterLiteralToLL("u'\\xfedc'")); ASSERT_EQUALS(0xfedcba98, simplecpp::characterLiteralToLL("L'\\xfedcba98'")); ASSERT_EQUALS(0xfedcba98, simplecpp::characterLiteralToLL("U'\\xfedcba98'")); - + ASSERT_EQUALS(0x12, simplecpp::characterLiteralToLL("u8'\\u0012'")); ASSERT_EQUALS(0x1234, simplecpp::characterLiteralToLL("u'\\u1234'")); ASSERT_EQUALS(0x00012345, simplecpp::characterLiteralToLL("L'\\U00012345'")); From 476ce139fa428c13ab49a91c20de82f3daa77f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 1 May 2021 18:13:02 +0200 Subject: [PATCH 125/381] Improved testing --- Makefile | 2 +- test.cpp | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b6865ef1..3251611e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -std=c++0x -g +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wno-multichar -std=c++0x -g LDFLAGS = -g %.o: %.cpp simplecpp.h diff --git a/test.cpp b/test.cpp index 86cf8a6c..aa341c6a 100644 --- a/test.cpp +++ b/test.cpp @@ -8,6 +8,7 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) +#define ASSERT_THROW(stmt, e) try { stmt; assertThrowFailed(__LINE__); } catch (const e&) {} static int assertEquals(const std::string &expected, const std::string &actual, int line) { @@ -26,6 +27,14 @@ static int assertEquals(const unsigned int &expected, const unsigned int &actual return assertEquals(std::to_string(expected), std::to_string(actual), line); } +static void assertThrowFailed(int line) +{ + numberOfFailedAssertions++; + std::cerr << "------ assertion failed ---------" << std::endl; + std::cerr << "line " << line << std::endl; + std::cerr << "exception not thrown" << std::endl; +} + static void testcase(const std::string &name, void (*f)(), int argc, char **argv) { if (argc == 1) @@ -170,6 +179,8 @@ static void characterLiteral() ASSERT_EQUALS('\10', simplecpp::characterLiteralToLL("'\\10'")); ASSERT_EQUALS('\010', simplecpp::characterLiteralToLL("'\\010'")); ASSERT_EQUALS('\377', simplecpp::characterLiteralToLL("'\\377'")); + ASSERT_EQUALS('\134t', simplecpp::characterLiteralToLL("'\\134t'")); // cppcheck ticket #7452 + ASSERT_EQUALS('\x0', simplecpp::characterLiteralToLL("'\\x0'")); ASSERT_EQUALS('\x10', simplecpp::characterLiteralToLL("'\\x10'")); @@ -205,6 +216,16 @@ static void characterLiteral() ASSERT_EQUALS(0x1234, simplecpp::characterLiteralToLL("u'\\u1234'")); ASSERT_EQUALS(0x00012345, simplecpp::characterLiteralToLL("L'\\U00012345'")); ASSERT_EQUALS(0x00012345, simplecpp::characterLiteralToLL("U'\\U00012345'")); + +#ifdef __GNUC__ + // BEGIN Implementation-specific results + ASSERT_EQUALS((int)('AB'), simplecpp::characterLiteralToLL("'AB'")); + ASSERT_EQUALS((int)('ABC'), simplecpp::characterLiteralToLL("'ABC'")); + ASSERT_EQUALS((int)('ABCD'), simplecpp::characterLiteralToLL("'ABCD'")); + // END Implementation-specific results +#endif + + ASSERT_THROW(simplecpp::characterLiteralToLL("'\\9'"), std::runtime_error); } static void combineOperators_floatliteral() From 6e4a3482b123bc516c24c1c4428fda3ac6453269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 2 May 2021 08:52:05 +0200 Subject: [PATCH 126/381] Moved implementation defined characterLiteralToLL test --- test.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index aa341c6a..0b4c7b8d 100644 --- a/test.cpp +++ b/test.cpp @@ -179,8 +179,6 @@ static void characterLiteral() ASSERT_EQUALS('\10', simplecpp::characterLiteralToLL("'\\10'")); ASSERT_EQUALS('\010', simplecpp::characterLiteralToLL("'\\010'")); ASSERT_EQUALS('\377', simplecpp::characterLiteralToLL("'\\377'")); - ASSERT_EQUALS('\134t', simplecpp::characterLiteralToLL("'\\134t'")); // cppcheck ticket #7452 - ASSERT_EQUALS('\x0', simplecpp::characterLiteralToLL("'\\x0'")); ASSERT_EQUALS('\x10', simplecpp::characterLiteralToLL("'\\x10'")); @@ -222,6 +220,7 @@ static void characterLiteral() ASSERT_EQUALS((int)('AB'), simplecpp::characterLiteralToLL("'AB'")); ASSERT_EQUALS((int)('ABC'), simplecpp::characterLiteralToLL("'ABC'")); ASSERT_EQUALS((int)('ABCD'), simplecpp::characterLiteralToLL("'ABCD'")); + ASSERT_EQUALS('\134t', simplecpp::characterLiteralToLL("'\\134t'")); // cppcheck ticket #7452 // END Implementation-specific results #endif From 905efb473abcbb1107011756344139c75bf87c4d Mon Sep 17 00:00:00 2001 From: keinflue <80230456+keinflue@users.noreply.github.com> Date: Fri, 7 May 2021 07:28:27 +0000 Subject: [PATCH 127/381] Support some more GCC extensions for character literals (#219) --- simplecpp.cpp | 9 ++++++++- test.cpp | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 111ba4ba..a636f2c5 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2402,6 +2402,12 @@ long long simplecpp::characterLiteralToLL(const std::string& str) throw std::runtime_error("unexpected end of character literal"); switch (escape) { + // obscure GCC extensions + case '%': + case '(': + case '[': + case '{': + // standard escape sequences case '\'': case '"': case '?': @@ -2431,8 +2437,9 @@ long long simplecpp::characterLiteralToLL(const std::string& str) value = static_cast('\v'); break; - // ESC extension + // GCC extension for ESC character case 'e': + case 'E': value = static_cast('\x1b'); break; diff --git a/test.cpp b/test.cpp index 0b4c7b8d..bee17cca 100644 --- a/test.cpp +++ b/test.cpp @@ -172,7 +172,15 @@ static void characterLiteral() ASSERT_EQUALS('\t', simplecpp::characterLiteralToLL("'\\t'")); ASSERT_EQUALS('\v', simplecpp::characterLiteralToLL("'\\v'")); + // GCC extension for ESC character ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\e'")); + ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\E'")); + + // more obscure GCC extensions + ASSERT_EQUALS('(', simplecpp::characterLiteralToLL("'\\('")); + ASSERT_EQUALS('[', simplecpp::characterLiteralToLL("'\\['")); + ASSERT_EQUALS('{', simplecpp::characterLiteralToLL("'\\{'")); + ASSERT_EQUALS('%', simplecpp::characterLiteralToLL("'\\%'")); ASSERT_EQUALS('\0', simplecpp::characterLiteralToLL("'\\0'")); ASSERT_EQUALS('\1', simplecpp::characterLiteralToLL("'\\1'")); From f99711907eb683b15b132e629cd4a03169845f7c Mon Sep 17 00:00:00 2001 From: keinflue <80230456+keinflue@users.noreply.github.com> Date: Sat, 8 May 2021 18:51:03 +0000 Subject: [PATCH 128/381] Basic support for UTF-8 source in character literals (#220) --- simplecpp.cpp | 40 ++++++++++++++++++++++++++++---- test.cpp | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a636f2c5..9c7817ee 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2334,10 +2334,11 @@ static unsigned long long stringToULLbounded( /* Converts character literal (including prefix, but not ud-suffix) * to long long value. * - * Assumes ASCII-compatible single-byte encoded str. + * Assumes ASCII-compatible single-byte encoded str for narrow literals + * and UTF-8 otherwise. * * For target assumes - * - UTF-8 execution character set encoding or encoding matching str + * - execution character set encoding matching str * - UTF-32 execution wide-character set encoding * - requirements for __STDC_UTF_16__, __STDC_UTF_32__ and __STDC_ISO_10646__ satisfied * - char16_t is 16bit wide @@ -2483,8 +2484,39 @@ long long simplecpp::characterLiteralToLL(const std::string& str) } else { value = static_cast(str[pos++]); - if (!narrow && value > 0x7f) - throw std::runtime_error("non-ASCII source characters supported only in narrow character literals"); + if (!narrow && value >= 0x80) { + // Assuming this is a UTF-8 encoded code point. + // This decoder does not completely validate the input, for example it doesn't reject overlong encodings. + + int additional_bytes; + if (value >= 0xf8) + throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); + else if (value >= 0xf0) + additional_bytes = 3; + else if (value >= 0xe0) + additional_bytes = 2; + else if (value >= 0xc0) + additional_bytes = 1; + else + throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); + + value &= (1 << (6 - additional_bytes)) - 1; + + while (additional_bytes--) { + if(pos + 1 >= str.size()) + throw std::runtime_error("assumed UTF-8 encoded source, but character literal ends unexpectedly"); + unsigned char c = str[pos++]; + if((c >> 6) != 2) // ensure c has form 0xb10xxxxxx + throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); + value = (value << 6) | (c & ((1 << 7) - 1)); + } + + if (value >= 0xd800 && value <= 0xdfff) + throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); + + if ((utf8 && value > 0x7f) || (utf16 && value > 0xffff) || value > 0x10ffff) + throw std::runtime_error("code point too large"); + } } if (((narrow || utf8) && value > std::numeric_limits::max()) || (utf16 && value >> 16) || value >> 32) diff --git a/test.cpp b/test.cpp index bee17cca..034bfff9 100644 --- a/test.cpp +++ b/test.cpp @@ -233,6 +233,69 @@ static void characterLiteral() #endif ASSERT_THROW(simplecpp::characterLiteralToLL("'\\9'"), std::runtime_error); + + // Input is manually encoded to (escaped) UTF-8 byte sequences + // to avoid dependence on source encoding used for this file + ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("U'\302\265'")); + ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("U'\305\227'")); + ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("U'\357\274\217'")); + ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("U'\343\201\202'")); + ASSERT_EQUALS(0x13000, simplecpp::characterLiteralToLL("U'\360\223\200\200'")); + + ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("L'\302\265'")); + ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("L'\305\227'")); + ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("L'\357\274\217'")); + ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("L'\343\201\202'")); + ASSERT_EQUALS(0x13000, simplecpp::characterLiteralToLL("L'\360\223\200\200'")); + + ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("u'\302\265'")); + ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("u'\305\227'")); + ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("u'\357\274\217'")); + ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("u'\343\201\202'")); + ASSERT_THROW(simplecpp::characterLiteralToLL("u'\360\223\200\200'"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\302\265'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\305\227'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\357\274\217'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\343\201\202'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\360\223\200\200'"), std::runtime_error); + + ASSERT_EQUALS('\x89', simplecpp::characterLiteralToLL("'\x89'")); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error); + + // following examples based on https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + ASSERT_EQUALS(0x80, simplecpp::characterLiteralToLL("U'\xc2\x80'")); + ASSERT_EQUALS(0x800, simplecpp::characterLiteralToLL("U'\xe0\xa0\x80'")); + ASSERT_EQUALS(0x10000, simplecpp::characterLiteralToLL("U'\xf0\x90\x80\x80'")); + + ASSERT_EQUALS(0x7f, simplecpp::characterLiteralToLL("U'\x7f'")); + ASSERT_EQUALS(0x7ff, simplecpp::characterLiteralToLL("U'\xdf\xbf'")); + ASSERT_EQUALS(0xffff, simplecpp::characterLiteralToLL("U'\xef\xbf\xbf'")); + + ASSERT_EQUALS(0xd7ff, simplecpp::characterLiteralToLL("U'\xed\x9f\xbf'")); + ASSERT_EQUALS(0xe000, simplecpp::characterLiteralToLL("U'\xee\x80\x80'")); + ASSERT_EQUALS(0xfffd, simplecpp::characterLiteralToLL("U'\xef\xbf\xbd'")); + ASSERT_EQUALS(0x10ffff, simplecpp::characterLiteralToLL("U'\xf4\x8f\xbf\xbf'")); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f\x8f'"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f\x8f'"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0 '"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x8f '"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f '"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf8'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xff'"), std::runtime_error); } static void combineOperators_floatliteral() From cce3632095bf787fd232f2f2f13a486f92e3c638 Mon Sep 17 00:00:00 2001 From: keinflue <80230456+keinflue@users.noreply.github.com> Date: Sun, 9 May 2021 15:02:57 +0000 Subject: [PATCH 129/381] Improve validation of UTF-8 encoded character literals (#221) --- simplecpp.cpp | 14 ++++++++++---- test.cpp | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9c7817ee..62959e0c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2486,16 +2486,17 @@ long long simplecpp::characterLiteralToLL(const std::string& str) if (!narrow && value >= 0x80) { // Assuming this is a UTF-8 encoded code point. - // This decoder does not completely validate the input, for example it doesn't reject overlong encodings. + // This decoder may not completely validate the input. + // Noncharacters are neither rejected nor replaced. int additional_bytes; - if (value >= 0xf8) + if (value >= 0xf5) // higher values would result in code points above 0x10ffff throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); else if (value >= 0xf0) additional_bytes = 3; else if (value >= 0xe0) additional_bytes = 2; - else if (value >= 0xc0) + else if (value >= 0xc2) // 0xc0 and 0xc1 are always overlong 2-bytes encodings additional_bytes = 1; else throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); @@ -2505,9 +2506,14 @@ long long simplecpp::characterLiteralToLL(const std::string& str) while (additional_bytes--) { if(pos + 1 >= str.size()) throw std::runtime_error("assumed UTF-8 encoded source, but character literal ends unexpectedly"); + unsigned char c = str[pos++]; - if((c >> 6) != 2) // ensure c has form 0xb10xxxxxx + + if (((c >> 6) != 2) // ensure c has form 0xb10xxxxxx + || (!value && additional_bytes == 1 && c < 0xa0) // overlong 3-bytes encoding + || (!value && additional_bytes == 2 && c < 0x90)) // overlong 4-bytes encoding throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); + value = (value << 6) | (c & ((1 << 7) - 1)); } diff --git a/test.cpp b/test.cpp index 034bfff9..e27f1a5b 100644 --- a/test.cpp +++ b/test.cpp @@ -263,6 +263,8 @@ static void characterLiteral() ASSERT_EQUALS('\x89', simplecpp::characterLiteralToLL("'\x89'")); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf4\x90\x80\x80'"), std::runtime_error); + // following examples based on https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt ASSERT_EQUALS(0x80, simplecpp::characterLiteralToLL("U'\xc2\x80'")); ASSERT_EQUALS(0x800, simplecpp::characterLiteralToLL("U'\xe0\xa0\x80'")); @@ -296,6 +298,19 @@ static void characterLiteral() ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf8'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xff'"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0\xaf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x80\xaf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\xaf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc1\xbf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x9f\xbf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\xbf\xbf'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0\x80'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x80\x80'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\x80'"), std::runtime_error); + + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xed\xa0\x80'"), std::runtime_error); + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xed\xbf\xbf'"), std::runtime_error); } static void combineOperators_floatliteral() From da202c30468f1110fd48ccec4c1994b41a45632e Mon Sep 17 00:00:00 2001 From: keinflue <80230456+keinflue@users.noreply.github.com> Date: Thu, 13 May 2021 05:41:51 +0000 Subject: [PATCH 130/381] Fix ## with multiple tokens in macro argument. (#222) * Fix ## with multiple tokens in macro argument. Only the last token in the argument token sequence substituted in front of a ## should be concatenated. Previously the argument token sequence was interpreted as a single token and concatenated. * Add test case from issue #202 for PR #222. * Fix comment wording for PR #222. Co-authored-by: keinflue <> --- simplecpp.cpp | 26 ++++++++------------------ test.cpp | 12 ++++++++++++ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 62959e0c..cf1a74d4 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1609,7 +1609,14 @@ namespace simplecpp { if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { if (!sameline(tok, tok->next->next->next)) throw invalidHashHash(tok->location, name()); - output->push_back(newMacroToken(expandArgStr(tok, parametertokens2), loc, isReplaced(expandedmacros))); + TokenList new_output(files); + if (!expandArg(&new_output, tok, parametertokens2)) + output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros))); + else if (new_output.empty()) // placemarker token + output->push_back(newMacroToken("", loc, isReplaced(expandedmacros))); + else + for (const Token *tok2 = new_output.cfront(); tok2; tok2 = tok2->next) + output->push_back(newMacroToken(tok2->str(), loc, isReplaced(expandedmacros))); tok = tok->next; } else { tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens2); @@ -1811,23 +1818,6 @@ namespace simplecpp { return true; } - /** - * Get string for token. If token is argument, the expanded string is returned. - * @param tok The token - * @param parametertokens parameters given when expanding this macro - * @return string - */ - std::string expandArgStr(const Token *tok, const std::vector ¶metertokens) const { - TokenList tokens(files); - if (expandArg(&tokens, tok, parametertokens)) { - std::string s; - for (const Token *tok2 = tokens.cfront(); tok2; tok2 = tok2->next) - s += tok2->str(); - return s; - } - return tok->str(); - } - /** * Expand #X => "X" * @param output destination tokenlist diff --git a/test.cpp b/test.cpp index e27f1a5b..281b4d18 100644 --- a/test.cpp +++ b/test.cpp @@ -1016,6 +1016,17 @@ static void hashhash12() } } +static void hashhash13() +{ + const char code[] = "#define X(x) x##U\n" + "X((1<<1)-1)"; + ASSERT_EQUALS("\n( 1 << 1 ) - 1U", preprocess(code)); + + const char code2[] = "#define CONCAT(x, y) x##y\n" + "CONCAT(&a, b)"; + ASSERT_EQUALS("\n& ab", preprocess(code2)); +} + static void hashhash_invalid_1() { std::istringstream istr("#define f(a) (##x)\nf(1)"); @@ -2238,6 +2249,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash10); // #108 : #define x # # TEST_CASE(hashhash11); // #60: #define x # # # TEST_CASE(hashhash12); + TEST_CASE(hashhash13); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); From 326e4eb12693a4401cd4bab034165d089d2a1096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 28 Jun 2021 19:45:23 +0200 Subject: [PATCH 131/381] Set Token::macro in nested macros --- simplecpp.cpp | 6 +++++- test.cpp | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index cf1a74d4..2e8ce9a9 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1504,8 +1504,11 @@ namespace simplecpp { expanded = true; } } - if (!expanded) + if (!expanded) { tokens->push_back(new Token(*tok)); + if (tok->macro.empty() && (par > 0 || tok->str() != "(")) + tokens->back()->macro = name(); + } } if (tok->op == '(') @@ -1812,6 +1815,7 @@ namespace simplecpp { partok = it->second.expand(output, loc, partok, macros, expandedmacros); else { output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros))); + output->back()->macro = partok->macro; partok = partok->next; } } diff --git a/test.cpp b/test.cpp index 281b4d18..3d1a5d28 100644 --- a/test.cpp +++ b/test.cpp @@ -2007,6 +2007,22 @@ static void tokenMacro4() ASSERT_EQUALS("A", tok->macro); } +static void tokenMacro5() +{ + const char code[] = "#define SET_BPF(code) (code)\n" + "#define SET_BPF_JUMP(code) SET_BPF(D | code)\n" + "SET_BPF_JUMP(A | B | C);"; + const simplecpp::DUI dui; + std::vector files; + std::map filedata; + std::istringstream istr(code); + simplecpp::TokenList tokenList(files); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + const simplecpp::Token *tok = tokenList.cfront()->next; + ASSERT_EQUALS("D", tok->str()); + ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); +} + static void undef() { std::istringstream istr("#define A\n" @@ -2327,6 +2343,7 @@ int main(int argc, char **argv) TEST_CASE(tokenMacro2); TEST_CASE(tokenMacro3); TEST_CASE(tokenMacro4); + TEST_CASE(tokenMacro5); TEST_CASE(undef); From 0a557cc377a52510d68475116db67d50c10caac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jul 2021 20:26:45 +0200 Subject: [PATCH 132/381] astyle formatting --- simplecpp.cpp | 14 +++++++------- test.cpp | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2e8ce9a9..7e9848f7 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2482,7 +2482,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) // Assuming this is a UTF-8 encoded code point. // This decoder may not completely validate the input. // Noncharacters are neither rejected nor replaced. - + int additional_bytes; if (value >= 0xf5) // higher values would result in code points above 0x10ffff throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); @@ -2494,26 +2494,26 @@ long long simplecpp::characterLiteralToLL(const std::string& str) additional_bytes = 1; else throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); - + value &= (1 << (6 - additional_bytes)) - 1; while (additional_bytes--) { - if(pos + 1 >= str.size()) + if (pos + 1 >= str.size()) throw std::runtime_error("assumed UTF-8 encoded source, but character literal ends unexpectedly"); - + unsigned char c = str[pos++]; - + if (((c >> 6) != 2) // ensure c has form 0xb10xxxxxx || (!value && additional_bytes == 1 && c < 0xa0) // overlong 3-bytes encoding || (!value && additional_bytes == 2 && c < 0x90)) // overlong 4-bytes encoding throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); - + value = (value << 6) | (c & ((1 << 7) - 1)); } if (value >= 0xd800 && value <= 0xdfff) throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); - + if ((utf8 && value > 0x7f) || (utf16 && value > 0xffff) || value > 0x10ffff) throw std::runtime_error("code point too large"); } diff --git a/test.cpp b/test.cpp index 3d1a5d28..9404e655 100644 --- a/test.cpp +++ b/test.cpp @@ -175,7 +175,7 @@ static void characterLiteral() // GCC extension for ESC character ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\e'")); ASSERT_EQUALS(0x1b, simplecpp::characterLiteralToLL("'\\E'")); - + // more obscure GCC extensions ASSERT_EQUALS('(', simplecpp::characterLiteralToLL("'\\('")); ASSERT_EQUALS('[', simplecpp::characterLiteralToLL("'\\['")); @@ -233,7 +233,7 @@ static void characterLiteral() #endif ASSERT_THROW(simplecpp::characterLiteralToLL("'\\9'"), std::runtime_error); - + // Input is manually encoded to (escaped) UTF-8 byte sequences // to avoid dependence on source encoding used for this file ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("U'\302\265'")); @@ -241,25 +241,25 @@ static void characterLiteral() ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("U'\357\274\217'")); ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("U'\343\201\202'")); ASSERT_EQUALS(0x13000, simplecpp::characterLiteralToLL("U'\360\223\200\200'")); - + ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("L'\302\265'")); ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("L'\305\227'")); ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("L'\357\274\217'")); ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("L'\343\201\202'")); ASSERT_EQUALS(0x13000, simplecpp::characterLiteralToLL("L'\360\223\200\200'")); - + ASSERT_EQUALS(0xb5, simplecpp::characterLiteralToLL("u'\302\265'")); ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("u'\305\227'")); ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("u'\357\274\217'")); ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("u'\343\201\202'")); ASSERT_THROW(simplecpp::characterLiteralToLL("u'\360\223\200\200'"), std::runtime_error); - + ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\302\265'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\305\227'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\357\274\217'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\343\201\202'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\360\223\200\200'"), std::runtime_error); - + ASSERT_EQUALS('\x89', simplecpp::characterLiteralToLL("'\x89'")); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error); @@ -269,7 +269,7 @@ static void characterLiteral() ASSERT_EQUALS(0x80, simplecpp::characterLiteralToLL("U'\xc2\x80'")); ASSERT_EQUALS(0x800, simplecpp::characterLiteralToLL("U'\xe0\xa0\x80'")); ASSERT_EQUALS(0x10000, simplecpp::characterLiteralToLL("U'\xf0\x90\x80\x80'")); - + ASSERT_EQUALS(0x7f, simplecpp::characterLiteralToLL("U'\x7f'")); ASSERT_EQUALS(0x7ff, simplecpp::characterLiteralToLL("U'\xdf\xbf'")); ASSERT_EQUALS(0xffff, simplecpp::characterLiteralToLL("U'\xef\xbf\xbf'")); @@ -283,7 +283,7 @@ static void characterLiteral() ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f\x8f'"), std::runtime_error); - + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f'"), std::runtime_error); @@ -295,10 +295,10 @@ static void characterLiteral() ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x8f '"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f '"), std::runtime_error); - + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf8'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xff'"), std::runtime_error); - + ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0\xaf'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x80\xaf'"), std::runtime_error); ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\xaf'"), std::runtime_error); @@ -1021,9 +1021,9 @@ static void hashhash13() const char code[] = "#define X(x) x##U\n" "X((1<<1)-1)"; ASSERT_EQUALS("\n( 1 << 1 ) - 1U", preprocess(code)); - + const char code2[] = "#define CONCAT(x, y) x##y\n" - "CONCAT(&a, b)"; + "CONCAT(&a, b)"; ASSERT_EQUALS("\n& ab", preprocess(code2)); } From 12d940e17f08ffae18ac5ea2801d3acec6a30d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 21 Jul 2021 20:27:54 +0200 Subject: [PATCH 133/381] Tracking #if/#elif better --- simplecpp.cpp | 14 ++++++++++++-- simplecpp.h | 11 ++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7e9848f7..52a82f90 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2804,7 +2804,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token return true; } -void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage) +void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage, std::list *ifCond) { std::map sizeOfType(rawtokens.sizeOfType); sizeOfType.insert(std::make_pair("char", sizeof(char))); @@ -3119,7 +3119,17 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tmp->previous; } try { - conditionIsTrue = (evaluate(expr, sizeOfType) != 0); + if (ifCond) { + std::string E; + for (const simplecpp::Token *tok = expr.cfront(); tok; tok = tok->next) + E += (E.empty() ? "" : " ") + tok->str(); + const long long result = evaluate(expr, sizeOfType); + conditionIsTrue = (result != 0); + ifCond->push_back(IfCond(rawtok->location, E, result)); + } else { + const long long result = evaluate(expr, sizeOfType); + conditionIsTrue = (result != 0); + } } catch (const std::exception &e) { if (outputList) { Output out(rawtok->location.files); diff --git a/simplecpp.h b/simplecpp.h index 80d025d6..82adad40 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -287,6 +287,14 @@ namespace simplecpp { bool macroValueKnown; }; + /** Tracking #if/#elif expressions */ + struct SIMPLECPP_LIB IfCond { + explicit IfCond(const Location& location, const std::string &E, long long result) : location(location), E(E), result(result) {} + Location location; // location of #if/#elif + std::string E; // preprocessed condition + long long result; // condition result + }; + /** * Command line preprocessor settings. * On the command line these are configured by -D, -U, -I, --include, -std @@ -314,8 +322,9 @@ namespace simplecpp { * @param dui defines, undefs, and include paths * @param outputList output: list that will receive output messages * @param macroUsage output: macro usage + * @param ifCond output: #if/#elif expressions */ - SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = NULL, std::list *macroUsage = NULL); + SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = NULL, std::list *macroUsage = NULL, std::list *ifCond = NULL); /** * Deallocate data From 139c660ed9f3b111061a25305a5f8597cfa3212b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 6 Nov 2021 09:11:22 +0100 Subject: [PATCH 134/381] Fix handling of #line when there are comments --- Makefile | 2 +- simplecpp.cpp | 20 ++++++++++++++++---- test.cpp | 26 ++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3251611e..f02aee72 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wno-multichar -std=c++0x -g +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -std=c++0x -g LDFLAGS = -g %.o: %.cpp simplecpp.h diff --git a/simplecpp.cpp b/simplecpp.cpp index 52a82f90..6445b093 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -501,14 +501,26 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen oldLastToken = cback(); const std::string lastline(lastLine()); if (lastline == "# file %str%") { + const Token *strtok = cback(); + while (strtok->comment) + strtok = strtok->previous; loc.push(location); - location.fileIndex = fileIndex(cback()->str().substr(1U, cback()->str().size() - 2U)); + location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U)); location.line = 1U; } else if (lastline == "# line %num%") { - lineDirective(location.fileIndex, std::atol(cback()->str().c_str()), &location); + const Token *numtok = cback(); + while (numtok->comment) + numtok = numtok->previous; + lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location); } else if (lastline == "# %num% %str%" || lastline == "# line %num% %str%") { - lineDirective(fileIndex(replaceAll(cback()->str().substr(1U, cback()->str().size() - 2U),"\\\\","\\")), - std::atol(cback()->previous->str().c_str()), &location); + const Token *strtok = cback(); + while (strtok->comment) + strtok = strtok->previous; + const Token *numtok = strtok->previous; + while (numtok->comment) + numtok = numtok->previous; + lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")), + std::atol(numtok->str().c_str()), &location); } // #endfile else if (lastline == "# endfile" && !loc.empty()) { diff --git a/test.cpp b/test.cpp index 9404e655..0e60e393 100644 --- a/test.cpp +++ b/test.cpp @@ -10,14 +10,25 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) #define ASSERT_THROW(stmt, e) try { stmt; assertThrowFailed(__LINE__); } catch (const e&) {} +static std::string pprint(const std::string &in) +{ + std::string ret; + for (int i = 0; i < (int)in.size(); ++i) { + if (in[i] == '\n') + ret += "\\n"; + ret += in[i]; + } + return ret; +} + static int assertEquals(const std::string &expected, const std::string &actual, int line) { if (expected != actual) { numberOfFailedAssertions++; std::cerr << "------ assertion failed ---------" << std::endl; std::cerr << "line " << line << std::endl; - std::cerr << "expected:" << expected << std::endl; - std::cerr << "actual:" << actual << std::endl; + std::cerr << "expected:" << pprint(expected) << std::endl; + std::cerr << "actual:" << pprint(actual) << std::endl; } return (expected == actual); } @@ -1394,6 +1405,16 @@ static void location4() ASSERT_EQUALS("\n#line 1 \"abc\\def.g\"\na", preprocess(code)); } +static void location5() +{ + // https://sourceforge.net/p/cppcheck/discussion/general/thread/eccf020a13/ + const char *code; + code = "#line 10 \"/a/Attribute/parser/FilterParser.y\" // lalr1.cc:377\n" + "int x;\n"; + ASSERT_EQUALS("\n#line 10 \"/a/Attribute/parser/FilterParser.y\"\n" + "int x ;", preprocess(code)); +} + static void missingHeader1() { const simplecpp::DUI dui; @@ -2298,6 +2319,7 @@ int main(int argc, char **argv) TEST_CASE(location2); TEST_CASE(location3); TEST_CASE(location4); + TEST_CASE(location5); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); From ab680f96ea9aa0fc7ed241f6dc51954d7a8d6735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 16 Dec 2021 21:16:46 +0100 Subject: [PATCH 135/381] fixed a define define problem, macro expanded twice --- simplecpp.cpp | 20 +++++++++++--------- simplecpp.h | 10 ++++++++++ test.cpp | 10 ++++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6445b093..852a5b89 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1405,10 +1405,12 @@ namespace simplecpp { }; private: /** Create new token where Token::macro is set for replaced tokens */ - Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced) const { + Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=NULL) const { Token *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); + if (expandedFromToken) + tok->setExpandedFrom(expandedFromToken, this); return tok; } @@ -1626,12 +1628,12 @@ namespace simplecpp { throw invalidHashHash(tok->location, name()); TokenList new_output(files); if (!expandArg(&new_output, tok, parametertokens2)) - output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros))); + output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok)); else if (new_output.empty()) // placemarker token output->push_back(newMacroToken("", loc, isReplaced(expandedmacros))); else for (const Token *tok2 = new_output.cfront(); tok2; tok2 = tok2->next) - output->push_back(newMacroToken(tok2->str(), loc, isReplaced(expandedmacros))); + output->push_back(newMacroToken(tok2->str(), loc, isReplaced(expandedmacros), tok2)); tok = tok->next; } else { tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens2); @@ -1722,7 +1724,7 @@ namespace simplecpp { const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { - output->push_back(newMacroToken(tok->str(), loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } @@ -1746,14 +1748,14 @@ namespace simplecpp { return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); } if (!sameline(tok, tok->next) || tok->next->op != '(') { - output->push_back(newMacroToken(tok->str(), loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } TokenList tokens(files); tokens.push_back(new Token(*tok)); const Token *tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) { - output->push_back(newMacroToken(tok->str(), loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } TokenList temp(files); @@ -1791,7 +1793,7 @@ namespace simplecpp { } } - output->push_back(newMacroToken(tok->str(), loc, true)); + output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } @@ -1823,10 +1825,10 @@ namespace simplecpp { return true; for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) { const std::map::const_iterator it = macros.find(partok->str()); - if (it != macros.end() && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) + if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) partok = it->second.expand(output, loc, partok, macros, expandedmacros); else { - output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros))); + output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); output->back()->macro = partok->macro; partok = partok->next; } diff --git a/simplecpp.h b/simplecpp.h index 82adad40..faeb2007 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -149,11 +149,21 @@ namespace simplecpp { return tok; } + void setExpandedFrom(const Token *tok, const void* m) { + mExpandedFrom = tok->mExpandedFrom; + mExpandedFrom.insert(m); + } + bool isExpandedFrom(const void* m) const { + return mExpandedFrom.find(m) != mExpandedFrom.end(); + } + void printAll() const; void printOut() const; private: TokenString string; + std::set mExpandedFrom; + // Not implemented - prevent assignment Token &operator=(const Token &tok); }; diff --git a/test.cpp b/test.cpp index 0e60e393..98a1f6da 100644 --- a/test.cpp +++ b/test.cpp @@ -702,6 +702,15 @@ static void define_define_17() ASSERT_EQUALS("\n\n1 ;", preprocess(code)); } +static void define_define_18() +{ + const char code[] = "#define FOO(v) BAR(v, 0)\n" + "#define BAR(v, x) (v)\n" + "#define var (p->var)\n" + "FOO(var);"; + ASSERT_EQUALS("\n\n\n( ( p -> var ) ) ;", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -2255,6 +2264,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_15); TEST_CASE(define_define_16); TEST_CASE(define_define_17); + TEST_CASE(define_define_18); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From 936016e8583402d1a4d64b73d9fd1ff1f2daeeba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 12 Feb 2022 11:35:30 +0100 Subject: [PATCH 136/381] small CI improvements - always run all jobs and removed obsolete ubuntu-16.04 (#233) --- .github/workflows/CI-unixish.yml | 4 ++-- .github/workflows/CI-windows.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 5205d85e..835189e6 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,8 +8,8 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-10.15] - fail-fast: true + os: [ubuntu-18.04, ubuntu-20.04, macos-10.15] + fail-fast: false runs-on: ${{ matrix.os }} diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index f64e34f9..b9df721f 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -17,7 +17,7 @@ jobs: matrix: # windows 2016 should default to VS 2017. Not supported by setup-msbuild os: [windows-2019] - fail-fast: true + fail-fast: false runs-on: ${{ matrix.os }} From 479d2224efa608ae701624c34002c9307d22cff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 12 Feb 2022 18:54:17 +0100 Subject: [PATCH 137/381] fixed -Wzero-as-null-pointer-constant Clang warnings (#232) --- CMakeLists.txt | 4 +-- Makefile | 4 ++- simplecpp.cpp | 74 ++++++++++++++++++++++++++++---------------------- simplecpp.h | 21 +++++++++----- 4 files changed, 60 insertions(+), 43 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1baef3d..4e22271e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # these are not really fixable add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) # TODO: fix these? - add_compile_options(-Wno-zero-as-null-pointer-constant -Wno-padded -Wno-sign-conversion -Wno-conversion -Wno-old-style-cast) + add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-conversion -Wno-old-style-cast) endif() add_executable(simplecpp simplecpp.cpp main.cpp) @@ -26,7 +26,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang" if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") target_compile_options(simplecpp-03-syntax PRIVATE -Wno-long-long) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long) + target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long -Wno-c++11-compat) endif() add_dependencies(simplecpp simplecpp-03-syntax) endif() diff --git a/Makefile b/Makefile index f02aee72..37ce32cb 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,9 @@ testrunner: test.o simplecpp.o test: testrunner simplecpp # The -std=c++03 makes sure that simplecpp.cpp is C++03 conformant. We don't require a C++11 compiler - g++ -std=c++03 -fsyntax-only simplecpp.cpp && ./testrunner && python run-tests.py + g++ -std=c++03 -fsyntax-only simplecpp.cpp + ./testrunner + python run-tests.py simplecpp: main.o simplecpp.o $(CXX) $(LDFLAGS) main.o simplecpp.o -o simplecpp diff --git a/simplecpp.cpp b/simplecpp.cpp index 852a5b89..6295f64c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -41,6 +41,10 @@ #undef TRUE #endif +#if (__cplusplus < 201103L) && !defined(__APPLE__) +#define nullptr NULL +#endif + static bool isHex(const std::string &s) { return s.size()>2 && (s.compare(0,2,"0x")==0 || s.compare(0,2,"0X")==0); @@ -169,17 +173,17 @@ void simplecpp::Location::adjust(const std::string &str) bool simplecpp::Token::isOneOf(const char ops[]) const { - return (op != '\0') && (std::strchr(ops, op) != NULL); + return (op != '\0') && (std::strchr(ops, op) != nullptr); } bool simplecpp::Token::startsWithOneOf(const char c[]) const { - return std::strchr(c, string[0]) != NULL; + return std::strchr(c, string[0]) != nullptr; } bool simplecpp::Token::endsWithOneOf(const char c[]) const { - return std::strchr(c, string[string.size() - 1U]) != NULL; + return std::strchr(c, string[string.size() - 1U]) != nullptr; } void simplecpp::Token::printAll() const @@ -207,21 +211,21 @@ void simplecpp::Token::printOut() const std::cout << std::endl; } -simplecpp::TokenList::TokenList(std::vector &filenames) : frontToken(NULL), backToken(NULL), files(filenames) {} +simplecpp::TokenList::TokenList(std::vector &filenames) : frontToken(nullptr), backToken(nullptr), files(filenames) {} simplecpp::TokenList::TokenList(std::istream &istr, std::vector &filenames, const std::string &filename, OutputList *outputList) - : frontToken(NULL), backToken(NULL), files(filenames) + : frontToken(nullptr), backToken(nullptr), files(filenames) { readfile(istr,filename,outputList); } -simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(NULL), backToken(NULL), files(other.files) +simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), backToken(nullptr), files(other.files) { *this = other; } #if __cplusplus >= 201103L -simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(NULL), backToken(NULL), files(other.files) +simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(nullptr), backToken(nullptr), files(other.files) { *this = std::move(other); } @@ -249,9 +253,9 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) if (this != &other) { clear(); backToken = other.backToken; - other.backToken = NULL; + other.backToken = nullptr; frontToken = other.frontToken; - other.frontToken = NULL; + other.frontToken = nullptr; sizeOfType = std::move(other.sizeOfType); } return *this; @@ -260,7 +264,7 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) void simplecpp::TokenList::clear() { - backToken = NULL; + backToken = nullptr; while (frontToken) { Token *next = frontToken->next; delete frontToken; @@ -455,7 +459,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen unsigned int multiline = 0U; - const Token *oldLastToken = NULL; + const Token *oldLastToken = nullptr; const unsigned short bom = getAndSkipBOM(istr); @@ -1220,9 +1224,9 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) namespace simplecpp { class Macro { public: - explicit Macro(std::vector &f) : nameTokDef(NULL), variadic(false), valueToken(NULL), endToken(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(false) {} + explicit Macro(std::vector &f) : nameTokDef(nullptr), variadic(false), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) {} - Macro(const Token *tok, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(true) { + Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previous, tok)) throw std::runtime_error("bad macro syntax"); if (tok->op != '#') @@ -1238,7 +1242,7 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax"); } - Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(NULL), files(f), tokenListDefine(f), valueDefinedInCode_(false) { + Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); std::istringstream istr(def); tokenListDefine.readfile(istr); @@ -1246,7 +1250,7 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); } - Macro(const Macro ¯o) : nameTokDef(NULL), files(macro.files), tokenListDefine(macro.files), valueDefinedInCode_(macro.valueDefinedInCode_) { + Macro(const Macro ¯o) : nameTokDef(nullptr), files(macro.files), tokenListDefine(macro.files), valueDefinedInCode_(macro.valueDefinedInCode_) { *this = macro; } @@ -1356,7 +1360,7 @@ namespace simplecpp { } if (!rawtok2 || par != 1U) break; - if (macro->second.expand(&output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != NULL) + if (macro->second.expand(&output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != nullptr) break; rawtok = rawtok2->next; } @@ -1418,7 +1422,7 @@ namespace simplecpp { nameTokDef = nametoken; variadic = false; if (!nameTokDef) { - valueToken = endToken = NULL; + valueToken = endToken = nullptr; args.clear(); return false; } @@ -1442,17 +1446,17 @@ namespace simplecpp { } if (!sameline(nametoken, argtok)) { endToken = argtok ? argtok->previous : argtok; - valueToken = NULL; + valueToken = nullptr; return false; } - valueToken = argtok ? argtok->next : NULL; + valueToken = argtok ? argtok->next : nullptr; } else { args.clear(); valueToken = nameTokDef->next; } if (!sameline(valueToken, nameTokDef)) - valueToken = NULL; + valueToken = nullptr; endToken = valueToken; while (sameline(endToken, nameTokDef)) endToken = endToken->next; @@ -1476,7 +1480,7 @@ namespace simplecpp { std::vector parametertokens; parametertokens.push_back(nameTokInst->next); unsigned int par = 0U; - for (const Token *tok = nameTokInst->next->next; calledInDefine ? sameline(tok, nameTokInst) : (tok != NULL); tok = tok->next) { + for (const Token *tok = nameTokInst->next->next; calledInDefine ? sameline(tok, nameTokInst) : (tok != nullptr); tok = tok->next) { if (tok->op == '(') ++par; else if (tok->op == ')') { @@ -1498,7 +1502,7 @@ namespace simplecpp { const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!lpar || lpar->op != '(') - return NULL; + return nullptr; unsigned int par = 0; const Token *tok = lpar; while (sameline(lpar, tok)) { @@ -1537,7 +1541,7 @@ namespace simplecpp { } for (Token *tok2 = tokens->front(); tok2; tok2 = tok2->next) tok2->location = lpar->location; - return sameline(lpar,tok) ? tok : NULL; + return sameline(lpar,tok) ? tok : nullptr; } const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const std::map ¯os, std::set expandedmacros, bool first=false) const { @@ -1765,10 +1769,10 @@ namespace simplecpp { else if (tok->str() == DEFINED) { const Token *tok2 = tok->next; - const Token *tok3 = tok2 ? tok2->next : NULL; - const Token *tok4 = tok3 ? tok3->next : NULL; - const Token *defToken = NULL; - const Token *lastToken = NULL; + const Token *tok3 = tok2 ? tok2->next : nullptr; + const Token *tok4 = tok3 ? tok3->next : nullptr; + const Token *defToken = nullptr; + const Token *lastToken = nullptr; if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { defToken = tok3; lastToken = tok4; @@ -2753,8 +2757,8 @@ std::map simplecpp::load(const simplecpp::To filelist.push_back(tokenlist->front()); } - for (const Token *rawtok = rawtokens.cfront(); rawtok || !filelist.empty(); rawtok = rawtok ? rawtok->next : NULL) { - if (rawtok == NULL) { + for (const Token *rawtok = rawtokens.cfront(); rawtok || !filelist.empty(); rawtok = rawtok ? rawtok->next : nullptr) { + if (rawtok == nullptr) { rawtok = filelist.back(); filelist.pop_back(); } @@ -2888,8 +2892,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL includetokenstack.push(f->second->cfront()); } - for (const Token *rawtok = NULL; rawtok || !includetokenstack.empty();) { - if (rawtok == NULL) { + for (const Token *rawtok = nullptr; rawtok || !includetokenstack.empty();) { + if (rawtok == nullptr) { rawtok = includetokenstack.top(); includetokenstack.pop(); continue; @@ -3034,7 +3038,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (pragmaOnce.find(header2) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); const TokenList *includetokens = filedata.find(header2)->second; - rawtok = includetokens ? includetokens->cfront() : NULL; + rawtok = includetokens ? includetokens->cfront() : nullptr; continue; } } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { @@ -3079,7 +3083,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL expr.push_back(new Token("0", tok->location)); } if (par) - tok = tok ? tok->next : NULL; + tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { if (outputList) { Output out(rawtok->location.files); @@ -3250,3 +3254,7 @@ void simplecpp::cleanup(std::map &filedata) delete it->second; filedata.clear(); } + +#if (__cplusplus < 201103L) && !defined(__APPLE__) +#undef nullptr +#endif diff --git a/simplecpp.h b/simplecpp.h index faeb2007..2da37660 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -40,6 +40,9 @@ # define SIMPLECPP_LIB #endif +#if (__cplusplus < 201103L) && !defined(__APPLE__) +#define nullptr NULL +#endif namespace simplecpp { @@ -97,12 +100,12 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc) : - location(loc), previous(NULL), next(NULL), string(s) { + location(loc), previous(nullptr), next(nullptr), string(s) { flags(); } Token(const Token &tok) : - macro(tok.macro), location(tok.location), previous(NULL), next(NULL), string(tok.string) { + macro(tok.macro), location(tok.location), previous(nullptr), next(nullptr), string(tok.string) { flags(); } @@ -191,7 +194,7 @@ namespace simplecpp { class SIMPLECPP_LIB TokenList { public: explicit TokenList(std::vector &filenames); - TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = NULL); + TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); TokenList(const TokenList &other); #if __cplusplus >= 201103L TokenList(TokenList &&other); @@ -211,7 +214,7 @@ namespace simplecpp { void dump() const; std::string stringify() const; - void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = NULL); + void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = nullptr); void constFold(); void removeComments(); @@ -258,7 +261,7 @@ namespace simplecpp { other.frontToken->previous = backToken; } backToken = other.backToken; - other.frontToken = other.backToken = NULL; + other.frontToken = other.backToken = nullptr; } /** sizeof(T) */ @@ -320,7 +323,7 @@ namespace simplecpp { SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); - SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = NULL); + SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr); /** * Preprocess @@ -334,7 +337,7 @@ namespace simplecpp { * @param macroUsage output: macro usage * @param ifCond output: #if/#elif expressions */ - SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = NULL, std::list *macroUsage = NULL, std::list *ifCond = NULL); + SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr); /** * Deallocate data @@ -348,4 +351,8 @@ namespace simplecpp { SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); } +#if (__cplusplus < 201103L) && !defined(__APPLE__) +#undef nullptr +#endif + #endif From 49317592f9992ee12379b0005ac2ba26c428dc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Feb 2022 08:52:43 +0100 Subject: [PATCH 138/381] added missing and adjusted existing copyright headers (#237) --- main.cpp | 17 +++++++++++++++++ simplecpp.cpp | 2 +- simplecpp.h | 2 +- test.cpp | 17 +++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/main.cpp b/main.cpp index 823ed9f7..dd0c4453 100644 --- a/main.cpp +++ b/main.cpp @@ -1,3 +1,20 @@ +/* + * simplecpp - A simple and high-fidelity C/C++ preprocessor library + * Copyright (C) 2016-2022 Daniel Marjamäki. + * + * This library is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + */ #include "simplecpp.h" diff --git a/simplecpp.cpp b/simplecpp.cpp index 6295f64c..e44cbb48 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1,6 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016 Daniel Marjamäki. + * Copyright (C) 2016-2022 Daniel Marjamäki. * * This library is free software: you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/simplecpp.h b/simplecpp.h index 2da37660..23b1eea8 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -1,6 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016 Daniel Marjamäki. + * Copyright (C) 2016-2022 Daniel Marjamäki. * * This library is free software: you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/test.cpp b/test.cpp index 98a1f6da..a3bcf2c4 100644 --- a/test.cpp +++ b/test.cpp @@ -1,3 +1,20 @@ +/* + * simplecpp - A simple and high-fidelity C/C++ preprocessor library + * Copyright (C) 2016-2022 Daniel Marjamäki. + * + * This library is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + */ #include #include From b1c05bfc21d8717c651e0578fe3b6d37e0099648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Feb 2022 08:53:14 +0100 Subject: [PATCH 139/381] removed unused include (#236) --- simplecpp.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e44cbb48..22132b02 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #include #include From 96d27cc0b5cee0f96938c230f1f501e790d1b31f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Feb 2022 08:57:20 +0100 Subject: [PATCH 140/381] check rfind() result in simplifyPath() before using it - fixes "unsigned integer overflow" reported by UBSAN (#196) --- simplecpp.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 22132b02..407b4371 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2222,14 +2222,19 @@ namespace simplecpp { continue; } // get previous subpath - const std::string::size_type pos1 = path.rfind('/', pos - 1U) + 1U; - const std::string previousSubPath = path.substr(pos1, pos-pos1); + std::string::size_type pos1 = path.rfind('/', pos - 1U); + if (pos1 == std::string::npos) { + pos1 = 0; + } else { + pos1 += 1U; + } + const std::string previousSubPath = path.substr(pos1, pos - pos1); if (previousSubPath == "..") { // don't simplify ++pos; } else { // remove previous subpath and ".." - path.erase(pos1,pos-pos1+4); + path.erase(pos1, pos - pos1 + 4); if (path.empty()) path = "."; // update pos From 0c36e4b4aa4ea506e85fc7e8cd07e42040d317f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Feb 2022 09:06:07 +0100 Subject: [PATCH 141/381] several optimizations (#163) --- simplecpp.cpp | 60 ++++++++++++++++++++++++++++++--------------------- simplecpp.h | 6 +++--- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 407b4371..fee5e1d7 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -32,6 +32,9 @@ #include #include #include +#if __cplusplus >= 201103L +#include +#endif #include #ifdef SIMPLECPP_WINDOWS @@ -154,7 +157,7 @@ const std::string simplecpp::Location::emptyFileName; void simplecpp::Location::adjust(const std::string &str) { - if (str.find_first_of("\r\n") == std::string::npos) { + if (strpbrk(str.c_str(), "\r\n") == nullptr) { col += str.size(); return; } @@ -452,6 +455,8 @@ void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int li } } +static const std::string COMMENT_END("*/"); + void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filename, OutputList *outputList) { std::stack loc; @@ -592,7 +597,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen ch = readChar(istr,bom); while (istr.good()) { currentToken += ch; - if (currentToken.size() >= 4U && endsWith(currentToken, "*/")) + if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END)) break; ch = readChar(istr,bom); } @@ -1199,12 +1204,12 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const for (const Token *tok = cback(); sameline(tok,cback()); tok = tok->previous) { if (tok->comment) continue; + if (++count > maxsize) + return ""; if (!ret.empty()) ret.insert(0, 1, ' '); ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") : tok->number ? std::string("%num%") : tok->str()); - if (++count > maxsize) - return ""; } return ret; } @@ -1221,6 +1226,13 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) namespace simplecpp { + class Macro; +#if __cplusplus >= 201103L + using MacroMap = std::unordered_map; +#else + typedef std::map MacroMap; +#endif + class Macro { public: explicit Macro(std::vector &f) : nameTokDef(nullptr), variadic(false), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) {} @@ -1280,7 +1292,7 @@ namespace simplecpp { */ const Token * expand(TokenList * const output, const Token * rawtok, - const std::map ¯os, + const MacroMap ¯os, std::vector &inputFiles) const { std::set expandedmacros; @@ -1334,7 +1346,7 @@ namespace simplecpp { break; if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) break; - const std::map::const_iterator macro = macros.find(macro2tok->str()); + const MacroMap::const_iterator macro = macros.find(macro2tok->str()); if (macro == macros.end() || !macro->second.functionLike()) break; TokenList rawtokens2(inputFiles); @@ -1497,7 +1509,7 @@ namespace simplecpp { const Token *appendTokens(TokenList *tokens, const Location &rawloc, const Token * const lpar, - const std::map ¯os, + const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!lpar || lpar->op != '(') @@ -1513,7 +1525,7 @@ namespace simplecpp { } else { if (!expandArg(tokens, tok, rawloc, macros, expandedmacros, parametertokens)) { bool expanded = false; - const std::map::const_iterator it = macros.find(tok->str()); + const MacroMap::const_iterator it = macros.find(tok->str()); if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { @@ -1543,7 +1555,7 @@ namespace simplecpp { return sameline(lpar,tok) ? tok : nullptr; } - const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const std::map ¯os, std::set expandedmacros, bool first=false) const { + const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros, bool first=false) const { if (!first) expandedmacros.insert(nameTokInst->str()); @@ -1598,7 +1610,7 @@ namespace simplecpp { } } - const std::map::const_iterator m = macros.find("__COUNTER__"); + const MacroMap::const_iterator m = macros.find("__COUNTER__"); if (!counter || m == macros.end()) parametertokens2.swap(parametertokens1); @@ -1689,7 +1701,7 @@ namespace simplecpp { return functionLike() ? parametertokens2.back()->next : nameTokInst->next; } - const Token *recursiveExpandToken(TokenList *output, TokenList &temp, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *recursiveExpandToken(TokenList *output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!(temp.cback() && temp.cback()->name && tok->next && tok->next->op == '(')) { output->takeTokens(temp); return tok->next; @@ -1700,7 +1712,7 @@ namespace simplecpp { return tok->next; } - const std::map::const_iterator it = macros.find(temp.cback()->str()); + const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) { output->takeTokens(temp); return tok->next; @@ -1724,7 +1736,7 @@ namespace simplecpp { return tok2->next; } - const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { output->push_back(newMacroToken(tok->str(), loc, true, tok)); @@ -1739,7 +1751,7 @@ namespace simplecpp { } // Macro.. - const std::map::const_iterator it = macros.find(tok->str()); + const MacroMap::const_iterator it = macros.find(tok->str()); if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { std::set expandedmacros2(expandedmacros); expandedmacros2.insert(tok->str()); @@ -1818,7 +1830,7 @@ namespace simplecpp { return true; } - bool expandArg(TokenList *output, const Token *tok, const Location &loc, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + bool expandArg(TokenList *output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!tok->name) return false; const unsigned int argnr = getArgNum(tok->str()); @@ -1827,7 +1839,7 @@ namespace simplecpp { if (variadic && argnr + 1U >= parametertokens.size()) // empty variadic parameter return true; for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) { - const std::map::const_iterator it = macros.find(partok->str()); + const MacroMap::const_iterator it = macros.find(partok->str()); if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) partok = it->second.expand(output, loc, partok, macros, expandedmacros); else { @@ -1849,7 +1861,7 @@ namespace simplecpp { * @param parametertokens parameters given when expanding this macro * @return token after the X */ - const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { TokenList tokenListHash(files); tok = expandToken(&tokenListHash, loc, tok->next, macros, expandedmacros, parametertokens); std::ostringstream ostr; @@ -1872,7 +1884,7 @@ namespace simplecpp { * @param parametertokens parameters given when expanding this macro * @return token after B */ - const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const std::map ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { Token *A = output->back(); if (!A) throw invalidHashHash(tok->location, name()); @@ -1929,7 +1941,7 @@ namespace simplecpp { tokens.push_back(new Token(strAB, tok->location)); // for function like macros, push the (...) if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { - const std::map::const_iterator it = macros.find(strAB); + const MacroMap::const_iterator it = macros.find(strAB); if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { const Token *tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); if (tok2) @@ -2799,10 +2811,10 @@ std::map simplecpp::load(const simplecpp::To return ret; } -static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, std::map ¯os, std::vector &files, simplecpp::OutputList *outputList) +static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) { const simplecpp::Token *tok = *tok1; - const std::map::const_iterator it = macros.find(tok->str()); + const simplecpp::MacroMap::const_iterator it = macros.find(tok->str()); if (it != macros.end()) { simplecpp::TokenList value(files); try { @@ -2851,7 +2863,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL sizeOfType.insert(std::make_pair("long double *", sizeof(long double *))); const bool hasInclude = (dui.std.size() == 5 && dui.std.compare(0,3,"c++") == 0 && dui.std >= "c++17"); - std::map macros; + MacroMap macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; const std::string::size_type eq = macrostr.find('='); @@ -2951,7 +2963,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL try { const Macro ¯o = Macro(rawtok->previous, files); if (dui.undefined.find(macro.name()) == dui.undefined.end()) { - std::map::iterator it = macros.find(macro.name()); + MacroMap::iterator it = macros.find(macro.name()); if (it == macros.end()) macros.insert(std::pair(macro.name(), macro)); else @@ -3238,7 +3250,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } if (macroUsage) { - for (std::map::const_iterator macroIt = macros.begin(); macroIt != macros.end(); ++macroIt) { + for (simplecpp::MacroMap::const_iterator macroIt = macros.begin(); macroIt != macros.end(); ++macroIt) { const Macro ¯o = macroIt->second; const std::list &usage = macro.usage(); for (std::list::const_iterator usageIt = usage.begin(); usageIt != usage.end(); ++usageIt) { diff --git a/simplecpp.h b/simplecpp.h index 23b1eea8..b5845f17 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -105,13 +106,12 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), location(tok.location), previous(nullptr), next(nullptr), string(tok.string) { - flags(); + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), location(tok.location), previous(nullptr), next(nullptr), string(tok.string) { } void flags() { name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$') - && (string.find('\'') == string.npos); + && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); op = (string.size() == 1U) ? string[0] : '\0'; From 77c778ac3a5fdf03458e9bc047e8d15288a30d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Feb 2022 20:42:32 +0100 Subject: [PATCH 142/381] cleaned and fixed up TokenList/Macro copy/move constructor/assignment (#167) --- simplecpp.cpp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index fee5e1d7..990c9bcb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -227,7 +227,7 @@ simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), b } #if __cplusplus >= 201103L -simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(nullptr), backToken(nullptr), files(other.files) +simplecpp::TokenList::TokenList(TokenList &&other) : TokenList(other) { *this = std::move(other); } @@ -242,6 +242,7 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(const TokenList &other) { if (this != &other) { clear(); + files = other.files; for (const Token *tok = other.cfront(); tok; tok = tok->next) push_back(new Token(*tok)); sizeOfType = other.sizeOfType; @@ -254,10 +255,11 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) { if (this != &other) { clear(); - backToken = other.backToken; - other.backToken = nullptr; frontToken = other.frontToken; other.frontToken = nullptr; + backToken = other.backToken; + other.backToken = nullptr; + files = other.files; sizeOfType = std::move(other.sizeOfType); } return *this; @@ -1261,20 +1263,22 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); } - Macro(const Macro ¯o) : nameTokDef(nullptr), files(macro.files), tokenListDefine(macro.files), valueDefinedInCode_(macro.valueDefinedInCode_) { - *this = macro; + Macro(const Macro &other) : nameTokDef(nullptr), files(other.files), tokenListDefine(other.files), valueDefinedInCode_(other.valueDefinedInCode_) { + *this = other; } - void operator=(const Macro ¯o) { - if (this != ¯o) { - valueDefinedInCode_ = macro.valueDefinedInCode_; - if (macro.tokenListDefine.empty()) - parseDefine(macro.nameTokDef); + Macro &operator=(const Macro &other) { + if (this != &other) { + files = other.files; + valueDefinedInCode_ = other.valueDefinedInCode_; + if (other.tokenListDefine.empty()) + parseDefine(other.nameTokDef); else { - tokenListDefine = macro.tokenListDefine; + tokenListDefine = other.tokenListDefine; parseDefine(tokenListDefine.cfront()); } } + return *this; } bool valueDefinedInCode() const { From 9d78bbb348e30b6caa5d26f3a2e9249f03659eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 16 Feb 2022 07:08:11 +0100 Subject: [PATCH 143/381] fixed and enabled -Wold-style-cast (#235) --- CMakeLists.txt | 4 ++-- Makefile | 2 +- simplecpp.cpp | 20 ++++++++++---------- simplecpp.h | 4 ++-- test.cpp | 16 ++++++++-------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e22271e..1f5e05b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 3.5) project (simplecpp LANGUAGES CXX) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef) + add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast) endif() if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -12,7 +12,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # these are not really fixable add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) # TODO: fix these? - add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-conversion -Wno-old-style-cast) + add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-conversion) endif() add_executable(simplecpp simplecpp.cpp main.cpp) diff --git a/Makefile b/Makefile index 37ce32cb..a45c90c9 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -std=c++0x -g +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++0x -g LDFLAGS = -g %.o: %.cpp simplecpp.h diff --git a/simplecpp.cpp b/simplecpp.cpp index 990c9bcb..51d5fb9d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -320,20 +320,20 @@ std::string simplecpp::TokenList::stringify() const static unsigned char readChar(std::istream &istr, unsigned int bom) { - unsigned char ch = (unsigned char)istr.get(); + unsigned char ch = static_cast(istr.get()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the // character is non-ASCII character then replace it with 0xff if (bom == 0xfeff || bom == 0xfffe) { - const unsigned char ch2 = (unsigned char)istr.get(); + const unsigned char ch2 = static_cast(istr.get()); const int ch16 = (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); - ch = (unsigned char)((ch16 >= 0x80) ? 0xff : ch16); + ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); } // Handling of newlines.. if (ch == '\r') { ch = '\n'; - if (bom == 0 && (char)istr.peek() == '\n') + if (bom == 0 && static_cast(istr.peek()) == '\n') (void)istr.get(); else if (bom == 0xfeff || bom == 0xfffe) { int c1 = istr.get(); @@ -351,16 +351,16 @@ static unsigned char readChar(std::istream &istr, unsigned int bom) static unsigned char peekChar(std::istream &istr, unsigned int bom) { - unsigned char ch = (unsigned char)istr.peek(); + unsigned char ch = static_cast(istr.peek()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the // character is non-ASCII character then replace it with 0xff if (bom == 0xfeff || bom == 0xfffe) { (void)istr.get(); - const unsigned char ch2 = (unsigned char)istr.peek(); + const unsigned char ch2 = static_cast(istr.peek()); istr.unget(); const int ch16 = (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); - ch = (unsigned char)((ch16 >= 0x80) ? 0xff : ch16); + ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); } // Handling of newlines.. @@ -383,9 +383,9 @@ static unsigned short getAndSkipBOM(std::istream &istr) // The UTF-16 BOM is 0xfffe or 0xfeff. if (ch1 >= 0xfe) { - unsigned short bom = ((unsigned char)istr.get() << 8); + unsigned short bom = (static_cast(istr.get()) << 8); if (istr.peek() >= 0xfe) - return bom | (unsigned char)istr.get(); + return bom | static_cast(istr.get()); istr.unget(); return 0; } @@ -486,7 +486,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen err.type = simplecpp::Output::UNHANDLED_CHAR_ERROR; err.location = location; std::ostringstream s; - s << (int)ch; + s << static_cast(ch); err.msg = "The code contains unhandled character(s) (character code=" + s.str() + "). Neither unicode nor extended ascii is supported."; outputList->push_back(err); } diff --git a/simplecpp.h b/simplecpp.h index b5845f17..8f1a3cae 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -110,10 +110,10 @@ namespace simplecpp { } void flags() { - name = (std::isalpha((unsigned char)string[0]) || string[0] == '_' || string[0] == '$') + name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); - number = std::isdigit((unsigned char)string[0]) || (string.size() > 1U && string[0] == '-' && std::isdigit((unsigned char)string[1])); + number = std::isdigit(static_cast(string[0])) || (string.size() > 1U && string[0] == '-' && std::isdigit(static_cast(string[1]))); op = (string.size() == 1U) ? string[0] : '\0'; } diff --git a/test.cpp b/test.cpp index a3bcf2c4..a321dfb3 100644 --- a/test.cpp +++ b/test.cpp @@ -30,7 +30,7 @@ static int numberOfFailedAssertions = 0; static std::string pprint(const std::string &in) { std::string ret; - for (int i = 0; i < (int)in.size(); ++i) { + for (std::string::size_type i = 0; i < in.size(); ++i) { if (in[i] == '\n') ret += "\\n"; ret += in[i]; @@ -223,10 +223,10 @@ static void characterLiteral() ASSERT_EQUALS('\u0012', simplecpp::characterLiteralToLL("'\\u0012'")); ASSERT_EQUALS('\U00000012', simplecpp::characterLiteralToLL("'\\U00000012'")); - ASSERT_EQUALS(((unsigned int)(unsigned char)'b' << 8) | (unsigned char)'c', simplecpp::characterLiteralToLL("'bc'")); - ASSERT_EQUALS(((unsigned int)(unsigned char)'\x23' << 8) | (unsigned char)'\x45', simplecpp::characterLiteralToLL("'\\x23\\x45'")); - ASSERT_EQUALS(((unsigned int)(unsigned char)'\11' << 8) | (unsigned char)'\222', simplecpp::characterLiteralToLL("'\\11\\222'")); - ASSERT_EQUALS(((unsigned int)(unsigned char)'\a' << 8) | (unsigned char)'\b', simplecpp::characterLiteralToLL("'\\a\\b'")); + ASSERT_EQUALS((static_cast(static_cast('b')) << 8) | static_cast('c'), simplecpp::characterLiteralToLL("'bc'")); + ASSERT_EQUALS((static_cast(static_cast('\x23')) << 8) | static_cast('\x45'), simplecpp::characterLiteralToLL("'\\x23\\x45'")); + ASSERT_EQUALS((static_cast(static_cast('\11')) << 8) | static_cast('\222'), simplecpp::characterLiteralToLL("'\\11\\222'")); + ASSERT_EQUALS((static_cast(static_cast('\a')) << 8) | static_cast('\b'), simplecpp::characterLiteralToLL("'\\a\\b'")); if (sizeof(int) <= 4) ASSERT_EQUALS(-1, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); else @@ -253,9 +253,9 @@ static void characterLiteral() #ifdef __GNUC__ // BEGIN Implementation-specific results - ASSERT_EQUALS((int)('AB'), simplecpp::characterLiteralToLL("'AB'")); - ASSERT_EQUALS((int)('ABC'), simplecpp::characterLiteralToLL("'ABC'")); - ASSERT_EQUALS((int)('ABCD'), simplecpp::characterLiteralToLL("'ABCD'")); + ASSERT_EQUALS(static_cast('AB'), simplecpp::characterLiteralToLL("'AB'")); + ASSERT_EQUALS(static_cast('ABC'), simplecpp::characterLiteralToLL("'ABC'")); + ASSERT_EQUALS(static_cast('ABCD'), simplecpp::characterLiteralToLL("'ABCD'")); ASSERT_EQUALS('\134t', simplecpp::characterLiteralToLL("'\\134t'")); // cppcheck ticket #7452 // END Implementation-specific results #endif From 596e0cc3bdc4342574f277993851d75d59af8ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 16 Feb 2022 21:28:55 +0100 Subject: [PATCH 144/381] fixed some compiler warnings / cleanups (#238) --- CMakeLists.txt | 10 +++++----- main.cpp | 2 +- simplecpp.cpp | 12 ++++++------ test.cpp | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f5e05b4..0383423a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,17 +2,17 @@ cmake_minimum_required (VERSION 3.5) project (simplecpp LANGUAGES CXX) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast) -endif() - -if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast -Wno-multichar) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Weverything) # no need for c++98 compatibility add_compile_options(-Wno-c++98-compat-pedantic) # these are not really fixable add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) + # we are not interested in these + add_compile_options(-Wno-multichar) # TODO: fix these? - add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-conversion) + add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32) endif() add_executable(simplecpp simplecpp.cpp main.cpp) diff --git a/main.cpp b/main.cpp index dd0c4453..f16cee77 100644 --- a/main.cpp +++ b/main.cpp @@ -24,7 +24,7 @@ int main(int argc, char **argv) { - const char *filename = NULL; + const char *filename = nullptr; // Settings.. simplecpp::DUI dui; diff --git a/simplecpp.cpp b/simplecpp.cpp index 51d5fb9d..c995e070 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1424,7 +1424,7 @@ namespace simplecpp { }; private: /** Create new token where Token::macro is set for replaced tokens */ - Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=NULL) const { + Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=nullptr) const { Token *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); @@ -2646,7 +2646,7 @@ static NonExistingFilesCache nonExistingFilesCache; #endif -static std::string _openHeader(std::ifstream &f, const std::string &path) +static std::string openHeader(std::ifstream &f, const std::string &path) { #ifdef SIMPLECPP_WINDOWS std::string simplePath = simplecpp::simplifyPath(path); @@ -2675,7 +2675,7 @@ static std::string getRelativeFileName(const std::string &sourcefile, const std: static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) { - return _openHeader(f, getRelativeFileName(sourcefile, header)); + return openHeader(f, getRelativeFileName(sourcefile, header)); } static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) @@ -2689,7 +2689,7 @@ static std::string getIncludePathFileName(const std::string &includePath, const static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) { for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string simplePath = _openHeader(f, getIncludePathFileName(*it, header)); + std::string simplePath = openHeader(f, getIncludePathFileName(*it, header)); if (!simplePath.empty()) return simplePath; } @@ -2699,7 +2699,7 @@ static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { if (isAbsolutePath(header)) - return _openHeader(f, header); + return openHeader(f, header); std::string ret; @@ -3132,7 +3132,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL expr.push_back(new Token(header2.empty() ? "0" : "1", tok->location)); } if (par) - tok = tok ? tok->next : NULL; + tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { if (outputList) { Output out(rawtok->location.files); diff --git a/test.cpp b/test.cpp index a321dfb3..abc2bf10 100644 --- a/test.cpp +++ b/test.cpp @@ -25,7 +25,7 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) -#define ASSERT_THROW(stmt, e) try { stmt; assertThrowFailed(__LINE__); } catch (const e&) {} +#define ASSERT_THROW(stmt, e) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e&) {} } while(false) static std::string pprint(const std::string &in) { @@ -86,7 +86,7 @@ static std::string readfile(const char code[], int sz=-1, simplecpp::OutputList return simplecpp::TokenList(istr,files,std::string(),outputList).stringify(); } -static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList = NULL) +static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList = nullptr) { std::istringstream istr(code); std::vector files; @@ -1459,7 +1459,7 @@ static void missingHeader2() std::istringstream istr("#include \"foo.h\"\n"); // this file exists std::vector files; std::map filedata; - filedata["foo.h"] = NULL; + filedata["foo.h"] = nullptr; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); From 444a566278870fb18335fcae12d1b92944eb3477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 21 Feb 2022 18:04:44 +0100 Subject: [PATCH 145/381] some CMake build adjustments for clang-14 (#239) --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0383423a..5c3aba80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,13 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-multichar) # TODO: fix these? add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32) + + if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14) + # not all tools support DWARF 5 yet so stick to version 4 + add_compile_options(-gdwarf-4) + # work around performance regression in clang-14 + add_compile_options(-mllvm -inline-deferral) + endif() endif() add_executable(simplecpp simplecpp.cpp main.cpp) From 6862696c6cd8985d79d8615a63bbe67eab52601d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Martins?= Date: Sat, 26 Feb 2022 19:56:45 +0000 Subject: [PATCH 146/381] consider cpp conditional expression tokens for macro usage (#240) --- simplecpp.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c995e070..055e8d7c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2912,6 +2912,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL includetokenstack.push(f->second->cfront()); } + std::map > maybeUsedMacros; + for (const Token *rawtok = nullptr; rawtok || !includetokenstack.empty();) { if (rawtok == nullptr) { rawtok = includetokenstack.top(); @@ -3077,11 +3079,13 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL bool conditionIsTrue; if (ifstates.top() == ALWAYS_FALSE || (ifstates.top() == ELSE_IS_TRUE && rawtok->str() != ELIF)) conditionIsTrue = false; - else if (rawtok->str() == IFDEF) + else if (rawtok->str() == IFDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end() || (hasInclude && rawtok->next->str() == HAS_INCLUDE)); - else if (rawtok->str() == IFNDEF) + maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + } else if (rawtok->str() == IFNDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) == macros.end() && !(hasInclude && rawtok->next->str() == HAS_INCLUDE)); - else { /*if (rawtok->str() == IF || rawtok->str() == ELIF)*/ + maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + } else { /*if (rawtok->str() == IF || rawtok->str() == ELIF)*/ TokenList expr(files); for (const Token *tok = rawtok->next; tok && tok->location.sameline(rawtok->location); tok = tok->next) { if (!tok->name) { @@ -3094,6 +3098,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool par = (tok && tok->op == '('); if (par) tok = tok->next; + maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); if (tok) { if (macros.find(tok->str()) != macros.end()) expr.push_back(new Token("1", tok->location)); @@ -3147,6 +3152,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } + maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + const Token *tmp = tok; if (!preprocessToken(expr, &tmp, macros, files, outputList)) { output.clear(); @@ -3256,7 +3263,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (macroUsage) { for (simplecpp::MacroMap::const_iterator macroIt = macros.begin(); macroIt != macros.end(); ++macroIt) { const Macro ¯o = macroIt->second; - const std::list &usage = macro.usage(); + std::list usage = macro.usage(); + const std::list& temp = maybeUsedMacros[macro.name()]; + usage.insert(usage.end(), temp.begin(), temp.end()); for (std::list::const_iterator usageIt = usage.begin(); usageIt != usage.end(); ++usageIt) { MacroUsage mu(usageIt->files, macro.valueDefinedInCode()); mu.macroName = macro.name(); From 86e4f2294e1f1808dd90eead8acf8199f9571885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 8 Mar 2022 19:59:59 +0100 Subject: [PATCH 147/381] only stringify the last line in `TokenList::readfile()` if necessary (#243) --- simplecpp.cpp | 31 ++++++++++++++++++++++++++++--- simplecpp.h | 1 + 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 055e8d7c..1ae40b96 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -509,6 +509,8 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (oldLastToken != cback()) { oldLastToken = cback(); + if (!isLastLinePreprocessor()) + continue; const std::string lastline(lastLine()); if (lastline == "# file %str%") { const Token *strtok = cback(); @@ -1203,19 +1205,42 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const { std::string ret; int count = 0; - for (const Token *tok = cback(); sameline(tok,cback()); tok = tok->previous) { + for (const Token *tok = cback(); ; tok = tok->previous) { + if (!sameline(tok, cback())) { + break; + } if (tok->comment) continue; if (++count > maxsize) return ""; if (!ret.empty()) ret.insert(0, 1, ' '); - ret.insert(0, tok->str()[0] == '\"' ? std::string("%str%") - : tok->number ? std::string("%num%") : tok->str()); + if (tok->str()[0] == '\"') + ret.insert(0, "%str%"); + else if (tok->number) + ret.insert(0, "%num%"); + else + ret.insert(0, tok->str()); } return ret; } +bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const +{ + const Token* prevTok = nullptr; + int count = 0; + for (const Token *tok = cback(); ; tok = tok->previous) { + if (!sameline(tok, cback())) + break; + if (tok->comment) + continue; + if (++count > maxsize) + return false; + prevTok = tok; + } + return prevTok && prevTok->str()[0] == '#'; +} + unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) { for (unsigned int i = 0; i < files.size(); ++i) { diff --git a/simplecpp.h b/simplecpp.h index 8f1a3cae..27abf62c 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -283,6 +283,7 @@ namespace simplecpp { void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); std::string lastLine(int maxsize=100000) const; + bool isLastLinePreprocessor(int maxsize=100000) const; unsigned int fileIndex(const std::string &filename); From b9ad0bad757798572e007ea29c55bf794e76ab55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 10 Mar 2022 08:01:51 +0100 Subject: [PATCH 148/381] some testrunner cleanups (#246) --- test.cpp | 217 +++++++++++++++++-------------------------------------- 1 file changed, 67 insertions(+), 150 deletions(-) diff --git a/test.cpp b/test.cpp index abc2bf10..55a4610e 100644 --- a/test.cpp +++ b/test.cpp @@ -549,23 +549,17 @@ static void define11() // location of expanded argument static void define_invalid_1() { - std::istringstream istr("#define A(\nB\n"); - std::vector files; - std::map filedata; + const char code[] = "#define A(\nB\n"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } static void define_invalid_2() { - std::istringstream istr("#define\nhas#"); - std::vector files; - std::map filedata; + const char code[] = "#define\nhas#"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } @@ -757,9 +751,8 @@ static void define_ifdef() "#endif\n" ")\n"; - const simplecpp::DUI dui; simplecpp::OutputList outputList; - preprocess(code, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,3,syntax_error,failed to expand 'A', it is invalid to use a preprocessor directive as macro parameter\n", toString(outputList)); } @@ -772,30 +765,25 @@ static void dollar() static void error1() { - std::istringstream istr("#error hello world!\n"); - std::vector files; - std::map filedata; + const char code[] = "#error hello world!\n"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,#error,#error hello world!\n", toString(outputList)); } static void error2() { - std::istringstream istr("#error it's an error\n"); - std::vector files; - std::map filedata; + const char code[] = "#error it's an error\n"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,#error,#error it's an error\n", toString(outputList)); } static void error3() { - std::istringstream istr("#error \"bla bla\\\n" - " bla bla.\"\n"); + const char code[] = "#error \"bla bla\\\n" + " bla bla.\"\n"; + std::istringstream istr(code); std::vector files; simplecpp::OutputList outputList; simplecpp::TokenList rawtokens(istr, files, "test.c", &outputList); @@ -805,7 +793,8 @@ static void error3() static void error4() { // "#error x\n1" - std::istringstream istr(std::string("\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31", 22)); + const std::string code("\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31", 22); + std::istringstream istr(code); std::vector files; std::map filedata; simplecpp::OutputList outputList; @@ -817,7 +806,8 @@ static void error4() static void error5() { // "#error x\n1" - std::istringstream istr(std::string("\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00", 22)); + const std::string code("\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00", 22); + std::istringstream istr(code); std::vector files; std::map filedata; simplecpp::OutputList outputList; @@ -828,37 +818,35 @@ static void error5() static void garbage() { - const simplecpp::DUI dui; simplecpp::OutputList outputList; outputList.clear(); - preprocess("#ifdef\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#ifdef\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Syntax error in #ifdef\n", toString(outputList)); outputList.clear(); - preprocess("#define TEST2() A ##\nTEST2()\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#define TEST2() A ##\nTEST2()\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'TEST2', Invalid ## usage when expanding 'TEST2'.\n", toString(outputList)); outputList.clear(); - preprocess("#define CON(a,b) a##b##\nCON(1,2)\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#define CON(a,b) a##b##\nCON(1,2)\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'CON', Invalid ## usage when expanding 'CON'.\n", toString(outputList)); } static void garbage_endif() { - const simplecpp::DUI dui; simplecpp::OutputList outputList; outputList.clear(); - preprocess("#elif A<0\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#elif A<0\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#elif without #if\n", toString(outputList)); outputList.clear(); - preprocess("#else\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#else\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#else without #if\n", toString(outputList)); outputList.clear(); - preprocess("#endif\n", dui, &outputList); + ASSERT_EQUALS("", preprocess("#endif\n", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#endif without #if\n", toString(outputList)); } @@ -980,25 +968,24 @@ static void hashhash9() "void operator >>= ( void ) { x = x >> 1 ; } ;"; ASSERT_EQUALS(expected, preprocess(code)); - const simplecpp::DUI dui; simplecpp::OutputList outputList; code = "#define A +##x\n" "A"; outputList.clear(); - preprocess(code, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); code = "#define A 2##=\n" "A"; outputList.clear(); - preprocess(code, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); code = "#define A <<##x\n" "A"; outputList.clear(); - preprocess(code, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); } @@ -1066,23 +1053,17 @@ static void hashhash13() static void hashhash_invalid_1() { - std::istringstream istr("#define f(a) (##x)\nf(1)"); - std::vector files; - std::map filedata; + const char code[] = "#define f(a) (##x)\nf(1)"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } static void hashhash_invalid_2() { - std::istringstream istr("#define f(a) (x##)\nf(1)"); - std::vector files; - std::map filedata; + const char code[] = "#define f(a) (x##)\nf(1)"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } @@ -1220,26 +1201,16 @@ static void ifDefinedNestedNoPar() static void ifDefinedInvalid1() // #50 - invalid unterminated defined { const char code[] = "#if defined(A"; - simplecpp::DUI dui; simplecpp::OutputList outputList; - std::vector files; - simplecpp::TokenList tokens2(files); - std::istringstream istr(code); - std::map filedata; - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } static void ifDefinedInvalid2() { const char code[] = "#if defined"; - simplecpp::DUI dui; simplecpp::OutputList outputList; - std::vector files; - simplecpp::TokenList tokens2(files); - std::istringstream istr(code); - std::map filedata; - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } @@ -1252,13 +1223,8 @@ static void ifDefinedHashHash() "#else\n" "#error FOO is not enabled\n" "#endif\n"; - simplecpp::DUI dui; simplecpp::OutputList outputList; - std::vector files; - simplecpp::TokenList tokens2(files); - std::istringstream istr(code); - std::map filedata; - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,4,#error,#error FOO is enabled\n", toString(outputList)); } @@ -1443,53 +1409,45 @@ static void location5() static void missingHeader1() { - const simplecpp::DUI dui; - std::istringstream istr("#include \"notexist.h\"\n"); - std::vector files; - std::map filedata; + const char code[] = "#include \"notexist.h\"\n"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,missing_header,Header not found: \"notexist.h\"\n", toString(outputList)); } static void missingHeader2() { - const simplecpp::DUI dui; - std::istringstream istr("#include \"foo.h\"\n"); // this file exists + const char code[] = "#include \"foo.h\"\n"; // this file exists + std::istringstream istr(code); std::vector files; std::map filedata; filedata["foo.h"] = nullptr; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI(), &outputList); ASSERT_EQUALS("", toString(outputList)); } static void missingHeader3() { - const simplecpp::DUI dui; - std::istringstream istr("#ifdef UNDEFINED\n#include \"notexist.h\"\n#endif\n"); // this file is not included - std::vector files; - std::map filedata; + const char code[] = "#ifdef UNDEFINED\n#include \"notexist.h\"\n#endif\n"; // this file is not included simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui, &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("", toString(outputList)); } static void nestedInclude() { - std::istringstream istr("#include \"test.h\"\n"); + const char code[] = "#include \"test.h\"\n"; + std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files,"test.h"); std::map filedata; filedata["test.h"] = &rawtokens; - const simplecpp::DUI dui; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, dui, &outputList); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } @@ -1499,13 +1457,7 @@ static void multiline1() const char code[] = "#define A \\\n" "1\n" "A"; - const simplecpp::DUI dui; - std::istringstream istr(code); - std::vector files; - std::map filedata; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, dui); - ASSERT_EQUALS("\n\n1", tokens2.stringify()); + ASSERT_EQUALS("\n\n1", preprocess(code, simplecpp::DUI())); } static void multiline2() @@ -1513,7 +1465,6 @@ static void multiline2() const char code[] = "#define A /*\\\n" "*/1\n" "A"; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1521,7 +1472,7 @@ static void multiline2() rawtokens.removeComments(); std::map filedata; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, dui); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -1530,7 +1481,6 @@ static void multiline3() // #28 - macro with multiline comment const char code[] = "#define A /*\\\n" " */ 1\n" "A"; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1538,7 +1488,7 @@ static void multiline3() // #28 - macro with multiline comment rawtokens.removeComments(); std::map filedata; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, dui); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -1548,7 +1498,6 @@ static void multiline4() // #28 - macro with multiline comment " /*\\\n" " */ 1\n" "A"; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1556,7 +1505,7 @@ static void multiline4() // #28 - macro with multiline comment rawtokens.removeComments(); std::map filedata; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, dui); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); ASSERT_EQUALS("\n\n\n1", tokens2.stringify()); } @@ -1564,7 +1513,6 @@ static void multiline5() // column { const char code[] = "#define A\\\n" "("; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1577,7 +1525,6 @@ static void multiline6() // multiline string in macro const char code[] = "#define string (\"\\\n" "x\")\n" "string\n"; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1591,7 +1538,6 @@ static void multiline7() // multiline string in macro const char code[] = "#define A(X) aaa { f(\"\\\n" "a\"); }\n" "A(1)"; - const simplecpp::DUI dui; std::istringstream istr(code); std::vector files; simplecpp::TokenList rawtokens(istr,files); @@ -1631,8 +1577,7 @@ static void nullDirective1() "#endif\n" "x = a;\n"; - const simplecpp::DUI dui; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); } static void nullDirective2() @@ -1643,8 +1588,7 @@ static void nullDirective2() "#endif\n" "x = a;\n"; - const simplecpp::DUI dui; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); } static void nullDirective3() @@ -1655,8 +1599,7 @@ static void nullDirective3() "#endif\n" "x = a;\n"; - const simplecpp::DUI dui; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, dui)); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); } static void include1() @@ -1745,8 +1688,7 @@ static void include5() // #3 - handle #include MACRO filedata["3.h"] = &rawtokens_h; simplecpp::TokenList out(files); - simplecpp::DUI dui; - simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + simplecpp::preprocess(out, rawtokens_c, files, filedata, simplecpp::DUI()); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -1763,8 +1705,7 @@ static void include6() // #57 - incomplete macro #include MACRO(,) filedata["57.c"] = &rawtokens; simplecpp::TokenList out(files); - simplecpp::DUI dui; - simplecpp::preprocess(out, rawtokens, files, filedata, dui); + simplecpp::preprocess(out, rawtokens, files, filedata, simplecpp::DUI()); } @@ -1797,13 +1738,8 @@ static void include8() // #include MACRO(X) const char code[] = "#define INCLUDE_LOCATION ../somewhere\n" "#define INCLUDE_FILE(F) \n" "#include INCLUDE_FILE(header)\n"; - - std::istringstream istr(code); - std::vector files; - std::map filedata; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,3,missing_header,Header not found: <../somewhere/header.h>\n", toString(outputList)); } @@ -1986,12 +1922,11 @@ static void tokenMacro1() { const char code[] = "#define A 123\n" "A"; - const simplecpp::DUI dui; std::vector files; std::map filedata; std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); ASSERT_EQUALS("A", tokenList.cback()->macro); } @@ -1999,12 +1934,11 @@ static void tokenMacro2() { const char code[] = "#define ADD(X,Y) X+Y\n" "ADD(1,2)"; - const simplecpp::DUI dui; std::vector files; std::map filedata; std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("", tok->macro); @@ -2021,12 +1955,11 @@ static void tokenMacro3() const char code[] = "#define ADD(X,Y) X+Y\n" "#define FRED 1\n" "ADD(FRED,2)"; - const simplecpp::DUI dui; std::vector files; std::map filedata; std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("FRED", tok->macro); @@ -2043,12 +1976,11 @@ static void tokenMacro4() const char code[] = "#define A B\n" "#define B 1\n" "A"; - const simplecpp::DUI dui; std::vector files; std::map filedata; std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("A", tok->macro); @@ -2059,12 +1991,11 @@ static void tokenMacro5() const char code[] = "#define SET_BPF(code) (code)\n" "#define SET_BPF_JUMP(code) SET_BPF(D | code)\n" "SET_BPF_JUMP(A | B | C);"; - const simplecpp::DUI dui; std::vector files; std::map filedata; std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); + simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront()->next; ASSERT_EQUALS("D", tok->str()); ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); @@ -2072,30 +2003,20 @@ static void tokenMacro5() static void undef() { - std::istringstream istr("#define A\n" + const char code[] = "#define A\n" "#undef A\n" "#ifdef A\n" "123\n" - "#endif"); - const simplecpp::DUI dui; - std::vector files; - std::map filedata; - simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, dui); - ASSERT_EQUALS("", tokenList.stringify()); + "#endif"; + ASSERT_EQUALS("", preprocess(code, simplecpp::DUI())); } static void userdef() { - std::istringstream istr("#ifdef A\n123\n#endif\n"); + const char code[] = "#ifdef A\n123\n#endif\n"; simplecpp::DUI dui; dui.defines.push_back("A=1"); - std::vector files; - const simplecpp::TokenList tokens1 = simplecpp::TokenList(istr, files); - std::map filedata; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, tokens1, files, filedata, dui); - ASSERT_EQUALS("\n123", tokens2.stringify()); + ASSERT_EQUALS("\n123", preprocess(code, dui)); } static void utf8() @@ -2116,13 +2037,9 @@ static void unicode() static void warning() { - std::istringstream istr("#warning MSG\n1"); - std::vector files; - std::map filedata; + const char code[] = "#warning MSG\n1"; simplecpp::OutputList outputList; - simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); - ASSERT_EQUALS("\n1", tokens2.stringify()); + ASSERT_EQUALS("\n1", preprocess(code, simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,#warning,#warning MSG\n", toString(outputList)); } @@ -2216,17 +2133,17 @@ static void preprocessSizeOf() { simplecpp::OutputList outputList; - preprocess("#if 3 > sizeof", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); outputList.clear(); - preprocess("#if 3 > sizeof A", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof A", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); outputList.clear(); - preprocess("#if 3 > sizeof(int", simplecpp::DUI(), &outputList); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof(int", simplecpp::DUI(), &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, invalid sizeof expression\n", toString(outputList)); } From 531389792b4ce1c792b4c609e0ae4aba1d8170c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 10 Mar 2022 08:09:36 +0100 Subject: [PATCH 149/381] CI-windows.yml: added "windows-2022" and missing exitcode checks (#247) --- .github/workflows/CI-windows.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index b9df721f..264a09c6 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -15,8 +15,7 @@ jobs: build: strategy: matrix: - # windows 2016 should default to VS 2017. Not supported by setup-msbuild - os: [windows-2019] + os: [windows-2019, windows-2022] fail-fast: false runs-on: ${{ matrix.os }} @@ -28,15 +27,22 @@ jobs: uses: microsoft/setup-msbuild@v1.0.2 - name: Run cmake + if: matrix.os == 'windows-2019' run: | - cmake -G "Visual Studio 16" . -A x64 + cmake -G "Visual Studio 16 2019" -A x64 . || exit /b !errorlevel! + dir + + - name: Run cmake + if: matrix.os == 'windows-2022' + run: | + cmake -G "Visual Studio 17 2022" -A x64 . || exit /b !errorlevel! dir - name: Build run: | - msbuild -m simplecpp.sln /p:Configuration=Release /p:Platform=x64 + msbuild -m simplecpp.sln /p:Configuration=Release /p:Platform=x64 || exit /b !errorlevel! - name: Test run: | - .\Release\testrunner.exe + .\Release\testrunner.exe || exit /b !errorlevel! From 60890cd50c1c801db7c1347bee3b11b6aa2dad4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 12 Mar 2022 11:58:22 +0100 Subject: [PATCH 150/381] fixed some Cppcheck warnings / use `Macro` forward declaration in header (#252) * fixed missingMemberCopy warning for mExpandedFrom in Token copy constructor * added forward declaration for Macro to avoid void* usage * fixed constParameter Cppcheck warning --- simplecpp.h | 9 +++++---- test.cpp | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 27abf62c..db8cb09a 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -48,6 +48,7 @@ namespace simplecpp { typedef std::string TokenString; + class Macro; /** * Location in source code @@ -106,7 +107,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), location(tok.location), previous(nullptr), next(nullptr), string(tok.string) { + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { } void flags() { @@ -152,11 +153,11 @@ namespace simplecpp { return tok; } - void setExpandedFrom(const Token *tok, const void* m) { + void setExpandedFrom(const Token *tok, const Macro* m) { mExpandedFrom = tok->mExpandedFrom; mExpandedFrom.insert(m); } - bool isExpandedFrom(const void* m) const { + bool isExpandedFrom(const Macro* m) const { return mExpandedFrom.find(m) != mExpandedFrom.end(); } @@ -165,7 +166,7 @@ namespace simplecpp { private: TokenString string; - std::set mExpandedFrom; + std::set mExpandedFrom; // Not implemented - prevent assignment Token &operator=(const Token &tok); diff --git a/test.cpp b/test.cpp index 55a4610e..d5857ef9 100644 --- a/test.cpp +++ b/test.cpp @@ -63,7 +63,7 @@ static void assertThrowFailed(int line) std::cerr << "exception not thrown" << std::endl; } -static void testcase(const std::string &name, void (*f)(), int argc, char **argv) +static void testcase(const std::string &name, void (*f)(), int argc, char * const *argv) { if (argc == 1) f(); From 2860eed9d9f0a3ded0cd0ff196f69331d60732f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 12 Mar 2022 12:02:01 +0100 Subject: [PATCH 151/381] more testrunner cleanups (#250) --- test.cpp | 158 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 86 insertions(+), 72 deletions(-) diff --git a/test.cpp b/test.cpp index d5857ef9..5ded0486 100644 --- a/test.cpp +++ b/test.cpp @@ -86,7 +86,19 @@ static std::string readfile(const char code[], int sz=-1, simplecpp::OutputList return simplecpp::TokenList(istr,files,std::string(),outputList).stringify(); } -static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList = nullptr) +static simplecpp::TokenList makeTokenList(const char code[], std::vector &files, const std::string &file) +{ + std::istringstream istr(code); + return simplecpp::TokenList(istr,files,file); +} + +static simplecpp::TokenList makeTokenList(const char code[]) +{ + std::vector files; + return makeTokenList(code, files, std::string()); +} + +static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { std::istringstream istr(code); std::vector files; @@ -100,7 +112,17 @@ static std::string preprocess(const char code[], const simplecpp::DUI &dui, simp static std::string preprocess(const char code[]) { - return preprocess(code, simplecpp::DUI()); + return preprocess(code, simplecpp::DUI(), nullptr); +} + +static std::string preprocess(const char code[], const simplecpp::DUI &dui) +{ + return preprocess(code, dui, nullptr); +} + +static std::string preprocess(const char code[], simplecpp::OutputList *outputList) +{ + return preprocess(code, simplecpp::DUI(), outputList); } static std::string toString(const simplecpp::OutputList &outputList) @@ -173,15 +195,13 @@ static void builtin() static std::string testConstFold(const char code[]) { - std::istringstream istr(code); - std::vector files; - simplecpp::TokenList expr(istr, files); try { + simplecpp::TokenList expr = makeTokenList(code); expr.constFold(); + return expr.stringify(); } catch (std::exception &) { return "exception"; } - return expr.stringify(); } static void characterLiteral() @@ -551,7 +571,7 @@ static void define_invalid_1() { const char code[] = "#define A(\nB\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } @@ -559,7 +579,7 @@ static void define_invalid_2() { const char code[] = "#define\nhas#"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); } @@ -752,7 +772,7 @@ static void define_ifdef() ")\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,3,syntax_error,failed to expand 'A', it is invalid to use a preprocessor directive as macro parameter\n", toString(outputList)); } @@ -767,7 +787,7 @@ static void error1() { const char code[] = "#error hello world!\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,#error,#error hello world!\n", toString(outputList)); } @@ -775,7 +795,7 @@ static void error2() { const char code[] = "#error it's an error\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,#error,#error it's an error\n", toString(outputList)); } @@ -821,15 +841,15 @@ static void garbage() simplecpp::OutputList outputList; outputList.clear(); - ASSERT_EQUALS("", preprocess("#ifdef\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#ifdef\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,Syntax error in #ifdef\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#define TEST2() A ##\nTEST2()\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#define TEST2() A ##\nTEST2()\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'TEST2', Invalid ## usage when expanding 'TEST2'.\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#define CON(a,b) a##b##\nCON(1,2)\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#define CON(a,b) a##b##\nCON(1,2)\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'CON', Invalid ## usage when expanding 'CON'.\n", toString(outputList)); } @@ -838,15 +858,15 @@ static void garbage_endif() simplecpp::OutputList outputList; outputList.clear(); - ASSERT_EQUALS("", preprocess("#elif A<0\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#elif A<0\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#elif without #if\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#else\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#else\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#else without #if\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#endif\n", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#endif\n", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,#endif without #if\n", toString(outputList)); } @@ -973,19 +993,19 @@ static void hashhash9() code = "#define A +##x\n" "A"; outputList.clear(); - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); code = "#define A 2##=\n" "A"; outputList.clear(); - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); code = "#define A <<##x\n" "A"; outputList.clear(); - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); } @@ -1055,7 +1075,7 @@ static void hashhash_invalid_1() { const char code[] = "#define f(a) (##x)\nf(1)"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } @@ -1063,7 +1083,7 @@ static void hashhash_invalid_2() { const char code[] = "#define f(a) (x##)\nf(1)"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } @@ -1202,7 +1222,7 @@ static void ifDefinedInvalid1() // #50 - invalid unterminated defined { const char code[] = "#if defined(A"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } @@ -1210,7 +1230,7 @@ static void ifDefinedInvalid2() { const char code[] = "#if defined"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); } @@ -1224,7 +1244,7 @@ static void ifDefinedHashHash() "#error FOO is not enabled\n" "#endif\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,4,#error,#error FOO is enabled\n", toString(outputList)); } @@ -1411,7 +1431,7 @@ static void missingHeader1() { const char code[] = "#include \"notexist.h\"\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,missing_header,Header not found: \"notexist.h\"\n", toString(outputList)); } @@ -1432,7 +1452,7 @@ static void missingHeader3() { const char code[] = "#ifdef UNDEFINED\n#include \"notexist.h\"\n#endif\n"; // this file is not included simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("", toString(outputList)); } @@ -1457,7 +1477,7 @@ static void multiline1() const char code[] = "#define A \\\n" "1\n" "A"; - ASSERT_EQUALS("\n\n1", preprocess(code, simplecpp::DUI())); + ASSERT_EQUALS("\n\n1", preprocess(code)); } static void multiline2() @@ -1513,11 +1533,9 @@ static void multiline5() // column { const char code[] = "#define A\\\n" "("; - std::istringstream istr(code); - std::vector files; - simplecpp::TokenList rawtokens(istr,files); + const simplecpp::TokenList rawtokens = makeTokenList(code); ASSERT_EQUALS("# define A (", rawtokens.stringify()); - ASSERT_EQUALS(11, rawtokens.back()->location.col); + ASSERT_EQUALS(11, rawtokens.cback()->location.col); } static void multiline6() // multiline string in macro @@ -1525,9 +1543,7 @@ static void multiline6() // multiline string in macro const char code[] = "#define string (\"\\\n" "x\")\n" "string\n"; - std::istringstream istr(code); - std::vector files; - simplecpp::TokenList rawtokens(istr,files); + const simplecpp::TokenList rawtokens = makeTokenList(code); ASSERT_EQUALS("# define string ( \"x\" )\n" "\n" "string", rawtokens.stringify()); @@ -1538,9 +1554,7 @@ static void multiline7() // multiline string in macro const char code[] = "#define A(X) aaa { f(\"\\\n" "a\"); }\n" "A(1)"; - std::istringstream istr(code); - std::vector files; - simplecpp::TokenList rawtokens(istr,files); + const simplecpp::TokenList rawtokens = makeTokenList(code); ASSERT_EQUALS("# define A ( X ) aaa { f ( \"a\" ) ; }\n" "\n" "A ( 1 )", rawtokens.stringify()); @@ -1577,7 +1591,7 @@ static void nullDirective1() "#endif\n" "x = a;\n"; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code)); } static void nullDirective2() @@ -1588,7 +1602,7 @@ static void nullDirective2() "#endif\n" "x = a;\n"; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code)); } static void nullDirective3() @@ -1599,7 +1613,7 @@ static void nullDirective3() "#endif\n" "x = a;\n"; - ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code, simplecpp::DUI())); + ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code)); } static void include1() @@ -1622,11 +1636,8 @@ static void include3() // #16 - crash when expanding macro from header std::vector files; - std::istringstream istr_c(code_c); - simplecpp::TokenList rawtokens_c(istr_c, files, "A.c"); - - std::istringstream istr_h(code_h); - simplecpp::TokenList rawtokens_h(istr_h, files, "A.h"); + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("A.c", files[0]); @@ -1650,11 +1661,8 @@ static void include4() // #27 - -include std::vector files; - std::istringstream istr_c(code_c); - simplecpp::TokenList rawtokens_c(istr_c, files, "27.c"); - - std::istringstream istr_h(code_h); - simplecpp::TokenList rawtokens_h(istr_h, files, "27.h"); + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "27.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "27.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("27.c", files[0]); @@ -1678,10 +1686,13 @@ static void include5() // #3 - handle #include MACRO const char code_h[] = "123\n"; std::vector files; - std::istringstream istr_c(code_c); - simplecpp::TokenList rawtokens_c(istr_c, files, "3.c"); - std::istringstream istr_h(code_h); - simplecpp::TokenList rawtokens_h(istr_h, files, "3.h"); + + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); + + ASSERT_EQUALS(2U, files.size()); + ASSERT_EQUALS("3.c", files[0]); + ASSERT_EQUALS("3.h", files[1]); std::map filedata; filedata["3.c"] = &rawtokens_c; @@ -1698,8 +1709,11 @@ static void include6() // #57 - incomplete macro #include MACRO(,) const char code[] = "#define MACRO(X,Y) X##Y\n#include MACRO(,)\n"; std::vector files; - std::istringstream istr(code); - simplecpp::TokenList rawtokens(istr, files, "57.c"); + + simplecpp::TokenList rawtokens = makeTokenList(code, files, "57.c"); + + ASSERT_EQUALS(1U, files.size()); + ASSERT_EQUALS("57.c", files[0]); std::map filedata; filedata["57.c"] = &rawtokens; @@ -1716,10 +1730,13 @@ static void include7() // #include MACRO const char code_h[] = "123\n"; std::vector files; - std::istringstream istr_c(code_c); - simplecpp::TokenList rawtokens_c(istr_c, files, "3.c"); - std::istringstream istr_h(code_h); - simplecpp::TokenList rawtokens_h(istr_h, files, "3.h"); + + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); + + ASSERT_EQUALS(2U, files.size()); + ASSERT_EQUALS("3.c", files[0]); + ASSERT_EQUALS("3.h", files[1]); std::map filedata; filedata["3.c"] = &rawtokens_c; @@ -1739,7 +1756,7 @@ static void include8() // #include MACRO(X) "#define INCLUDE_FILE(F) \n" "#include INCLUDE_FILE(header)\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,3,missing_header,Header not found: <../somewhere/header.h>\n", toString(outputList)); } @@ -1898,11 +1915,8 @@ static void stringify1() std::vector files; - std::istringstream istr_c(code_c); - simplecpp::TokenList rawtokens_c(istr_c, files, "A.c"); - - std::istringstream istr_h(code_h); - simplecpp::TokenList rawtokens_h(istr_h, files, "A.h"); + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("A.c", files[0]); @@ -2008,7 +2022,7 @@ static void undef() "#ifdef A\n" "123\n" "#endif"; - ASSERT_EQUALS("", preprocess(code, simplecpp::DUI())); + ASSERT_EQUALS("", preprocess(code)); } static void userdef() @@ -2039,7 +2053,7 @@ static void warning() { const char code[] = "#warning MSG\n1"; simplecpp::OutputList outputList; - ASSERT_EQUALS("\n1", preprocess(code, simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("\n1", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,#warning,#warning MSG\n", toString(outputList)); } @@ -2133,17 +2147,17 @@ static void preprocessSizeOf() { simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess("#if 3 > sizeof", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#if 3 > sizeof A", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof A", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, missing sizeof argument\n", toString(outputList)); outputList.clear(); - ASSERT_EQUALS("", preprocess("#if 3 > sizeof(int", simplecpp::DUI(), &outputList)); + ASSERT_EQUALS("", preprocess("#if 3 > sizeof(int", &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, invalid sizeof expression\n", toString(outputList)); } From 47bc4648be7ac3c6b4cc9c49833ee710c3aeea94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Mar 2022 09:34:35 +0100 Subject: [PATCH 152/381] added support for built-in defines __DATE__ and __TIME__ (#251) --- simplecpp.cpp | 27 +++++++++++++++++++++++++++ test.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1ae40b96..f6f55c5d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -2867,6 +2868,28 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token return true; } +static void getLocaltime(struct tm <ime) { + time_t t; + time(&t); +#ifndef _WIN32 + localtime_r(&t, <ime); +#else + localtime_s(<ime, &t); +#endif +} + +static std::string getDateDefine(struct tm *timep) { + char buf[] = "??? ?? ????"; + strftime(buf, sizeof(buf), "%b %d %Y", timep); + return std::string("\"").append(buf).append("\""); +} + +static std::string getTimeDefine(struct tm *timep) { + char buf[] = "??:??:??"; + strftime(buf, sizeof(buf), "%T", timep); + return std::string("\"").append(buf).append("\""); +} + void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage, std::list *ifCond) { std::map sizeOfType(rawtokens.sizeOfType); @@ -2909,6 +2932,10 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files))); macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", files))); macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files))); + struct tm ltime = {}; + getLocaltime(ltime); + macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(<ime), files))); + macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), files))); if (dui.std == "c++11") macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201103L", files))); diff --git a/test.cpp b/test.cpp index 5ded0486..990547a0 100644 --- a/test.cpp +++ b/test.cpp @@ -2161,6 +2161,47 @@ static void preprocessSizeOf() ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, invalid sizeof expression\n", toString(outputList)); } +static void timeDefine() +{ + const char code[] = "__TIME__"; + const std::string t = preprocess(code); + // "19:09:53" + ASSERT_EQUALS(10, t.size()); + // TODO: split string and check proper ranges instead + ASSERT_EQUALS('"', t[0]); + ASSERT_EQUALS(true, isdigit(t[1]) != 0); + ASSERT_EQUALS(true, isdigit(t[2]) != 0); + ASSERT_EQUALS(':', t[3]); + ASSERT_EQUALS(true, isdigit(t[4]) != 0); + ASSERT_EQUALS(true, isdigit(t[5]) != 0); + ASSERT_EQUALS(':', t[6]); + ASSERT_EQUALS(true, isdigit(t[7]) != 0); + ASSERT_EQUALS(true, isdigit(t[8]) != 0); + ASSERT_EQUALS('"', t[9]); +} + +static void dateDefine() +{ + const char code[] = "__DATE__"; + const std::string dt = preprocess(code); + // "\"Mar 11 2022\"" + ASSERT_EQUALS(13, dt.size()); + // TODO: split string and check proper ranges instead + ASSERT_EQUALS('"', dt[0]); + ASSERT_EQUALS(true, dt[1] >= 'A' && dt[1] <= 'Z'); // uppercase letter + ASSERT_EQUALS(true, dt[2] >= 'a' && dt[2] <= 'z'); // lowercase letter + ASSERT_EQUALS(true, dt[3] >= 'a' && dt[3] <= 'z'); // lowercase letter + ASSERT_EQUALS(' ', dt[4]); + ASSERT_EQUALS(true, isdigit(dt[5]) != 0); + ASSERT_EQUALS(true, isdigit(dt[6]) != 0); + ASSERT_EQUALS(' ', dt[7]); + ASSERT_EQUALS(true, isdigit(dt[8]) != 0); + ASSERT_EQUALS(true, isdigit(dt[9]) != 0); + ASSERT_EQUALS(true, isdigit(dt[10]) != 0); + ASSERT_EQUALS(true, isdigit(dt[11]) != 0); + ASSERT_EQUALS('"', dt[12]); +} + int main(int argc, char **argv) { TEST_CASE(backslash); @@ -2342,5 +2383,8 @@ int main(int argc, char **argv) TEST_CASE(preprocessSizeOf); + TEST_CASE(timeDefine); + TEST_CASE(dateDefine); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From a58ae6283b9b62ba694034576707c612c3cbce6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 14 Mar 2022 19:21:03 +0100 Subject: [PATCH 153/381] exposed -std to define mapping in interface / add support for more -std values / added setting of __STDC_VERSION__ (#242) --- simplecpp.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++--------- simplecpp.h | 6 +++++ test.cpp | 23 ++++++++++++++++++ 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f6f55c5d..467e94eb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1254,11 +1254,11 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) namespace simplecpp { - class Macro; + class Macro; #if __cplusplus >= 201103L - using MacroMap = std::unordered_map; + using MacroMap = std::unordered_map; #else - typedef std::map MacroMap; + typedef std::map MacroMap; #endif class Macro { @@ -2937,14 +2937,16 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(<ime), files))); macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), files))); - if (dui.std == "c++11") - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201103L", files))); - else if (dui.std == "c++14") - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201402L", files))); - else if (dui.std == "c++17") - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "201703L", files))); - else if (dui.std == "c++20") - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", "202002L", files))); + if (!dui.std.empty()) { + std::string std_def = simplecpp::getCStdString(dui.std); + if (!std_def.empty()) { + macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, files))); + } else { + std_def = simplecpp::getCppStdString(dui.std); + if (!std_def.empty()) + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, files))); + } + } // TRUE => code in current #if block should be kept // ELSE_IS_TRUE => code in current #if block should be dropped. the code in the #else should be kept. @@ -3336,6 +3338,47 @@ void simplecpp::cleanup(std::map &filedata) filedata.clear(); } +std::string simplecpp::getCStdString(const std::string &std) +{ + if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") { + // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments + return ""; + } + if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99"|| std == "gnu9x") + return "199901L"; + if (std == "c11" || std == "c1x" || std == "iso9899:2011" || std == "gnu11" || std == "gnu1x") + return "201112L"; + if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") + return "201710L"; + if (std == "c2x" || std == "gnu2x") { + // Clang 11 returns "201710L" + return "202000L"; + } + return ""; +} + +std::string simplecpp::getCppStdString(const std::string &std) +{ + if (std == "c++98" || std == "c++03" || std == "gnu++98" || std == "gnu++03") + return "199711L"; + if (std == "c++11" || std == "gnu++11" || std == "c++0x" || std == "gnu++0x") + return "201103L"; + if (std == "c++14" || std == "c++1y" || std == "gnu++14" || std == "gnu++1y") + return "201402L"; + if (std == "c++17" || std == "c++1z" || std == "gnu++17" || std == "gnu++1z") + return "201703L"; + if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") { + // GCC 10 returns "201703L" + return "202002L"; + } + /* + if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") { + // supported by GCC 11+ + return ""; + } */ + return ""; +} + #if (__cplusplus < 201103L) && !defined(__APPLE__) #undef nullptr #endif diff --git a/simplecpp.h b/simplecpp.h index db8cb09a..4b97aa35 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -351,6 +351,12 @@ namespace simplecpp { /** Convert Cygwin path to Windows path */ SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); + + /** Returns the __STDC_VERSION__ value for a given standard */ + SIMPLECPP_LIB static std::string getCStdString(const std::string &std); + + /** Returns the __cplusplus value for a given standard */ + SIMPLECPP_LIB static std::string getCppStdString(const std::string &std); } #if (__cplusplus < 201103L) && !defined(__APPLE__) diff --git a/test.cpp b/test.cpp index 990547a0..9e12d48e 100644 --- a/test.cpp +++ b/test.cpp @@ -2202,6 +2202,26 @@ static void dateDefine() ASSERT_EQUALS('"', dt[12]); } +static void stdcVersionDefine() +{ + const char code[] = "#if defined(__STDC_VERSION__)\n" + " __STDC_VERSION__\n" + "#endif\n"; + simplecpp::DUI dui; + dui.std = "c11"; + ASSERT_EQUALS("\n201112L", preprocess(code, dui)); +} + +static void cpluscplusDefine() +{ + const char code[] = "#if defined(__cplusplus)\n" + " __cplusplus\n" + "#endif\n"; + simplecpp::DUI dui; + dui.std = "c++11"; + ASSERT_EQUALS("\n201103L", preprocess(code, dui)); +} + int main(int argc, char **argv) { TEST_CASE(backslash); @@ -2386,5 +2406,8 @@ int main(int argc, char **argv) TEST_CASE(timeDefine); TEST_CASE(dateDefine); + TEST_CASE(stdcVersionDefine); + TEST_CASE(cpluscplusDefine); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From e4cb74850479b910c773ed410224beeb07440550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 16 Mar 2022 20:29:21 +0100 Subject: [PATCH 154/381] .gitignore: added CLion output files (#253) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index f3bb1e3a..183545f1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ *.app simplecpp testrunner + +# CLion +/.idea +/cmake-build-* From e7d85ea922dd6df3193c60385f087bfcb4d5b554 Mon Sep 17 00:00:00 2001 From: Patrick Dowling Date: Fri, 18 Mar 2022 08:19:17 +0100 Subject: [PATCH 155/381] Special-case ## pasting to string/character constants (issue #168) (#255) This enables use of macros to add literals/operator "". --- run-tests.py | 2 +- simplecpp.cpp | 114 +++++++++++++++++++++++++++++------------------ test.cpp | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 43 deletions(-) diff --git a/run-tests.py b/run-tests.py index 0211ed04..3dc4a9b7 100644 --- a/run-tests.py +++ b/run-tests.py @@ -38,7 +38,7 @@ def cleanup(out): 'has_attribute.cpp', 'header_lookup1.c', # missing include 'line-directive-output.c', - 'macro_paste_hashhash.c', + # 'macro_paste_hashhash.c', 'microsoft-ext.c', 'normalize-3.c', # gcc has different output \uAC00 vs \U0000AC00 on cygwin/linux 'pr63831-1.c', # __has_attribute => works differently on cygwin/linux diff --git a/simplecpp.cpp b/simplecpp.cpp index 467e94eb..ce3b156c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -58,6 +58,17 @@ static bool isOct(const std::string &s) return s.size()>1 && (s[0]=='0') && (s[1] >= '0') && (s[1] < '8'); } +static bool isStringLiteral(const std::string &s) +{ + return s.size() > 1 && (s[0]=='\"') && (*s.rbegin()=='\"'); +} + +static bool isCharLiteral(const std::string &s) +{ + // char literal patterns can include 'a', '\t', '\000', '\xff', 'abcd', and maybe '' + // This only checks for the surrounding '' but doesn't parse the content. + return s.size() > 1 && (s[0]=='\'') && (*s.rbegin()=='\''); +} static const simplecpp::TokenString DEFINE("define"); static const simplecpp::TokenString UNDEF("undef"); @@ -1922,7 +1933,8 @@ namespace simplecpp { throw invalidHashHash(tok->location, name()); bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; - if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual) + bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); + if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar) throw invalidHashHash(tok->location, name()); Token *B = tok->next->next; @@ -1933,55 +1945,73 @@ namespace simplecpp { (!canBeConcatenatedWithEqual && B->op == '=')) throw invalidHashHash(tok->location, name()); - std::string strAB; - - const bool varargs = variadic && args.size() >= 1U && B->str() == args[args.size()-1U]; + // Superficial check; more in-depth would in theory be possible _after_ expandArg + if (canBeConcatenatedStringOrChar && (B->number || !B->name)) + throw invalidHashHash(tok->location, name()); TokenList tokensB(files); - if (expandArg(&tokensB, B, parametertokens)) { - if (tokensB.empty()) - strAB = A->str(); - else if (varargs && A->op == ',') { - strAB = ","; + const Token *nextTok = B->next; + + if (canBeConcatenatedStringOrChar) { + // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. + // TODO The question is whether the ## or varargs may still apply, and how to provoke? + if (expandArg(&tokensB, B, parametertokens)) { + for (Token *b = tokensB.front(); b; b = b->next) + b->location = loc; } else { - strAB = A->str() + tokensB.cfront()->str(); - tokensB.deleteToken(tokensB.front()); + tokensB.push_back(new Token(*B)); + tokensB.back()->location = loc; } - } else { - strAB = A->str() + B->str(); - } - - const Token *nextTok = B->next; - if (varargs && tokensB.empty() && tok->previous->str() == ",") - output->deleteToken(A); - else if (strAB != "," && macros.find(strAB) == macros.end()) { - A->setstr(strAB); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; output->takeTokens(tokensB); - } else if (nextTok->op == '#' && nextTok->next->op == '#') { - TokenList output2(files); - output2.push_back(new Token(strAB, tok->location)); - nextTok = expandHashHash(&output2, loc, nextTok, macros, expandedmacros, parametertokens); - output->deleteToken(A); - output->takeTokens(output2); } else { - output->deleteToken(A); - TokenList tokens(files); - tokens.push_back(new Token(strAB, tok->location)); - // for function like macros, push the (...) - if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { - const MacroMap::const_iterator it = macros.find(strAB); - if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { - const Token *tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); - if (tok2) - nextTok = tok2->next; + std::string strAB; + + const bool varargs = variadic && args.size() >= 1U && B->str() == args[args.size()-1U]; + + if (expandArg(&tokensB, B, parametertokens)) { + if (tokensB.empty()) + strAB = A->str(); + else if (varargs && A->op == ',') { + strAB = ","; + } else { + strAB = A->str() + tokensB.cfront()->str(); + tokensB.deleteToken(tokensB.front()); + } + } else { + strAB = A->str() + B->str(); + } + + if (varargs && tokensB.empty() && tok->previous->str() == ",") + output->deleteToken(A); + else if (strAB != "," && macros.find(strAB) == macros.end()) { + A->setstr(strAB); + for (Token *b = tokensB.front(); b; b = b->next) + b->location = loc; + output->takeTokens(tokensB); + } else if (nextTok->op == '#' && nextTok->next->op == '#') { + TokenList output2(files); + output2.push_back(new Token(strAB, tok->location)); + nextTok = expandHashHash(&output2, loc, nextTok, macros, expandedmacros, parametertokens); + output->deleteToken(A); + output->takeTokens(output2); + } else { + output->deleteToken(A); + TokenList tokens(files); + tokens.push_back(new Token(strAB, tok->location)); + // for function like macros, push the (...) + if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { + const MacroMap::const_iterator it = macros.find(strAB); + if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { + const Token *tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); + if (tok2) + nextTok = tok2->next; + } } + expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens); + for (Token *b = tokensB.front(); b; b = b->next) + b->location = loc; + output->takeTokens(tokensB); } - expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens); - for (Token *b = tokensB.front(); b; b = b->next) - b->location = loc; - output->takeTokens(tokensB); } return nextTok; diff --git a/test.cpp b/test.cpp index 9e12d48e..02de49b9 100644 --- a/test.cpp +++ b/test.cpp @@ -1071,6 +1071,105 @@ static void hashhash13() ASSERT_EQUALS("\n& ab", preprocess(code2)); } +static void hashhash_string_literal() +{ + const char code[] = + "#define UL(x) x##_ul\n" + "\"ABC\"_ul;\n" + "UL(\"ABC\");"; + + ASSERT_EQUALS("\n\"ABC\" _ul ;\n\"ABC\" _ul ;", preprocess(code)); +} + +static void hashhash_string_wrapped() +{ + const char code[] = + "#define CONCAT(a,b) a##b\n" + "#define STR(x) CONCAT(x,s)\n" + "STR(\"ABC\");"; + + ASSERT_EQUALS("\n\n\"ABC\" s ;", preprocess(code)); +} + +static void hashhash_char_literal() +{ + const char code[] = + "#define CH(x) x##_ch\n" + "CH('a');"; + + ASSERT_EQUALS("\n'a' _ch ;", preprocess(code)); +} + +static void hashhash_multichar_literal() +{ + const char code[] = + "#define CH(x) x##_ch\n" + "CH('abcd');"; + + ASSERT_EQUALS("\n'abcd' _ch ;", preprocess(code)); +} + +static void hashhash_char_escaped() +{ + const char code[] = + "#define CH(x) x##_ch\n" + "CH('\\'');"; + + ASSERT_EQUALS("\n'\\'' _ch ;", preprocess(code)); +} + +static void hashhash_string_nothing() +{ + const char code[] = + "#define CONCAT(a,b) a##b\n" + "CONCAT(\"ABC\",);"; + + ASSERT_EQUALS("\n\"ABC\" ;", preprocess(code)); +} + +static void hashhash_string_char() +{ + const char code[] = + "#define CONCAT(a,b) a##b\n" + "CONCAT(\"ABC\", 'c');"; + + // This works, but maybe shouldn't since the result isn't useful. + ASSERT_EQUALS("\n\"ABC\" 'c' ;", preprocess(code)); +} + +static void hashhash_string_name() +{ + const char code[] = + "#define CONCAT(a,b) a##b\n" + "#define LIT _literal\n" + "CONCAT(\"string\", LIT);"; + + // TODO is this correct? clang fails because that's not really a valid thing but gcc seems to accept it + // see https://gist.github.com/patrickdowling/877a25294f069bf059f3b07f9b5b7039 + + ASSERT_EQUALS("\n\n\"string\" LIT ;", preprocess(code)); +} + +static void hashhashhash_int_literal() +{ + const char code[] = + "#define CONCAT(a,b,c) a##b##c\n" + "#define PASTER(a,b,c) CONCAT(a,b,c)\n" + "PASTER(\"123\",_i,ul);"; + + ASSERT_EQUALS("\n\n\"123\" _iul ;", preprocess(code)); +} + +static void hashhash_int_literal() +{ + const char code[] = + "#define PASTE(a,b) a##b\n" + "PASTE(123,_i);\n" + "1234_i;\n"; + + ASSERT_EQUALS("\n123_i ;\n1234_i ;", preprocess(code)); +} + static void hashhash_invalid_1() { const char code[] = "#define f(a) (##x)\nf(1)"; @@ -1087,6 +1186,16 @@ static void hashhash_invalid_2() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); } +static void hashhash_invalid_3() +{ + const char code[] = + "#define BAD(x) x##12345\nBAD(\"ABC\")"; + + simplecpp::OutputList outputList; + preprocess(code, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD'.\n", toString(outputList)); +} + static void has_include_1() { const char code[] = "#ifdef __has_include\n" @@ -2306,8 +2415,19 @@ int main(int argc, char **argv) TEST_CASE(hashhash11); // #60: #define x # # # TEST_CASE(hashhash12); TEST_CASE(hashhash13); + TEST_CASE(hashhash_string_literal); + TEST_CASE(hashhash_string_wrapped); + TEST_CASE(hashhash_char_literal); + TEST_CASE(hashhash_multichar_literal); + TEST_CASE(hashhash_char_escaped); + TEST_CASE(hashhash_string_nothing); + TEST_CASE(hashhash_string_char); + TEST_CASE(hashhash_string_name); + TEST_CASE(hashhashhash_int_literal); + TEST_CASE(hashhash_int_literal); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); + TEST_CASE(hashhash_invalid_3); // c++17 __has_include TEST_CASE(has_include_1); From 452996cf81c9175dee54cd534314b96ea6d02066 Mon Sep 17 00:00:00 2001 From: Patrick Dowling Date: Mon, 21 Mar 2022 18:51:54 +0100 Subject: [PATCH 156/381] Show more details for invalid ## usage (#256) --- simplecpp.cpp | 37 +++++++++++++++++++++++++++---------- test.cpp | 31 +++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ce3b156c..c8c47bbb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1446,8 +1446,8 @@ namespace simplecpp { /** base class for errors */ struct Error { Error(const Location &loc, const std::string &s) : location(loc), what(s) {} - Location location; - std::string what; + const Location location; + const std::string what; }; /** Struct that is thrown when macro is expanded with wrong number of parameters */ @@ -1457,7 +1457,24 @@ namespace simplecpp { /** Struct that is thrown when there is invalid ## usage */ struct invalidHashHash : public Error { - invalidHashHash(const Location &loc, const std::string ¯oName) : Error(loc, "Invalid ## usage when expanding \'" + macroName + "\'.") {} + static inline std::string format(const std::string ¯oName, const std::string &message) { + return "Invalid ## usage when expanding \'" + macroName + "\': " + message; + } + + invalidHashHash(const Location &loc, const std::string ¯oName, const std::string &message) + : Error(loc, format(macroName, message)) { } + + static inline invalidHashHash unexpectedToken(const Location &loc, const std::string ¯oName, const Token *tokenA) { + return invalidHashHash(loc, macroName, "Unexpected token '"+ tokenA->str()+"'"); + } + + static inline invalidHashHash cannotCombine(const Location &loc, const std::string ¯oName, const Token *tokenA, const Token *tokenB) { + return invalidHashHash(loc, macroName, "Pasting '"+ tokenA->str()+ "' and '"+ tokenB->str() + "' yields an invalid token."); + } + + static inline invalidHashHash unexpectedNewline(const Location &loc, const std::string ¯oName) { + return invalidHashHash(loc, macroName, "Unexpected newline"); + } }; private: /** Create new token where Token::macro is set for replaced tokens */ @@ -1681,7 +1698,7 @@ namespace simplecpp { // A##B => AB if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { if (!sameline(tok, tok->next->next->next)) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::unexpectedNewline(tok->location, name()); TokenList new_output(files); if (!expandArg(&new_output, tok, parametertokens2)) output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok)); @@ -1928,26 +1945,26 @@ namespace simplecpp { const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { Token *A = output->back(); if (!A) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash(tok->location, name(), "Missing first argument"); if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::unexpectedNewline(tok->location, name()); bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::unexpectedToken(tok->location, name(), A); Token *B = tok->next->next; if (!B->name && !B->number && B->op && !B->isOneOf("#=")) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::unexpectedToken(tok->location, name(), B); if ((canBeConcatenatedWithEqual && B->op != '=') || (!canBeConcatenatedWithEqual && B->op == '=')) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::cannotCombine(tok->location, name(), A, B); // Superficial check; more in-depth would in theory be possible _after_ expandArg if (canBeConcatenatedStringOrChar && (B->number || !B->name)) - throw invalidHashHash(tok->location, name()); + throw invalidHashHash::cannotCombine(tok->location, name(), A, B); TokenList tokensB(files); const Token *nextTok = B->next; diff --git a/test.cpp b/test.cpp index 02de49b9..b95ab75b 100644 --- a/test.cpp +++ b/test.cpp @@ -846,11 +846,11 @@ static void garbage() outputList.clear(); ASSERT_EQUALS("", preprocess("#define TEST2() A ##\nTEST2()\n", &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'TEST2', Invalid ## usage when expanding 'TEST2'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'TEST2', Invalid ## usage when expanding 'TEST2': Unexpected newline\n", toString(outputList)); outputList.clear(); ASSERT_EQUALS("", preprocess("#define CON(a,b) a##b##\nCON(1,2)\n", &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'CON', Invalid ## usage when expanding 'CON'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'CON', Invalid ## usage when expanding 'CON': Unexpected newline\n", toString(outputList)); } static void garbage_endif() @@ -994,19 +994,19 @@ static void hashhash9() "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '+' and 'x' yields an invalid token.\n", toString(outputList)); code = "#define A 2##=\n" "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '2' and '=' yields an invalid token.\n", toString(outputList)); code = "#define A <<##x\n" "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '<<' and 'x' yields an invalid token.\n", toString(outputList)); } static void hashhash10() @@ -1175,7 +1175,7 @@ static void hashhash_invalid_1() const char code[] = "#define f(a) (##x)\nf(1)"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f': Unexpected token '('\n", toString(outputList)); } static void hashhash_invalid_2() @@ -1183,17 +1183,27 @@ static void hashhash_invalid_2() const char code[] = "#define f(a) (x##)\nf(1)"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'f', Invalid ## usage when expanding 'f': Unexpected token ')'\n", toString(outputList)); } -static void hashhash_invalid_3() +static void hashhash_invalid_string_number() { const char code[] = "#define BAD(x) x##12345\nBAD(\"ABC\")"; simplecpp::OutputList outputList; preprocess(code, simplecpp::DUI(), &outputList); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD'.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Pasting '\"ABC\"' and '12345' yields an invalid token.\n", toString(outputList)); +} + +static void hashhash_invalid_missing_args() +{ + const char code[] = + "#define BAD(x) ##x\nBAD()"; + + simplecpp::OutputList outputList; + preprocess(code, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Missing first argument\n", toString(outputList)); } static void has_include_1() @@ -2427,7 +2437,8 @@ int main(int argc, char **argv) TEST_CASE(hashhash_int_literal); TEST_CASE(hashhash_invalid_1); TEST_CASE(hashhash_invalid_2); - TEST_CASE(hashhash_invalid_3); + TEST_CASE(hashhash_invalid_string_number); + TEST_CASE(hashhash_invalid_missing_args); // c++17 __has_include TEST_CASE(has_include_1); From 7f9f5b197d53637b675cf247e872b20e340b54e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 22 Mar 2022 21:12:00 +0100 Subject: [PATCH 157/381] fixed Cppcheck compilation with MSBuild (#257) --- simplecpp.cpp | 8 +++++--- simplecpp.h | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c8c47bbb..e168ee74 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -58,12 +58,14 @@ static bool isOct(const std::string &s) return s.size()>1 && (s[0]=='0') && (s[1] >= '0') && (s[1] < '8'); } -static bool isStringLiteral(const std::string &s) +// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild +static bool isStringLiteral_(const std::string &s) { return s.size() > 1 && (s[0]=='\"') && (*s.rbegin()=='\"'); } -static bool isCharLiteral(const std::string &s) +// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild +static bool isCharLiteral_(const std::string &s) { // char literal patterns can include 'a', '\t', '\000', '\xff', 'abcd', and maybe '' // This only checks for the surrounding '' but doesn't parse the content. @@ -1950,7 +1952,7 @@ namespace simplecpp { throw invalidHashHash::unexpectedNewline(tok->location, name()); bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; - bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); + bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar) throw invalidHashHash::unexpectedToken(tok->location, name(), A); diff --git a/simplecpp.h b/simplecpp.h index 4b97aa35..87dad248 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -353,10 +353,10 @@ namespace simplecpp { SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); /** Returns the __STDC_VERSION__ value for a given standard */ - SIMPLECPP_LIB static std::string getCStdString(const std::string &std); + SIMPLECPP_LIB std::string getCStdString(const std::string &std); /** Returns the __cplusplus value for a given standard */ - SIMPLECPP_LIB static std::string getCppStdString(const std::string &std); + SIMPLECPP_LIB std::string getCppStdString(const std::string &std); } #if (__cplusplus < 201103L) && !defined(__APPLE__) From 0fe0ab6aa3f39757deea0c0f8a068c0d6a61980c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 24 Mar 2022 19:05:31 +0100 Subject: [PATCH 158/381] added preliminary C++23 support and updated some comments (#258) --- simplecpp.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e168ee74..33b56f50 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3400,7 +3400,7 @@ std::string simplecpp::getCStdString(const std::string &std) if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") return "201710L"; if (std == "c2x" || std == "gnu2x") { - // Clang 11 returns "201710L" + // Clang 11, 12, 13 return "201710L" - correct in 14 return "202000L"; } return ""; @@ -3417,14 +3417,14 @@ std::string simplecpp::getCppStdString(const std::string &std) if (std == "c++17" || std == "c++1z" || std == "gnu++17" || std == "gnu++1z") return "201703L"; if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") { - // GCC 10 returns "201703L" + // GCC 10 returns "201703L" - correct in 11+ return "202002L"; } - /* if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") { - // supported by GCC 11+ - return ""; - } */ + // supported by GCC 11+ and Clang 12+ + // Clang 12, 13, 14 do not support "c++23" and "gnu++23" + return "202100L"; + } return ""; } From 22a298bc657b4238f2305236ddd5ddbf35afc298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 13 Apr 2022 20:50:28 +0200 Subject: [PATCH 159/381] CI-unixish.yml: run tests with sanitized builds (#260) --- .github/workflows/CI-unixish.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 835189e6..097e0c26 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -35,4 +35,24 @@ jobs: if: matrix.os == 'ubuntu-20.04' run: | valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner - + + - name: Run AddressSanitizer + if: matrix.os == 'ubuntu-20.04' + run: | + make clean + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" + env: + ASAN_OPTIONS: detect_stack_use_after_return=1 + + - name: Run UndefinedBehaviorSanitizer + if: matrix.os == 'ubuntu-20.04' + run: | + make clean + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + + # TODO: enable when false positives are fixed + - name: Run MemorySanitizer + if: false && matrix.os == 'ubuntu-20.04' && matrix.compiler == 'clang++' + run: | + make clean + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" From e3178151384691321714dd78313653975f25e370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 13 Apr 2022 22:46:19 +0200 Subject: [PATCH 160/381] even more testrunner cleanups (#254) --- test.cpp | 157 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/test.cpp b/test.cpp index b95ab75b..1b7788c6 100644 --- a/test.cpp +++ b/test.cpp @@ -78,32 +78,35 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) +static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) +{ + std::istringstream istr(code); + return simplecpp::TokenList(istr,filenames,filename,outputList); +} -static std::string readfile(const char code[], int sz=-1, simplecpp::OutputList *outputList=nullptr) +static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(sz == -1 ? std::string(code) : std::string(code,sz)); - std::vector files; - return simplecpp::TokenList(istr,files,std::string(),outputList).stringify(); + std::istringstream istr(std::string(code, size)); + return simplecpp::TokenList(istr,filenames,filename,outputList); } -static simplecpp::TokenList makeTokenList(const char code[], std::vector &files, const std::string &file) +static std::string readfile(const char code[], simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(code); - return simplecpp::TokenList(istr,files,file); + std::vector files; + return makeTokenList(code,files,std::string(),outputList).stringify(); } -static simplecpp::TokenList makeTokenList(const char code[]) +static std::string readfile(const char code[], std::size_t size, simplecpp::OutputList *outputList=nullptr) { std::vector files; - return makeTokenList(code, files, std::string()); + return makeTokenList(code,size,files,std::string(),outputList).stringify(); } static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { - std::istringstream istr(code); std::vector files; std::map filedata; - simplecpp::TokenList tokens(istr,files); + simplecpp::TokenList tokens = makeTokenList(code,files); tokens.removeComments(); simplecpp::TokenList tokens2(files); simplecpp::preprocess(tokens2, tokens, files, filedata, dui, outputList); @@ -167,15 +170,15 @@ static void backslash() // preprocessed differently simplecpp::OutputList outputList; - readfile("//123 \\\n456", -1, &outputList); + readfile("//123 \\\n456", &outputList); ASSERT_EQUALS("", toString(outputList)); - readfile("//123 \\ \n456", -1, &outputList); + readfile("//123 \\ \n456", &outputList); ASSERT_EQUALS("file0,1,portability_backslash,Combination 'backslash space newline' is not portable.\n", toString(outputList)); outputList.clear(); - readfile("#define A \\\n123", -1, &outputList); + readfile("#define A \\\n123", &outputList); ASSERT_EQUALS("", toString(outputList)); - readfile("#define A \\ \n123", -1, &outputList); + readfile("#define A \\ \n123", &outputList); ASSERT_EQUALS("file0,1,portability_backslash,Combination 'backslash space newline' is not portable.\n", toString(outputList)); } @@ -196,7 +199,8 @@ static void builtin() static std::string testConstFold(const char code[]) { try { - simplecpp::TokenList expr = makeTokenList(code); + std::vector files; + simplecpp::TokenList expr = makeTokenList(code, files); expr.constFold(); return expr.stringify(); } catch (std::exception &) { @@ -803,36 +807,35 @@ static void error3() { const char code[] = "#error \"bla bla\\\n" " bla bla.\"\n"; - std::istringstream istr(code); std::vector files; simplecpp::OutputList outputList; - simplecpp::TokenList rawtokens(istr, files, "test.c", &outputList); + const simplecpp::TokenList rawtokens = makeTokenList(code, files, "test.c", &outputList); ASSERT_EQUALS("", toString(outputList)); } static void error4() { // "#error x\n1" - const std::string code("\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31", 22); - std::istringstream istr(code); + const char code[] = "\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31"; std::vector files; std::map filedata; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + const simplecpp::TokenList rawtoken = makeTokenList(code, sizeof(code),files,"test.c"); + simplecpp::preprocess(tokens2, rawtoken, files, filedata, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } static void error5() { // "#error x\n1" - const std::string code("\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00", 22); - std::istringstream istr(code); + const char code[] = "\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00"; std::vector files; std::map filedata; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files,"test.c"), files, filedata, simplecpp::DUI(), &outputList); + const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code),files,"test.c"); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1557,13 +1560,13 @@ static void missingHeader1() static void missingHeader2() { const char code[] = "#include \"foo.h\"\n"; // this file exists - std::istringstream istr(code); std::vector files; std::map filedata; filedata["foo.h"] = nullptr; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI(), &outputList); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -1578,9 +1581,8 @@ static void missingHeader3() static void nestedInclude() { const char code[] = "#include \"test.h\"\n"; - std::istringstream istr(code); std::vector files; - simplecpp::TokenList rawtokens(istr,files,"test.h"); + simplecpp::TokenList rawtokens = makeTokenList(code,files,"test.h"); std::map filedata; filedata["test.h"] = &rawtokens; @@ -1604,9 +1606,8 @@ static void multiline2() const char code[] = "#define A /*\\\n" "*/1\n" "A"; - std::istringstream istr(code); std::vector files; - simplecpp::TokenList rawtokens(istr,files); + simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /**/ 1\n\nA", rawtokens.stringify()); rawtokens.removeComments(); std::map filedata; @@ -1620,9 +1621,8 @@ static void multiline3() // #28 - macro with multiline comment const char code[] = "#define A /*\\\n" " */ 1\n" "A"; - std::istringstream istr(code); std::vector files; - simplecpp::TokenList rawtokens(istr,files); + simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /* */ 1\n\nA", rawtokens.stringify()); rawtokens.removeComments(); std::map filedata; @@ -1637,9 +1637,8 @@ static void multiline4() // #28 - macro with multiline comment " /*\\\n" " */ 1\n" "A"; - std::istringstream istr(code); std::vector files; - simplecpp::TokenList rawtokens(istr,files); + simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /* */ 1\n\n\nA", rawtokens.stringify()); rawtokens.removeComments(); std::map filedata; @@ -1652,7 +1651,8 @@ static void multiline5() // column { const char code[] = "#define A\\\n" "("; - const simplecpp::TokenList rawtokens = makeTokenList(code); + std::vector files; + const simplecpp::TokenList rawtokens = makeTokenList(code, files); ASSERT_EQUALS("# define A (", rawtokens.stringify()); ASSERT_EQUALS(11, rawtokens.cback()->location.col); } @@ -1662,7 +1662,8 @@ static void multiline6() // multiline string in macro const char code[] = "#define string (\"\\\n" "x\")\n" "string\n"; - const simplecpp::TokenList rawtokens = makeTokenList(code); + std::vector files; + const simplecpp::TokenList rawtokens = makeTokenList(code, files); ASSERT_EQUALS("# define string ( \"x\" )\n" "\n" "string", rawtokens.stringify()); @@ -1673,7 +1674,8 @@ static void multiline7() // multiline string in macro const char code[] = "#define A(X) aaa { f(\"\\\n" "a\"); }\n" "A(1)"; - const simplecpp::TokenList rawtokens = makeTokenList(code); + std::vector files; + const simplecpp::TokenList rawtokens = makeTokenList(code, files); ASSERT_EQUALS("# define A ( X ) aaa { f ( \"a\" ) ; }\n" "\n" "A ( 1 )", rawtokens.stringify()); @@ -1911,11 +1913,11 @@ static void readfile_char_error() { simplecpp::OutputList outputList; - readfile("A = L's", -1, &outputList); + readfile("A = L's", &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); outputList.clear(); - readfile("A = 's\n'", -1, &outputList); + readfile("A = 's\n'", &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); } @@ -1969,36 +1971,36 @@ static void readfile_string_error() { simplecpp::OutputList outputList; - readfile("A = \"abs", -1, &outputList); + readfile("A = \"abs", &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); outputList.clear(); - readfile("A = u8\"abs\n\"", -1, &outputList); + readfile("A = u8\"abs\n\"", &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); outputList.clear(); - readfile("A = R\"as\n(abc)as\"", -1, &outputList); + readfile("A = R\"as\n(abc)as\"", &outputList); ASSERT_EQUALS("file0,1,syntax_error,Invalid newline in raw string delimiter.\n", toString(outputList)); outputList.clear(); - readfile("A = u8R\"as\n(abc)as\"", -1, &outputList); + readfile("A = u8R\"as\n(abc)as\"", &outputList); ASSERT_EQUALS("file0,1,syntax_error,Invalid newline in raw string delimiter.\n", toString(outputList)); outputList.clear(); - readfile("A = R\"as(abc)a\"", -1, &outputList); + readfile("A = R\"as(abc)a\"", &outputList); ASSERT_EQUALS("file0,1,syntax_error,Raw string missing terminating delimiter.\n", toString(outputList)); outputList.clear(); - readfile("A = LR\"as(abc)a\"", -1, &outputList); + readfile("A = LR\"as(abc)a\"", &outputList); ASSERT_EQUALS("file0,1,syntax_error,Raw string missing terminating delimiter.\n", toString(outputList)); outputList.clear(); - readfile("#define A \"abs", -1, &outputList); + readfile("#define A \"abs", &outputList); ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\"). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList)); outputList.clear(); // Don't warn for a multiline define - readfile("#define A \"abs\\\n\"", -1, &outputList); + readfile("#define A \"abs\\\n\"", &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -2010,11 +2012,11 @@ static void readfile_cpp14_number() static void readfile_unhandled_chars() { simplecpp::OutputList outputList; - readfile("// 你好世界", -1, &outputList); + readfile("// 你好世界", &outputList); ASSERT_EQUALS("", toString(outputList)); - readfile("s=\"你好世界\"", -1, &outputList); + readfile("s=\"你好世界\"", &outputList); ASSERT_EQUALS("", toString(outputList)); - readfile("int 你好世界=0;", -1, &outputList); + readfile("int 你好世界=0;", &outputList); ASSERT_EQUALS("file0,1,unhandled_char_error,The code contains unhandled character(s) (character code=228). Neither unicode nor extended ascii is supported.\n", toString(outputList)); } @@ -2057,9 +2059,9 @@ static void tokenMacro1() "A"; std::vector files; std::map filedata; - std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); ASSERT_EQUALS("A", tokenList.cback()->macro); } @@ -2069,9 +2071,9 @@ static void tokenMacro2() "ADD(1,2)"; std::vector files; std::map filedata; - std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("", tok->macro); @@ -2090,9 +2092,9 @@ static void tokenMacro3() "ADD(FRED,2)"; std::vector files; std::map filedata; - std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("FRED", tok->macro); @@ -2111,9 +2113,9 @@ static void tokenMacro4() "A"; std::vector files; std::map filedata; - std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("A", tok->macro); @@ -2126,9 +2128,9 @@ static void tokenMacro5() "SET_BPF_JUMP(A | B | C);"; std::vector files; std::map filedata; - std::istringstream istr(code); simplecpp::TokenList tokenList(files); - simplecpp::preprocess(tokenList, simplecpp::TokenList(istr,files), files, filedata, simplecpp::DUI()); + const simplecpp::TokenList rawtokens = makeTokenList(code,files); + simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront()->next; ASSERT_EQUALS("D", tok->str()); ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); @@ -2159,13 +2161,34 @@ static void utf8() static void unicode() { - ASSERT_EQUALS("12", readfile("\xFE\xFF\x00\x31\x00\x32", 6)); - ASSERT_EQUALS("12", readfile("\xFF\xFE\x31\x00\x32\x00", 6)); - ASSERT_EQUALS("//\n1", readfile("\xFE\xFF\x00\x2f\x00\x2f\x00\x0a\x00\x31", 10)); - ASSERT_EQUALS("//\n1", readfile("\xFF\xFE\x2f\x00\x2f\x00\x0a\x00\x31\x00", 10)); - ASSERT_EQUALS("\"a\"", readfile("\xFE\xFF\x00\x22\x00\x61\x00\x22", 8)); - ASSERT_EQUALS("\"a\"", readfile("\xFF\xFE\x22\x00\x61\x00\x22\x00", 8)); - ASSERT_EQUALS("\n//1", readfile("\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00",16)); + { + const char code[] = "\xFE\xFF\x00\x31\x00\x32"; + ASSERT_EQUALS("12", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x31\x00\x32\x00"; + ASSERT_EQUALS("12", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE\xFF\x00\x2f\x00\x2f\x00\x0a\x00\x31"; + ASSERT_EQUALS("//\n1", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x2f\x00\x2f\x00\x0a\x00\x31\x00"; + ASSERT_EQUALS("//\n1", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE\xFF\x00\x22\x00\x61\x00\x22"; + ASSERT_EQUALS("\"a\"", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x22\x00\x61\x00\x22\x00"; + ASSERT_EQUALS("\"a\"", readfile(code, sizeof(code))); + } + { + const char code[] = "\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00"; + ASSERT_EQUALS("\n//1", readfile(code, sizeof(code))); + } } static void warning() From fb07fcc23d55e68dc4ab6bf4163a3b29ce89e033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 27 Apr 2022 20:15:34 +0200 Subject: [PATCH 161/381] Makefile: use python3 to run tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a45c90c9..a209666e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ test: testrunner simplecpp # The -std=c++03 makes sure that simplecpp.cpp is C++03 conformant. We don't require a C++11 compiler g++ -std=c++03 -fsyntax-only simplecpp.cpp ./testrunner - python run-tests.py + python3 run-tests.py simplecpp: main.o simplecpp.o $(CXX) $(LDFLAGS) main.o simplecpp.o -o simplecpp From 6c2c90c1a2dacc3f456b212cdd06900a0adfce8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 4 Jul 2022 21:18:53 +0200 Subject: [PATCH 162/381] astyle formatting --- simplecpp.cpp | 9 ++++++--- test.cpp | 10 +++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 33b56f50..91dbc825 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2917,7 +2917,8 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token return true; } -static void getLocaltime(struct tm <ime) { +static void getLocaltime(struct tm <ime) +{ time_t t; time(&t); #ifndef _WIN32 @@ -2927,13 +2928,15 @@ static void getLocaltime(struct tm <ime) { #endif } -static std::string getDateDefine(struct tm *timep) { +static std::string getDateDefine(struct tm *timep) +{ char buf[] = "??? ?? ????"; strftime(buf, sizeof(buf), "%b %d %Y", timep); return std::string("\"").append(buf).append("\""); } -static std::string getTimeDefine(struct tm *timep) { +static std::string getTimeDefine(struct tm *timep) +{ char buf[] = "??:??:??"; strftime(buf, sizeof(buf), "%T", timep); return std::string("\"").append(buf).append("\""); diff --git a/test.cpp b/test.cpp index 1b7788c6..c2ddd644 100644 --- a/test.cpp +++ b/test.cpp @@ -806,7 +806,7 @@ static void error2() static void error3() { const char code[] = "#error \"bla bla\\\n" - " bla bla.\"\n"; + " bla bla.\"\n"; std::vector files; simplecpp::OutputList outputList; const simplecpp::TokenList rawtokens = makeTokenList(code, files, "test.c", &outputList); @@ -2139,10 +2139,10 @@ static void tokenMacro5() static void undef() { const char code[] = "#define A\n" - "#undef A\n" - "#ifdef A\n" - "123\n" - "#endif"; + "#undef A\n" + "#ifdef A\n" + "123\n" + "#endif"; ASSERT_EQUALS("", preprocess(code)); } From 9ab77d67fb2d50744cbca677373b490b8b2c956e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 4 Jul 2022 21:25:50 +0200 Subject: [PATCH 163/381] Diagnose undefined behavior when token concatenation produces an universal character --- simplecpp.cpp | 14 +++++++++++++- test.cpp | 22 ++++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 91dbc825..af31f99d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1471,12 +1471,16 @@ namespace simplecpp { } static inline invalidHashHash cannotCombine(const Location &loc, const std::string ¯oName, const Token *tokenA, const Token *tokenB) { - return invalidHashHash(loc, macroName, "Pasting '"+ tokenA->str()+ "' and '"+ tokenB->str() + "' yields an invalid token."); + return invalidHashHash(loc, macroName, "Combining '"+ tokenA->str()+ "' and '"+ tokenB->str() + "' yields an invalid token."); } static inline invalidHashHash unexpectedNewline(const Location &loc, const std::string ¯oName) { return invalidHashHash(loc, macroName, "Unexpected newline"); } + + static inline invalidHashHash universalCharacterUB(const Location &loc, const std::string ¯oName, const Token* tokenA, const std::string& strAB) { + return invalidHashHash(loc, macroName, "Combining '\\"+ tokenA->str()+ "' and '"+ strAB.substr(tokenA->str().size()) + "' yields universal character '\\" + strAB + "'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4."); + } }; private: /** Create new token where Token::macro is set for replaced tokens */ @@ -2000,6 +2004,14 @@ namespace simplecpp { strAB = A->str() + B->str(); } + // producing universal character is undefined behavior + if (A->previous && A->previous->str() == "\\") { + if (strAB[0] == 'u' && strAB.size() == 5) + throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); + else if (strAB[0] == 'U' && strAB.size() == 9) + throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); + } + if (varargs && tokensB.empty() && tok->previous->str() == ",") output->deleteToken(A); else if (strAB != "," && macros.find(strAB) == macros.end()) { diff --git a/test.cpp b/test.cpp index c2ddd644..e5f9ee0c 100644 --- a/test.cpp +++ b/test.cpp @@ -997,19 +997,19 @@ static void hashhash9() "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '+' and 'x' yields an invalid token.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '+' and 'x' yields an invalid token.\n", toString(outputList)); code = "#define A 2##=\n" "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '2' and '=' yields an invalid token.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '2' and '=' yields an invalid token.\n", toString(outputList)); code = "#define A <<##x\n" "A"; outputList.clear(); ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Pasting '<<' and 'x' yields an invalid token.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '<<' and 'x' yields an invalid token.\n", toString(outputList)); } static void hashhash10() @@ -1196,7 +1196,7 @@ static void hashhash_invalid_string_number() simplecpp::OutputList outputList; preprocess(code, simplecpp::DUI(), &outputList); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Pasting '\"ABC\"' and '12345' yields an invalid token.\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Combining '\"ABC\"' and '12345' yields an invalid token.\n", toString(outputList)); } static void hashhash_invalid_missing_args() @@ -1209,6 +1209,15 @@ static void hashhash_invalid_missing_args() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Missing first argument\n", toString(outputList)); } +static void hashhash_universal_character() +{ + const char code[] = + "#define A(x,y) x##y\nint A(\\u01,04);"; + simplecpp::OutputList outputList; + preprocess(code, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '\\u01' and '04' yields universal character '\\u0104'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4.\n", toString(outputList)); +} + static void has_include_1() { const char code[] = "#ifdef __has_include\n" @@ -2462,6 +2471,11 @@ int main(int argc, char **argv) TEST_CASE(hashhash_invalid_2); TEST_CASE(hashhash_invalid_string_number); TEST_CASE(hashhash_invalid_missing_args); + // C standard, 5.1.1.2, paragraph 4: + // If a character sequence that matches the syntax of a universal + // character name is produced by token concatenation (6.10.3.3), + // the behavior is undefined." + TEST_CASE(hashhash_universal_character); // c++17 __has_include TEST_CASE(has_include_1); From 161467f2b16319ed524b2d67e18b28a737fccfc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 27 Jul 2022 21:43:27 +0200 Subject: [PATCH 164/381] remove extra comments in expression --- simplecpp.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index af31f99d..4b1f6d24 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2683,8 +2683,19 @@ static void simplifyNumbers(simplecpp::TokenList &expr) } } +static void simplifyComments(simplecpp::TokenList &expr) +{ + for (simplecpp::Token *tok = expr.front(); tok;) { + simplecpp::Token *d = tok; + tok = tok->next; + if (d->comment) + expr.deleteToken(d); + } +} + static long long evaluate(simplecpp::TokenList &expr, const std::map &sizeOfType) { + simplifyComments(expr); simplifySizeof(expr, sizeOfType); simplifyName(expr); simplifyNumbers(expr); From 572d5c1914551896abb6cace515f8e98250a21e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 27 Jul 2022 21:48:24 +0200 Subject: [PATCH 165/381] CI: Remove macos-10.15, it has been deprecated --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 097e0c26..bed36802 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-18.04, ubuntu-20.04, macos-10.15] + os: [ubuntu-18.04, ubuntu-20.04] fail-fast: false runs-on: ${{ matrix.os }} From 360d3d7af26ecd8b04d000b906e57357db1f0120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Aug 2022 21:21:13 +0200 Subject: [PATCH 166/381] CI-unixish.yml: added `ubuntu-22.04`, `macos-11` and `macos-12` (#265) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index bed36802..9128d3ee 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-18.04, ubuntu-20.04] + os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] fail-fast: false runs-on: ${{ matrix.os }} From 43f17335ffb86f5d3be4849afdf0b5549e1e9ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Aug 2022 21:23:32 +0200 Subject: [PATCH 167/381] main.cpp: added -q (quiet mode) (#234) --- main.cpp | 69 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/main.cpp b/main.cpp index f16cee77..d8e80542 100644 --- a/main.cpp +++ b/main.cpp @@ -28,11 +28,12 @@ int main(int argc, char **argv) // Settings.. simplecpp::DUI dui; + bool quiet = false; for (int i = 1; i < argc; i++) { const char *arg = argv[i]; if (*arg == '-') { char c = arg[1]; - if (c != 'D' && c != 'U' && c != 'I' && c != 'i' && c != 's') + if (c != 'D' && c != 'U' && c != 'I' && c != 'i' && c != 's' && c != 'q') continue; // Ignored const char *value = arg[2] ? (argv[i] + 2) : argv[++i]; switch (c) { @@ -53,6 +54,9 @@ int main(int argc, char **argv) if (std::strncmp(arg, "-std=",5)==0) dui.std = arg + 5; break; + case 'q': + quiet = true; + break; } } else { filename = arg; @@ -66,6 +70,8 @@ int main(int argc, char **argv) std::cout << " -IPATH Include path." << std::endl; std::cout << " -include=FILE Include FILE." << std::endl; std::cout << " -UNAME Undefine NAME." << std::endl; + std::cout << " -std=STD Specify standard." << std::endl; + std::cout << " -q Quiet mode (no output)." << std::endl; std::exit(0); } @@ -82,36 +88,39 @@ int main(int argc, char **argv) simplecpp::preprocess(outputTokens, rawtokens, files, included, dui, &outputList); // Output - std::cout << outputTokens.stringify() << std::endl; - for (const simplecpp::Output &output : outputList) { - std::cerr << output.location.file() << ':' << output.location.line << ": "; - switch (output.type) { - case simplecpp::Output::ERROR: - std::cerr << "#error: "; - break; - case simplecpp::Output::WARNING: - std::cerr << "#warning: "; - break; - case simplecpp::Output::MISSING_HEADER: - std::cerr << "missing header: "; - break; - case simplecpp::Output::INCLUDE_NESTED_TOO_DEEPLY: - std::cerr << "include nested too deeply: "; - break; - case simplecpp::Output::SYNTAX_ERROR: - std::cerr << "syntax error: "; - break; - case simplecpp::Output::PORTABILITY_BACKSLASH: - std::cerr << "portability: "; - break; - case simplecpp::Output::UNHANDLED_CHAR_ERROR: - std::cerr << "unhandled char error: "; - break; - case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: - std::cerr << "explicit include not found: "; - break; + if (!quiet) { + std::cout << outputTokens.stringify() << std::endl; + + for (const simplecpp::Output &output : outputList) { + std::cerr << output.location.file() << ':' << output.location.line << ": "; + switch (output.type) { + case simplecpp::Output::ERROR: + std::cerr << "#error: "; + break; + case simplecpp::Output::WARNING: + std::cerr << "#warning: "; + break; + case simplecpp::Output::MISSING_HEADER: + std::cerr << "missing header: "; + break; + case simplecpp::Output::INCLUDE_NESTED_TOO_DEEPLY: + std::cerr << "include nested too deeply: "; + break; + case simplecpp::Output::SYNTAX_ERROR: + std::cerr << "syntax error: "; + break; + case simplecpp::Output::PORTABILITY_BACKSLASH: + std::cerr << "portability: "; + break; + case simplecpp::Output::UNHANDLED_CHAR_ERROR: + std::cerr << "unhandled char error: "; + break; + case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: + std::cerr << "explicit include not found: "; + break; + } + std::cerr << output.msg << std::endl; } - std::cerr << output.msg << std::endl; } // cleanup included tokenlists From ab59545b083cf95ad8a85231f71aa6d320cd6f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 25 Aug 2022 08:54:36 +0200 Subject: [PATCH 168/381] added clang-tidy CI job and fixed warnings / added CMake option "DISABLE_CPP03_SYNTAX_CHECK" / CMake cleanups (#248) --- .clang-tidy | 4 ++++ .github/workflows/clang-tidy.yml | 37 ++++++++++++++++++++++++++++++++ CMakeLists.txt | 30 +++++++++++++++----------- simplecpp.cpp | 22 +++++++++---------- 4 files changed, 70 insertions(+), 23 deletions(-) create mode 100644 .clang-tidy create mode 100644 .github/workflows/clang-tidy.yml diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000..3470298c --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,4 @@ +--- +Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-core.NullDereference,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move' +HeaderFilterRegex: '.*' +WarningsAsErrors: '*' \ No newline at end of file diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml new file mode 100644 index 00000000..2d15897a --- /dev/null +++ b/.github/workflows/clang-tidy.yml @@ -0,0 +1,37 @@ +# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions +# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners +name: clang-tidy + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v2 + + - name: Install missing software + run: | + sudo apt-get update + sudo apt-get install -y cmake make + + - name: Install clang + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 14 + + - name: Prepare CMake + run: | + mkdir cmake.output + cd cmake.output + cmake -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON .. + cd .. + env: + CXX: clang-14 + + - name: Clang-Tidy + run: | + run-clang-tidy-14 -q -j $(nproc) -p=cmake.output -extra-arg=-w diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c3aba80..a1ce6ba4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required (VERSION 3.5) project (simplecpp LANGUAGES CXX) +option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast -Wno-multichar) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -22,23 +24,27 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() endif() -add_executable(simplecpp simplecpp.cpp main.cpp) +add_library(simplecpp_obj OBJECT simplecpp.cpp) + +add_executable(simplecpp $ main.cpp) set_property(TARGET simplecpp PROPERTY CXX_STANDARD 11) -# it is not possible to set a standard older than C++14 with Visual Studio -if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - # we need to create a dummy library as -fsyntax-only will not produce any output files causing the build to fail - add_library(simplecpp-03-syntax STATIC simplecpp.cpp) - target_compile_options(simplecpp-03-syntax PRIVATE -std=c++03) - if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - target_compile_options(simplecpp-03-syntax PRIVATE -Wno-long-long) - elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long -Wno-c++11-compat) +if (NOT DISABLE_CPP03_SYNTAX_CHECK) + # it is not possible to set a standard older than C++14 with Visual Studio + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # we need to create a dummy library as -fsyntax-only will not produce any output files causing the build to fail + add_library(simplecpp-03-syntax OBJECT simplecpp.cpp) + target_compile_options(simplecpp-03-syntax PRIVATE -std=c++03) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + target_compile_options(simplecpp-03-syntax PRIVATE -Wno-long-long) + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long -Wno-c++11-compat) + endif() + add_dependencies(simplecpp simplecpp-03-syntax) endif() - add_dependencies(simplecpp simplecpp-03-syntax) endif() -add_executable(testrunner simplecpp.cpp test.cpp) +add_executable(testrunner $ test.cpp) set_property(TARGET testrunner PROPERTY CXX_STANDARD 11) enable_testing() diff --git a/simplecpp.cpp b/simplecpp.cpp index 4b1f6d24..61417dd6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1842,7 +1842,7 @@ namespace simplecpp { return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros2, parametertokens); } - else if (tok->str() == DEFINED) { + if (tok->str() == DEFINED) { const Token *tok2 = tok->next; const Token *tok3 = tok2 ? tok2->next : nullptr; const Token *tok4 = tok3 ? tok3->next : nullptr; @@ -1989,7 +1989,7 @@ namespace simplecpp { } else { std::string strAB; - const bool varargs = variadic && args.size() >= 1U && B->str() == args[args.size()-1U]; + const bool varargs = variadic && !args.empty() && B->str() == args[args.size()-1U]; if (expandArg(&tokensB, B, parametertokens)) { if (tokensB.empty()) @@ -2008,7 +2008,7 @@ namespace simplecpp { if (A->previous && A->previous->str() == "\\") { if (strAB[0] == 'u' && strAB.size() == 5) throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); - else if (strAB[0] == 'U' && strAB.size() == 9) + if (strAB[0] == 'U' && strAB.size() == 9) throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); } @@ -2487,7 +2487,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) std::size_t pos; - if (str.size() >= 1 && str[0] == '\'') { + if (!str.empty() && str[0] == '\'') { narrow = true; pos = 1; } else if (str.size() >= 2 && str[0] == 'u' && str[1] == '\'') { @@ -2611,7 +2611,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) int additional_bytes; if (value >= 0xf5) // higher values would result in code points above 0x10ffff throw std::runtime_error("assumed UTF-8 encoded source, but sequence is invalid"); - else if (value >= 0xf0) + if (value >= 0xf0) additional_bytes = 3; else if (value >= 0xe0) additional_bytes = 2; @@ -2678,7 +2678,7 @@ static void simplifyNumbers(simplecpp::TokenList &expr) continue; if (tok->str().compare(0,2,"0x") == 0) tok->setstr(toString(stringToULL(tok->str()))); - else if (!tok->number && tok->str().find('\'') != tok->str().npos) + else if (!tok->number && tok->str().find('\'') != std::string::npos) tok->setstr(toString(simplecpp::characterLiteralToLL(tok->str()))); } } @@ -2840,7 +2840,7 @@ static bool hasFile(const std::map &filedat return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); } -std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &fileNumbers, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) +std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { std::map ret; @@ -2856,16 +2856,16 @@ std::map simplecpp::load(const simplecpp::To std::ifstream fin(filename.c_str()); if (!fin.is_open()) { if (outputList) { - simplecpp::Output err(fileNumbers); + simplecpp::Output err(filenames); err.type = simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND; - err.location = Location(fileNumbers); + err.location = Location(filenames); err.msg = "Can not open include file '" + filename + "' that is explicitly included."; outputList->push_back(err); } continue; } - TokenList *tokenlist = new TokenList(fin, fileNumbers, filename, outputList); + TokenList *tokenlist = new TokenList(fin, filenames, filename, outputList); if (!tokenlist->front()) { delete tokenlist; continue; @@ -2904,7 +2904,7 @@ std::map simplecpp::load(const simplecpp::To if (!f.is_open()) continue; - TokenList *tokens = new TokenList(f, fileNumbers, header2, outputList); + TokenList *tokens = new TokenList(f, filenames, header2, outputList); ret[header2] = tokens; if (tokens->front()) filelist.push_back(tokens->front()); From 4f0bd3ff13fa05e0df9eeab9880304e179a9ddea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 25 Aug 2022 08:55:58 +0200 Subject: [PATCH 169/381] reduced padding in `simplecpp::Macro` (#267) --- simplecpp.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 61417dd6..8890bb5a 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1276,7 +1276,7 @@ namespace simplecpp { class Macro { public: - explicit Macro(std::vector &f) : nameTokDef(nullptr), variadic(false), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) {} + explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), valueDefinedInCode_(false) {} Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previous, tok)) @@ -2063,9 +2063,6 @@ namespace simplecpp { /** arguments for macro */ std::vector args; - /** is macro variadic? */ - bool variadic; - /** first token in replacement string */ const Token *valueToken; @@ -2081,6 +2078,9 @@ namespace simplecpp { /** usage of this macro */ mutable std::list usageList; + /** is macro variadic? */ + bool variadic; + /** was the value of this macro actually defined in the code? */ bool valueDefinedInCode_; }; From 9fcfa19479e5bb8bd902233a4f09446e3ace96c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 25 Aug 2022 15:29:07 +0200 Subject: [PATCH 170/381] fail the `clang-tidy` build step in case of compiler warnings / ignore `-Wfour-char-constants` and temporally disable `-Wshadow-field-in-constructor` clang warnings (#268) --- .github/workflows/clang-tidy.yml | 2 +- CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 2d15897a..6d4f1670 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -34,4 +34,4 @@ jobs: - name: Clang-Tidy run: | - run-clang-tidy-14 -q -j $(nproc) -p=cmake.output -extra-arg=-w + run-clang-tidy-14 -q -j $(nproc) -p=cmake.output diff --git a/CMakeLists.txt b/CMakeLists.txt index a1ce6ba4..6b51992c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,9 +12,9 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # these are not really fixable add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) # we are not interested in these - add_compile_options(-Wno-multichar) + add_compile_options(-Wno-multichar -Wno-four-char-constants) # TODO: fix these? - add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32) + add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32 -Wno-shadow-field-in-constructor) if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14) # not all tools support DWARF 5 yet so stick to version 4 From d4bc1834ec5c2dc3f0143657275252aeaecc8436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 26 Aug 2022 23:27:12 +0200 Subject: [PATCH 171/381] upgraded CI to clang-15 (#266) --- .clang-tidy | 7 +++- .github/workflows/clang-tidy.yml | 12 +++--- CMakeLists.txt | 12 ++++-- main.cpp | 6 +-- simplecpp.cpp | 71 ++++++++++++++++---------------- simplecpp.h | 4 +- test.cpp | 12 +++--- 7 files changed, 66 insertions(+), 58 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 3470298c..244e860f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,4 +1,9 @@ --- Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-core.NullDereference,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move' HeaderFilterRegex: '.*' -WarningsAsErrors: '*' \ No newline at end of file +WarningsAsErrors: '*' +CheckOptions: + - key: misc-const-correctness.WarnPointersAsValues + value: '1' + - key: misc-const-correctness.TransformPointersAsValues + value: '1' \ No newline at end of file diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 6d4f1670..e1594f60 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,17 +21,15 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 14 + sudo ./llvm.sh 15 + sudo apt-get install clang-tidy-15 - name: Prepare CMake run: | - mkdir cmake.output - cd cmake.output - cmake -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON .. - cd .. + cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-14 + CXX: clang-15 - name: Clang-Tidy run: | - run-clang-tidy-14 -q -j $(nproc) -p=cmake.output + run-clang-tidy-15 -q -j $(nproc) -p=cmake.output diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b51992c..26398823 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,11 +16,15 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # TODO: fix these? add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32 -Wno-shadow-field-in-constructor) - if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14) - # not all tools support DWARF 5 yet so stick to version 4 + if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 14) + # TODO: verify this regression still exists in clang-15 + if (CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + # work around performance regression - see https://github.com/llvm/llvm-project/issues/53555 + add_compile_options(-mllvm -inline-deferral) + endif() + + # use force DWARF 4 debug format since not all tools might be able to handle DWARF 5 yet - e.g. valgrind on ubuntu 20.04 add_compile_options(-gdwarf-4) - # work around performance regression in clang-14 - add_compile_options(-mllvm -inline-deferral) endif() endif() diff --git a/main.cpp b/main.cpp index d8e80542..a5d7996a 100644 --- a/main.cpp +++ b/main.cpp @@ -30,12 +30,12 @@ int main(int argc, char **argv) simplecpp::DUI dui; bool quiet = false; for (int i = 1; i < argc; i++) { - const char *arg = argv[i]; + const char * const arg = argv[i]; if (*arg == '-') { - char c = arg[1]; + const char c = arg[1]; if (c != 'D' && c != 'U' && c != 'I' && c != 'i' && c != 's' && c != 'q') continue; // Ignored - const char *value = arg[2] ? (argv[i] + 2) : argv[++i]; + const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; switch (c) { case 'D': // define symbol dui.defines.push_back(value); diff --git a/simplecpp.cpp b/simplecpp.cpp index 8890bb5a..f7dd51f3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -95,6 +95,7 @@ static const simplecpp::TokenString HAS_INCLUDE("__has_include"); template static std::string toString(T t) { + // NOLINTNEXTLINE(misc-const-correctness) - false positive std::ostringstream ostr; ostr << t; return ostr.str(); @@ -284,7 +285,7 @@ void simplecpp::TokenList::clear() { backToken = nullptr; while (frontToken) { - Token *next = frontToken->next; + Token * const next = frontToken->next; delete frontToken; frontToken = next; } @@ -350,9 +351,9 @@ static unsigned char readChar(std::istream &istr, unsigned int bom) if (bom == 0 && static_cast(istr.peek()) == '\n') (void)istr.get(); else if (bom == 0xfeff || bom == 0xfffe) { - int c1 = istr.get(); - int c2 = istr.get(); - int ch16 = (bom == 0xfeff) ? (c1<<8 | c2) : (c2<<8 | c1); + const int c1 = istr.get(); + const int c2 = istr.get(); + const int ch16 = (bom == 0xfeff) ? (c1<<8 | c2) : (c2<<8 | c1); if (ch16 != '\n') { istr.unget(); istr.unget(); @@ -397,7 +398,7 @@ static unsigned short getAndSkipBOM(std::istream &istr) // The UTF-16 BOM is 0xfffe or 0xfeff. if (ch1 >= 0xfe) { - unsigned short bom = (static_cast(istr.get()) << 8); + const unsigned short bom = (static_cast(istr.get()) << 8); if (istr.peek() >= 0xfe) return bom | static_cast(istr.get()); istr.unget(); @@ -428,7 +429,7 @@ static std::string escapeString(const std::string &str) std::ostringstream ostr; ostr << '\"'; for (std::size_t i = 1U; i < str.size() - 1; ++i) { - char c = str[i]; + const char c = str[i]; if (c == '\\' || c == '\"' || c == '\'') ostr << '\\'; ostr << c; @@ -859,7 +860,7 @@ void simplecpp::TokenList::combineOperators() start = start->previous; } if (indentlevel == -1 && start) { - const Token *ftok = start; + const Token * const ftok = start; bool isFuncDecl = ftok->name; while (isFuncDecl) { if (!start->name && start->str() != "::" && start->op != '*' && start->op != '&') @@ -957,10 +958,10 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) if (tok->op == '*') result = (stringToLL(tok->previous->str()) * stringToLL(tok->next->str())); else if (tok->op == '/' || tok->op == '%') { - long long rhs = stringToLL(tok->next->str()); + const long long rhs = stringToLL(tok->next->str()); if (rhs == 0) throw std::overflow_error("division/modulo by zero"); - long long lhs = stringToLL(tok->previous->str()); + const long long lhs = stringToLL(tok->previous->str()); if (rhs == -1 && lhs == std::numeric_limits::min()) throw std::overflow_error("division overflow"); if (tok->op == '/') @@ -1159,7 +1160,7 @@ void simplecpp::TokenList::removeComments() { Token *tok = frontToken; while (tok) { - Token *tok1 = tok; + Token * const tok1 = tok; tok = tok->next; if (tok1->comment) deleteToken(tok1); @@ -1395,7 +1396,7 @@ namespace simplecpp { TokenList rawtokens2(inputFiles); const Location loc(macro2tok->location); while (macro2tok) { - Token *next = macro2tok->next; + Token * const next = macro2tok->next; rawtokens2.push_back(new Token(macro2tok->str(), loc)); output2.deleteToken(macro2tok); macro2tok = next; @@ -1791,7 +1792,7 @@ namespace simplecpp { TokenList temp2(files); temp2.push_back(new Token(temp.cback()->str(), tok->location)); - const Token *tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); + const Token * const tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) return tok->next; output->takeTokens(temp); @@ -1832,7 +1833,7 @@ namespace simplecpp { } TokenList tokens(files); tokens.push_back(new Token(*tok)); - const Token *tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); + const Token * const tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) { output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; @@ -1843,9 +1844,9 @@ namespace simplecpp { } if (tok->str() == DEFINED) { - const Token *tok2 = tok->next; - const Token *tok3 = tok2 ? tok2->next : nullptr; - const Token *tok4 = tok3 ? tok3->next : nullptr; + const Token * const tok2 = tok->next; + const Token * const tok3 = tok2 ? tok2->next : nullptr; + const Token * const tok4 = tok3 ? tok3->next : nullptr; const Token *defToken = nullptr; const Token *lastToken = nullptr; if (sameline(tok, tok4) && tok2->op == '(' && tok3->name && tok4->op == ')') { @@ -1955,12 +1956,12 @@ namespace simplecpp { if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) throw invalidHashHash::unexpectedNewline(tok->location, name()); - bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; - bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); + const bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; + const bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar) throw invalidHashHash::unexpectedToken(tok->location, name(), A); - Token *B = tok->next->next; + Token * const B = tok->next->next; if (!B->name && !B->number && B->op && !B->isOneOf("#=")) throw invalidHashHash::unexpectedToken(tok->location, name(), B); @@ -2033,7 +2034,7 @@ namespace simplecpp { if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { const MacroMap::const_iterator it = macros.find(strAB); if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { - const Token *tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); + const Token * const tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); if (tok2) nextTok = tok2->next; } @@ -2094,7 +2095,7 @@ namespace simplecpp { std::string::size_type pos = 0; if (cygwinPath.size() >= 11 && startsWith(cygwinPath, "/cygdrive/")) { - unsigned char driveLetter = cygwinPath[10]; + const unsigned char driveLetter = cygwinPath[10]; if (std::isalpha(driveLetter)) { if (cygwinPath.size() == 11) { windowsPath = toupper(driveLetter); @@ -2440,10 +2441,10 @@ static unsigned long long stringToULLbounded( std::size_t maxlen = std::string::npos ) { - std::string sub = s.substr(pos, maxlen); - const char* start = sub.c_str(); + const std::string sub = s.substr(pos, maxlen); + const char * const start = sub.c_str(); char* end; - unsigned long long value = std::strtoull(start, &end, base); + const unsigned long long value = std::strtoull(start, &end, base); pos += end - start; if (end - start < minlen) throw std::runtime_error("expected digit"); @@ -2516,7 +2517,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) if (str[pos] == '\\') { pos++; - char escape = str[pos++]; + const char escape = str[pos++]; if (pos >= str.size()) throw std::runtime_error("unexpected end of character literal"); @@ -2583,7 +2584,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) case 'u': case 'U': { // universal character names have exactly 4 or 8 digits - std::size_t ndigits = (escape == 'u' ? 4 : 8); + const std::size_t ndigits = (escape == 'u' ? 4 : 8); value = stringToULLbounded(str, pos, 16, ndigits, ndigits); // UTF-8 encodes code points above 0x7f in multiple code units @@ -2626,7 +2627,7 @@ long long simplecpp::characterLiteralToLL(const std::string& str) if (pos + 1 >= str.size()) throw std::runtime_error("assumed UTF-8 encoded source, but character literal ends unexpectedly"); - unsigned char c = str[pos++]; + const unsigned char c = str[pos++]; if (((c >> 6) != 2) // ensure c has form 0xb10xxxxxx || (!value && additional_bytes == 1 && c < 0xa0) // overlong 3-bytes encoding @@ -2686,7 +2687,7 @@ static void simplifyNumbers(simplecpp::TokenList &expr) static void simplifyComments(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok;) { - simplecpp::Token *d = tok; + simplecpp::Token * const d = tok; tok = tok->next; if (d->comment) expr.deleteToken(d); @@ -2890,11 +2891,11 @@ std::map simplecpp::load(const simplecpp::To const std::string &sourcefile = rawtok->location.file(); - const Token *htok = rawtok->nextSkipComments(); + const Token * const htok = rawtok->nextSkipComments(); if (!sameline(rawtok, htok)) continue; - bool systemheader = (htok->str()[0] == '<'); + const bool systemheader = (htok->str()[0] == '<'); const std::string header(realFilename(htok->str().substr(1U, htok->str().size() - 2U))); if (hasFile(ret, sourcefile, header, dui, systemheader)) continue; @@ -2915,7 +2916,7 @@ std::map simplecpp::load(const simplecpp::To static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) { - const simplecpp::Token *tok = *tok1; + const simplecpp::Token * const tok = *tok1; const simplecpp::MacroMap::const_iterator it = macros.find(tok->str()); if (it != macros.end()) { simplecpp::TokenList value(files); @@ -3098,7 +3099,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL try { const Macro ¯o = Macro(rawtok->previous, files); if (dui.undefined.find(macro.name()) == dui.undefined.end()) { - MacroMap::iterator it = macros.find(macro.name()); + const MacroMap::iterator it = macros.find(macro.name()); if (it == macros.end()) macros.insert(std::pair(macro.name(), macro)); else @@ -3156,7 +3157,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL return; } - const Token *inctok = inc2.cfront(); + const Token * const inctok = inc2.cfront(); const bool systemheader = (inctok->op == '<'); const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); @@ -3166,7 +3167,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::ifstream f; header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); if (f.is_open()) { - TokenList *tokens = new TokenList(f, files, header2, outputList); + TokenList * const tokens = new TokenList(f, files, header2, outputList); filedata[header2] = tokens; } } @@ -3188,7 +3189,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } else if (pragmaOnce.find(header2) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); - const TokenList *includetokens = filedata.find(header2)->second; + const TokenList * const includetokens = filedata.find(header2)->second; rawtok = includetokens ? includetokens->cfront() : nullptr; continue; } diff --git a/simplecpp.h b/simplecpp.h index 87dad248..8e04a571 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -239,8 +239,8 @@ namespace simplecpp { void deleteToken(Token *tok) { if (!tok) return; - Token *prev = tok->previous; - Token *next = tok->next; + Token * const prev = tok->previous; + Token * const next = tok->next; if (prev) prev->next = next; if (next) diff --git a/test.cpp b/test.cpp index e5f9ee0c..988ed91a 100644 --- a/test.cpp +++ b/test.cpp @@ -1492,10 +1492,10 @@ static void ifalt() // using "and", "or", etc static void ifexpr() { - const char *code = "#define MACRO() (1)\n" - "#if ~MACRO() & 8\n" - "1\n" - "#endif"; + const char code[] = "#define MACRO() (1)\n" + "#if ~MACRO() & 8\n" + "1\n" + "#endif"; ASSERT_EQUALS("\n\n1", preprocess(code)); } @@ -2125,7 +2125,7 @@ static void tokenMacro4() simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront(); + const simplecpp::Token * const tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("A", tok->macro); } @@ -2140,7 +2140,7 @@ static void tokenMacro5() simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); - const simplecpp::Token *tok = tokenList.cfront()->next; + const simplecpp::Token * const tok = tokenList.cfront()->next; ASSERT_EQUALS("D", tok->str()); ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); } From cf2bbe30b8b4aea5ab9f1289dfeb5963edee9149 Mon Sep 17 00:00:00 2001 From: Andrey Alekseenko Date: Tue, 20 Sep 2022 21:51:44 +0200 Subject: [PATCH 172/381] Improve support for __has_include (#271) --- simplecpp.cpp | 83 +++++++++++++++++++++++++++++++++++++++++++++++---- test.cpp | 58 +++++++++++++++++++++++++++++++++-- 2 files changed, 133 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index f7dd51f3..fe7c4155 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2404,6 +2404,63 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::mapnext) { + if (tok->str() != "__has_include") + continue; + simplecpp::Token *tok1 = tok->next; + if (!tok1) { + throw std::runtime_error("missing __has_include argument"); + } + simplecpp::Token *tok2 = tok1->next; + if (!tok2) { + throw std::runtime_error("missing __has_include argument"); + } + if (tok1->op == '(') { + tok1 = tok1->next; + while (tok2->op != ')') { + tok2 = tok2->next; + if (!tok2) { + throw std::runtime_error("invalid __has_include expression"); + } + } + } + + const std::string &sourcefile = tok->location.file(); + const bool systemheader = (tok1 && tok1->op == '<'); + std::string header; + if (systemheader) { + simplecpp::Token *tok3 = tok1->next; + if (!tok3) { + throw std::runtime_error("missing __has_include closing angular bracket"); + } + while (tok3->op != '>') { + tok3 = tok3->next; + if (!tok3) { + throw std::runtime_error("invalid __has_include expression"); + } + } + + for (simplecpp::Token *headerToken = tok1->next; headerToken != tok3; headerToken = headerToken->next) + header += headerToken->str(); + header = realFilename(header); + } + else { + header = realFilename(tok1->str().substr(1U, tok1->str().size() - 2U)); + } + std::ifstream f; + const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); + tok->setstr(header2.empty() ? "0" : "1"); + + tok2 = tok2->next; + while (tok->next != tok2) + expr.deleteToken(tok->next); + } +} + static const char * const altopData[] = {"and","or","bitand","bitor","compl","not","not_eq","xor"}; static const std::set altop(&altopData[0], &altopData[8]); static void simplifyName(simplecpp::TokenList &expr) @@ -2694,10 +2751,11 @@ static void simplifyComments(simplecpp::TokenList &expr) } } -static long long evaluate(simplecpp::TokenList &expr, const std::map &sizeOfType) +static long long evaluate(simplecpp::TokenList &expr, const simplecpp::DUI &dui, const std::map &sizeOfType) { simplifyComments(expr); simplifySizeof(expr, sizeOfType); + simplifyHasInclude(expr, dui); simplifyName(expr); simplifyNumbers(expr); expr.constFold(); @@ -3258,17 +3316,30 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool par = (tok && tok->op == '('); if (par) tok = tok->next; + bool closingAngularBracket = false; if (tok) { const std::string &sourcefile = rawtok->location.file(); - const bool systemheader = (tok->str()[0] == '<'); - const std::string header(realFilename(tok->str().substr(1U, tok->str().size() - 2U))); + const bool systemheader = (tok && tok->op == '<'); + std::string header; + + if (systemheader) { + while ((tok = tok->next) && tok->op != '>') + header += tok->str(); + header = realFilename(header); + if (tok && tok->op == '>') + closingAngularBracket = true; + } + else { + header = realFilename(tok->str().substr(1U, tok->str().size() - 2U)); + closingAngularBracket = true; + } std::ifstream f; const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); expr.push_back(new Token(header2.empty() ? "0" : "1", tok->location)); } if (par) tok = tok ? tok->next : nullptr; - if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { + if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')') || (!closingAngularBracket)) { if (outputList) { Output out(rawtok->location.files); out.type = Output::SYNTAX_ERROR; @@ -3298,11 +3369,11 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::string E; for (const simplecpp::Token *tok = expr.cfront(); tok; tok = tok->next) E += (E.empty() ? "" : " ") + tok->str(); - const long long result = evaluate(expr, sizeOfType); + const long long result = evaluate(expr, dui, sizeOfType); conditionIsTrue = (result != 0); ifCond->push_back(IfCond(rawtok->location, E, result)); } else { - const long long result = evaluate(expr, sizeOfType); + const long long result = evaluate(expr, dui, sizeOfType); conditionIsTrue = (result != 0); } } catch (const std::exception &e) { diff --git a/test.cpp b/test.cpp index 988ed91a..2e8f1549 100644 --- a/test.cpp +++ b/test.cpp @@ -1221,7 +1221,7 @@ static void hashhash_universal_character() static void has_include_1() { const char code[] = "#ifdef __has_include\n" - " #ifdef __has_include(\"simplecpp.h\")\n" + " #if __has_include(\"simplecpp.h\")\n" " A\n" " #else\n" " B\n" @@ -1230,13 +1230,64 @@ static void has_include_1() simplecpp::DUI dui; dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } static void has_include_2() { const char code[] = "#if defined( __has_include)\n" - " #ifdef __has_include(\"simplecpp.h\")\n" + " #if /*commant*/ __has_include /*comment*/(\"simplecpp.h\") // comment\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + +static void has_include_3() +{ + const char code[] = "#ifdef __has_include\n" + " #if __has_include()\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "c++17"; + // Test file not found... + ASSERT_EQUALS("\n\n\n\nB", preprocess(code, dui)); + // Unless -I is set (preferably, we should differentiate -I and -isystem...) + dui.includePaths.push_back("./testsuite"); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + +static void has_include_4() +{ + const char code[] = "#ifdef __has_include\n" + " #if __has_include(\"testsuite/realFileName1.cpp\")\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + +static void has_include_5() +{ + const char code[] = "#if defined( __has_include)\n" + " #if !__has_include()\n" " A\n" " #else\n" " B\n" @@ -2480,6 +2531,9 @@ int main(int argc, char **argv) // c++17 __has_include TEST_CASE(has_include_1); TEST_CASE(has_include_2); + TEST_CASE(has_include_3); + TEST_CASE(has_include_4); + TEST_CASE(has_include_5); TEST_CASE(ifdef1); TEST_CASE(ifdef2); From fa8fd1a1f9d83e4c386818247cd0eecd29aa9b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 29 Sep 2022 21:50:03 +0200 Subject: [PATCH 173/381] `TokenList` move constructor was also calling the copy constructor (#272) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index fe7c4155..4aac986f 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -242,7 +242,7 @@ simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), b } #if __cplusplus >= 201103L -simplecpp::TokenList::TokenList(TokenList &&other) : TokenList(other) +simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(nullptr), backToken(nullptr), files(other.files) { *this = std::move(other); } From 3df6160ca85b1311c9497692a651f5c02681a969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 29 Oct 2022 21:23:10 +0200 Subject: [PATCH 174/381] main.cpp: detect and bail out on unknown options (#273) --- main.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/main.cpp b/main.cpp index a5d7996a..e84d9de7 100644 --- a/main.cpp +++ b/main.cpp @@ -24,45 +24,60 @@ int main(int argc, char **argv) { - const char *filename = nullptr; + bool error = false; // Settings.. + const char *filename = nullptr; simplecpp::DUI dui; bool quiet = false; for (int i = 1; i < argc; i++) { const char * const arg = argv[i]; if (*arg == '-') { + bool found = false; const char c = arg[1]; - if (c != 'D' && c != 'U' && c != 'I' && c != 'i' && c != 's' && c != 'q') - continue; // Ignored const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; switch (c) { case 'D': // define symbol dui.defines.push_back(value); + found = true; break; case 'U': // undefine symbol dui.undefined.insert(value); + found = true; break; case 'I': // include path dui.includePaths.push_back(value); + found = true; break; case 'i': - if (std::strncmp(arg, "-include=",9)==0) + if (std::strncmp(arg, "-include=",9)==0) { dui.includes.push_back(arg+9); + found = true; + } break; case 's': - if (std::strncmp(arg, "-std=",5)==0) + if (std::strncmp(arg, "-std=",5)==0) { dui.std = arg + 5; + found = true; + } break; case 'q': quiet = true; + found = true; break; } + if (!found) { + std::cout << "Option '" << arg << "' is unknown." << std::endl; + error = true; + } } else { filename = arg; } } + if (error) + std::exit(1); + if (!filename) { std::cout << "Syntax:" << std::endl; std::cout << "simplecpp [options] filename" << std::endl; From 501c9c633784f37cbc769bb0388c8c343228c6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 20 Nov 2022 08:39:50 +0100 Subject: [PATCH 175/381] avoid unnecessary copy in `openHeader()` - detected by upcoming clang-tidy check `performance-unnecessary-copy-on-last-use` (#276) --- simplecpp.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4aac986f..32d2b007 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2862,11 +2862,15 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (systemheader) { ret = openHeaderIncludePath(f, dui, header); - return ret.empty() ? openHeaderRelative(f, sourcefile, header) : ret; + if (ret.empty()) + return openHeaderRelative(f, sourcefile, header); + return ret; } ret = openHeaderRelative(f, sourcefile, header); - return ret.empty() ? openHeaderIncludePath(f, dui, header) : ret; + if (ret.empty()) + openHeaderIncludePath(f, dui, header); + return ret; } static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) From 3082bd70b8d2b6627d47300ab2341a73af73337b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 20 Nov 2022 08:40:22 +0100 Subject: [PATCH 176/381] adjusted includes based on include-what-you-use (#275) --- simplecpp.cpp | 6 ++++-- simplecpp.h | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 32d2b007..eed3751e 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -24,13 +24,15 @@ #include #include +#include +#include #include #include #include -#include +#include // IWYU pragma: keep #include #include -#include +#include // IWYU pragma: keep #include #include #if __cplusplus >= 201103L diff --git a/simplecpp.h b/simplecpp.h index 8e04a571..24443b07 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -20,7 +20,6 @@ #define simplecppH #include -#include #include #include #include From 76283eed517e8b7bd4735631f8ce5a1e6ddd08cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 23 Nov 2022 19:15:08 +0100 Subject: [PATCH 177/381] improved testing and CI (#278) --- .github/workflows/CI-unixish.yml | 30 +++++++++++++++++++++--------- .github/workflows/CI-windows.yml | 5 +++-- run-tests.py | 16 ++++++++++++++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 9128d3ee..a03e1d52 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - name: Install missing software on ubuntu - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' run: | sudo apt-get update sudo apt-get install valgrind @@ -28,16 +28,28 @@ jobs: - name: make test run: make -j$(nproc) test CXX=${{ matrix.compiler }} - - name: ensure that simplecpp.cpp uses c++03 - run: CXX=${{ matrix.compiler }} ; $CXX -fsyntax-only -std=c++98 simplecpp.cpp - - name: Run valgrind - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' run: | + make clean + # this valgrind version doesn't support DWARF 5 yet + make -j$(nproc) CXX=${{ matrix.compiler }} CXXFLAGS="-gdwarf-4" valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner + - name: Run with libstdc++ debug mode + if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'gcc' + run: | + make clean + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" + + - name: Run with libc++ debug mode + if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' + run: | + make clean + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -DLIBCXX_ENABLE_DEBUG_MODE" LDFLAGS="-lc++" + - name: Run AddressSanitizer - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' run: | make clean make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" @@ -45,14 +57,14 @@ jobs: ASAN_OPTIONS: detect_stack_use_after_return=1 - name: Run UndefinedBehaviorSanitizer - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' run: | make clean make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" - # TODO: enable when false positives are fixed + # TODO: requires instrumented libc++ - name: Run MemorySanitizer - if: false && matrix.os == 'ubuntu-20.04' && matrix.compiler == 'clang++' + if: false && matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 264a09c6..220c34a2 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -16,6 +16,7 @@ jobs: strategy: matrix: os: [windows-2019, windows-2022] + config: [Release, Debug] fail-fast: false runs-on: ${{ matrix.os }} @@ -40,9 +41,9 @@ jobs: - name: Build run: | - msbuild -m simplecpp.sln /p:Configuration=Release /p:Platform=x64 || exit /b !errorlevel! + msbuild -m simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! - name: Test run: | - .\Release\testrunner.exe || exit /b !errorlevel! + .\${{ matrix.config }}\testrunner.exe || exit /b !errorlevel! diff --git a/run-tests.py b/run-tests.py index 3dc4a9b7..c053ed32 100644 --- a/run-tests.py +++ b/run-tests.py @@ -2,6 +2,7 @@ import glob import os import subprocess +import sys def cleanup(out): ret = '' @@ -80,6 +81,7 @@ def cleanup(out): numberOfSkipped = 0 numberOfFailed = 0 +numberOfFixed = 0 usedTodos = [] @@ -101,10 +103,13 @@ def cleanup(out): gcc_output = cleanup(comm[0]) simplecpp_cmd = ['./simplecpp'] - simplecpp_cmd.extend(cmd.split(' ')) + # -E is not supported and we bail out on unknown options + simplecpp_cmd.extend(cmd.replace('-E ', '', 1).split(' ')) p = subprocess.Popen(simplecpp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) comm = p.communicate() + simplecpp_ec = p.returncode simplecpp_output = cleanup(comm[0]) + simplecpp_err = comm[0].decode('utf-8').strip() if simplecpp_output != clang_output and simplecpp_output != gcc_output: filename = cmd[cmd.rfind('/')+1:] @@ -113,14 +118,21 @@ def cleanup(out): usedTodos.append(filename) else: print('FAILED ' + cmd) + if simplecpp_ec: + print('simplecpp failed - ' + simplecpp_err) numberOfFailed = numberOfFailed + 1 for filename in todo: if not filename in usedTodos: print('FIXED ' + filename) + numberOfFixed = numberOfFixed + 1 print('Number of tests: ' + str(len(commands))) print('Number of skipped: ' + str(numberOfSkipped)) -print('Number of todos: ' + str(len(usedTodos))) +print('Number of todos (fixed): ' + str(len(usedTodos)) + ' (' + str(numberOfFixed) + ')') print('Number of failed: ' + str(numberOfFailed)) +if numberOfFailed or numberOfFixed: + sys.exit(1) + +sys.exit(0) From 2c97b44e02c7688e2baa472472a13f575e5a738a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 3 Dec 2022 15:42:37 +0100 Subject: [PATCH 178/381] address deprecation warnings in GitHub workflows / fixed check for `libstdc++` debug mode step (#280) --- .github/workflows/CI-unixish.yml | 6 +++--- .github/workflows/CI-windows.yml | 4 ++-- .github/workflows/clang-tidy.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index a03e1d52..d800ae4b 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,13 +8,13 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] + os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] fail-fast: false runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install missing software on ubuntu if: matrix.os == 'ubuntu-22.04' @@ -37,7 +37,7 @@ jobs: valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner - name: Run with libstdc++ debug mode - if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'gcc' + if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'g++' run: | make clean make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 220c34a2..aca68c93 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -22,10 +22,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup msbuild.exe - uses: microsoft/setup-msbuild@v1.0.2 + uses: microsoft/setup-msbuild@v1.1 - name: Run cmake if: matrix.os == 'windows-2019' diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index e1594f60..1202f5df 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install missing software run: | From 8abab5c73dd6fc951cc22f21d89381e5b4e70c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 7 Dec 2022 09:34:02 +0100 Subject: [PATCH 179/381] optimized `TokenList::readfile()` a bit (#277) --- simplecpp.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index eed3751e..c090dbf9 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -138,7 +138,7 @@ static bool startsWith(const std::string &str, const std::string &s) static bool endsWith(const std::string &s, const std::string &e) { - return (s.size() >= e.size() && s.compare(s.size() - e.size(), e.size(), e) == 0); + return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); } static bool sameline(const simplecpp::Token *tok1, const simplecpp::Token *tok2) @@ -568,7 +568,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen TokenString currentToken; - if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && (lastLine() == "# error" || lastLine() == "# warning")) { + if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && isLastLinePreprocessor() && (lastLine() == "# error" || lastLine() == "# warning")) { char prev = ' '; while (istr.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; @@ -629,7 +629,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen currentToken.erase(pos,2); ++multiline; } - if (multiline || startsWith(lastLine(10),"# ")) { + if (multiline || isLastLinePreprocessor()) { pos = 0; while ((pos = currentToken.find('\n',pos)) != std::string::npos) { currentToken.erase(pos,1); @@ -710,7 +710,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen else back()->setstr(prefix + s); - if (newlines > 0 && lastLine().compare(0,9,"# define ") == 0) { + if (newlines > 0 && isLastLinePreprocessor() && lastLine().compare(0,9,"# define ") == 0) { multiline += newlines; location.adjust(s); } else { @@ -723,7 +723,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen currentToken += ch; } - if (currentToken == "<" && lastLine() == "# include") { + if (*currentToken.begin() == '<' && isLastLinePreprocessor() && lastLine() == "# include") { currentToken = readUntil(istr, location, '<', '>', outputList, bom); if (currentToken.size() < 2U) return; From 45db93b7447b34b6391b1de9b95d6f4ad06c5595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 8 Dec 2022 20:13:12 +0100 Subject: [PATCH 180/381] fixed missing return in `openHeader()` (#281) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c090dbf9..2b7b8b00 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2871,7 +2871,7 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const ret = openHeaderRelative(f, sourcefile, header); if (ret.empty()) - openHeaderIncludePath(f, dui, header); + return openHeaderIncludePath(f, dui, header); return ret; } From 0214612418c1ab9461047457b664154b70fa6f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 2 Jan 2023 22:11:29 +0100 Subject: [PATCH 181/381] CI-unixish.yml: specify proper define for "safe libc++" (#284) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index d800ae4b..7a64ee14 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -46,7 +46,7 @@ jobs: if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -DLIBCXX_ENABLE_DEBUG_MODE" LDFLAGS="-lc++" + make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" - name: Run AddressSanitizer if: matrix.os == 'ubuntu-22.04' From 7c5b21d11e8fdda28817418b5977e95c656c20c8 Mon Sep 17 00:00:00 2001 From: bavison Date: Wed, 18 Jan 2023 19:34:33 +0000 Subject: [PATCH 182/381] Includes of system headers are never implicitly relative to the source file (#282) Sometimes, #include directives with angle-bracket filespec delimiters are used (or abused) to defeat the preprocessor's behaviour where it tries to find a header file at a path relative to the file containing the directive. Without this fix, any non-root header file, foo/bar.h, which does #include while intending to include a root-level header file, will instead enter an infinite inclusion loop, terminating when the inclusion stack overflows with a "#include nested too deeply" error. --- simplecpp.cpp | 8 +++----- test.cpp | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2b7b8b00..85399150 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2864,8 +2864,6 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (systemheader) { ret = openHeaderIncludePath(f, dui, header); - if (ret.empty()) - return openHeaderRelative(f, sourcefile, header); return ret; } @@ -2894,8 +2892,8 @@ static std::string getFileName(const std::mapop == '<'); + const bool systemheader = (inctok->str()[0] == '<'); const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui, systemheader); if (header2.empty()) { diff --git a/test.cpp b/test.cpp index 2e8f1549..c8323c85 100644 --- a/test.cpp +++ b/test.cpp @@ -1653,6 +1653,22 @@ static void nestedInclude() ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } +static void systemInclude() +{ + const char code[] = "#include \n"; + std::vector files; + simplecpp::TokenList rawtokens = makeTokenList(code,files,"local/limits.h"); + std::map filedata; + filedata["limits.h"] = nullptr; + filedata["local/limits.h"] = &rawtokens; + + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); + + ASSERT_EQUALS("", toString(outputList)); +} + static void multiline1() { const char code[] = "#define A \\\n" @@ -2566,6 +2582,7 @@ int main(int argc, char **argv) TEST_CASE(missingHeader2); TEST_CASE(missingHeader3); TEST_CASE(nestedInclude); + TEST_CASE(systemInclude); TEST_CASE(nullDirective1); TEST_CASE(nullDirective2); From 9dc2c3df53ee0caf76906e596306f7ad70fc2a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 18 Jan 2023 20:48:11 +0100 Subject: [PATCH 183/381] added `DUI::clearIncludeCache` to clear the non-existing files cache (#287) --- simplecpp.cpp | 29 ++++++++++++++++++++--------- simplecpp.h | 3 ++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 85399150..95d72136 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2796,6 +2796,11 @@ class NonExistingFilesCache { m_pathSet.insert(path); } + void clear() { + ScopedLock lock(m_criticalSection); + m_pathSet.clear(); + } + private: std::set m_pathSet; CRITICAL_SECTION m_criticalSection; @@ -2807,22 +2812,18 @@ static NonExistingFilesCache nonExistingFilesCache; static std::string openHeader(std::ifstream &f, const std::string &path) { -#ifdef SIMPLECPP_WINDOWS std::string simplePath = simplecpp::simplifyPath(path); +#ifdef SIMPLECPP_WINDOWS if (nonExistingFilesCache.contains(simplePath)) return ""; // file is known not to exist, skip expensive file open call - +#endif f.open(simplePath.c_str()); if (f.is_open()) return simplePath; - else { - nonExistingFilesCache.add(simplePath); - return ""; - } -#else - f.open(path.c_str()); - return f.is_open() ? simplecpp::simplifyPath(path) : ""; +#ifdef SIMPLECPP_WINDOWS + nonExistingFilesCache.add(simplePath); #endif + return ""; } static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) @@ -2905,6 +2906,11 @@ static bool hasFile(const std::map &filedat std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { +#ifdef SIMPLECPP_WINDOWS + if (dui.clearIncludeCache) + nonExistingFilesCache .clear(); +#endif + std::map ret; std::list filelist; @@ -3030,6 +3036,11 @@ static std::string getTimeDefine(struct tm *timep) void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage, std::list *ifCond) { +#ifdef SIMPLECPP_WINDOWS + if (dui.clearIncludeCache) + nonExistingFilesCache.clear(); +#endif + std::map sizeOfType(rawtokens.sizeOfType); sizeOfType.insert(std::make_pair("char", sizeof(char))); sizeOfType.insert(std::make_pair("short", sizeof(short))); diff --git a/simplecpp.h b/simplecpp.h index 24443b07..7f4b3c63 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -314,12 +314,13 @@ namespace simplecpp { * On the command line these are configured by -D, -U, -I, --include, -std */ struct SIMPLECPP_LIB DUI { - DUI() {} + DUI() : clearIncludeCache(false) {} std::list defines; std::set undefined; std::list includePaths; std::list includes; std::string std; + bool clearIncludeCache; }; SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); From 8975f0277bb17deda422f0d348bc31e55a8feaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 30 Jan 2023 08:12:20 +0100 Subject: [PATCH 184/381] some `Token::flags()` improvements (#285) --- simplecpp.h | 4 ++-- test.cpp | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 7f4b3c63..3412da59 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -113,8 +113,8 @@ namespace simplecpp { name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); - number = std::isdigit(static_cast(string[0])) || (string.size() > 1U && string[0] == '-' && std::isdigit(static_cast(string[1]))); - op = (string.size() == 1U) ? string[0] : '\0'; + number = std::isdigit(static_cast(string[0])) || (string.size() > 1U && (string[0] == '-' || string[0] == '+') && std::isdigit(static_cast(string[1]))); + op = (string.size() == 1U && !name && !comment && !number) ? string[0] : '\0'; } const TokenString& str() const { diff --git a/test.cpp b/test.cpp index c8323c85..41fbc972 100644 --- a/test.cpp +++ b/test.cpp @@ -2440,6 +2440,64 @@ static void cpluscplusDefine() ASSERT_EQUALS("\n201103L", preprocess(code, dui)); } +static void assertToken(const std::string& s, bool name, bool number, bool comment, char op, int line) +{ + const std::vector f; + const simplecpp::Location l(f); + const simplecpp::Token t(s, l); + assertEquals(name, t.name, line); + assertEquals(number, t.number, line); + assertEquals(comment, t.comment, line); + assertEquals(op, t.op, line); +} + +#define ASSERT_TOKEN(s, na, nu, c) assertToken(s, na, nu, c, '\0', __LINE__) +#define ASSERT_TOKEN_OP(s, na, nu, c, o) assertToken(s, na, nu, c, o, __LINE__) + +static void token() +{ + // name + ASSERT_TOKEN("n", true, false, false); + ASSERT_TOKEN("name", true, false, false); + ASSERT_TOKEN("name_1", true, false, false); + ASSERT_TOKEN("name2", true, false, false); + ASSERT_TOKEN("name$", true, false, false); + + // character literal + ASSERT_TOKEN("'n'", false, false, false); + ASSERT_TOKEN("'\\''", false, false, false); + ASSERT_TOKEN("'\\u0012'", false, false, false); + ASSERT_TOKEN("'\\xff'", false, false, false); + ASSERT_TOKEN("u8'\\u0012'", false, false, false); + ASSERT_TOKEN("u'\\u0012'", false, false, false); + ASSERT_TOKEN("L'\\u0012'", false, false, false); + ASSERT_TOKEN("U'\\u0012'", false, false, false); + + // include + ASSERT_TOKEN("", false, false, false); + + // comment + ASSERT_TOKEN("/*comment*/", false, false, true); + ASSERT_TOKEN("// TODO", false, false, true); + + // string literal + ASSERT_TOKEN("\"literal\"", false, false, false); + + // op + ASSERT_TOKEN_OP("<", false, false, false, '<'); + ASSERT_TOKEN_OP(">", false, false, false, '>'); + ASSERT_TOKEN_OP("(", false, false, false, '('); + ASSERT_TOKEN_OP(")", false, false, false, ')'); + + // number + ASSERT_TOKEN("2", false, true, false); + ASSERT_TOKEN("22", false, true, false); + ASSERT_TOKEN("-2", false, true, false); + ASSERT_TOKEN("-22", false, true, false); + ASSERT_TOKEN("+2", false, true, false); + ASSERT_TOKEN("+22", false, true, false); +} + int main(int argc, char **argv) { TEST_CASE(backslash); @@ -2648,5 +2706,7 @@ int main(int argc, char **argv) TEST_CASE(stdcVersionDefine); TEST_CASE(cpluscplusDefine); + TEST_CASE(token); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 57853a34dc8fdd5751281db321253fc8ef171bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 2 Mar 2023 20:43:13 +0100 Subject: [PATCH 185/381] optimized `simplifyPath()` a bit (#289) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 95d72136..7f3e8bef 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2355,7 +2355,7 @@ namespace simplecpp { if (unc) path = '/' + path; - return path.find_first_of("*?") == std::string::npos ? realFilename(path) : path; + return strpbrk(path.c_str(), "*?") == nullptr ? realFilename(path) : path; } } From 00d1f672c5ae96e8a344d8e389edbb6fabeb576b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 2 Mar 2023 22:13:46 +0100 Subject: [PATCH 186/381] added `TokenList::Stream` class to wrap `std::istream` usage and implemented alternative C I/O version (#244) --- .clang-tidy | 2 +- CMakeLists.txt | 4 +- main.cpp | 26 +++- simplecpp.cpp | 364 ++++++++++++++++++++++++++++++++----------------- simplecpp.h | 9 +- test.cpp | 54 +++++++- 6 files changed, 317 insertions(+), 142 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 244e860f..1005f909 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-core.NullDereference,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move' +Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-core.NullDereference,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move,-modernize-use-override' HeaderFilterRegex: '.*' WarningsAsErrors: '*' CheckOptions: diff --git a/CMakeLists.txt b/CMakeLists.txt index 26398823..e88afe5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,9 +10,11 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # no need for c++98 compatibility add_compile_options(-Wno-c++98-compat-pedantic) # these are not really fixable - add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors) + add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors -Wno-weak-vtables) # we are not interested in these add_compile_options(-Wno-multichar -Wno-four-char-constants) + # ignore C++11-specific warning + add_compile_options(-Wno-suggest-override -Wno-suggest-destructor-override) # TODO: fix these? add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32 -Wno-shadow-field-in-constructor) diff --git a/main.cpp b/main.cpp index e84d9de7..6592bf6c 100644 --- a/main.cpp +++ b/main.cpp @@ -25,9 +25,10 @@ int main(int argc, char **argv) { bool error = false; + const char *filename = nullptr; + bool use_istream = false; // Settings.. - const char *filename = nullptr; simplecpp::DUI dui; bool quiet = false; for (int i = 1; i < argc; i++) { @@ -54,6 +55,10 @@ int main(int argc, char **argv) dui.includes.push_back(arg+9); found = true; } + else if (std::strncmp(arg, "-is",3)==0) { + use_istream = true; + found = true; + } break; case 's': if (std::strncmp(arg, "-std=",5)==0) { @@ -87,20 +92,29 @@ int main(int argc, char **argv) std::cout << " -UNAME Undefine NAME." << std::endl; std::cout << " -std=STD Specify standard." << std::endl; std::cout << " -q Quiet mode (no output)." << std::endl; + std::cout << " -is Use std::istream interface." << std::endl; std::exit(0); } // Perform preprocessing simplecpp::OutputList outputList; std::vector files; - std::ifstream f(filename); - simplecpp::TokenList rawtokens(f,files,filename,&outputList); - rawtokens.removeComments(); - std::map included = simplecpp::load(rawtokens, files, dui, &outputList); + simplecpp::TokenList *rawtokens; + if (use_istream) { + std::ifstream f(filename); + rawtokens = new simplecpp::TokenList(f, files,filename,&outputList); + } + else { + rawtokens = new simplecpp::TokenList(filename,files,&outputList); + } + rawtokens->removeComments(); + std::map included = simplecpp::load(*rawtokens, files, dui, &outputList); for (std::pair i : included) i.second->removeComments(); simplecpp::TokenList outputTokens(files); - simplecpp::preprocess(outputTokens, rawtokens, files, included, dui, &outputList); + simplecpp::preprocess(outputTokens, *rawtokens, files, included, dui, &outputList); + delete rawtokens; + rawtokens = nullptr; // Output if (!quiet) { diff --git a/simplecpp.cpp b/simplecpp.cpp index 7f3e8bef..2adfa898 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -230,12 +230,209 @@ void simplecpp::Token::printOut() const std::cout << std::endl; } +class simplecpp::TokenList::Stream { +public: + virtual ~Stream() {} + + virtual int get() = 0; + virtual int peek() = 0; + virtual void unget() = 0; + virtual bool good() = 0; + + unsigned char readChar() + { + unsigned char ch = static_cast(get()); + + // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the + // character is non-ASCII character then replace it with 0xff + if (isUtf16) { + const unsigned char ch2 = static_cast(get()); + const int ch16 = makeUtf16Char(ch, ch2); + ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); + } + + // Handling of newlines.. + if (ch == '\r') { + ch = '\n'; + + int ch2 = get(); + if (isUtf16) { + const int c2 = get(); + ch2 = makeUtf16Char(ch2, c2); + } + + if (ch2 != '\n') + ungetChar(); + } + + return ch; + } + + unsigned char peekChar() + { + unsigned char ch = static_cast(peek()); + + // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the + // character is non-ASCII character then replace it with 0xff + if (isUtf16) { + (void)get(); + const unsigned char ch2 = static_cast(peek()); + unget(); + const int ch16 = makeUtf16Char(ch, ch2); + ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); + } + + // Handling of newlines.. + if (ch == '\r') + ch = '\n'; + + return ch; + } + + void ungetChar() + { + unget(); + if (isUtf16) + unget(); + } + +protected: + void init() { + // initialize since we use peek() in getAndSkipBOM() + isUtf16 = false; + bom = getAndSkipBOM(); + isUtf16 = (bom == 0xfeff || bom == 0xfffe); + } + +private: + inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const + { + return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); + } + + unsigned short getAndSkipBOM() + { + const int ch1 = peek(); + + // The UTF-16 BOM is 0xfffe or 0xfeff. + if (ch1 >= 0xfe) { + (void)get(); + const unsigned short byte = (static_cast(ch1) << 8); + if (peek() >= 0xfe) + return byte | static_cast(get()); + unget(); + return 0; + } + + // Skip UTF-8 BOM 0xefbbbf + if (ch1 == 0xef) { + (void)get(); + if (peek() == 0xbb) { + (void)get(); + if (peek() == 0xbf) { + (void)get(); + return 0; + } + unget(); + } + unget(); + } + + return 0; + } + + unsigned short bom; +protected: + bool isUtf16; +}; + +class StdIStream : public simplecpp::TokenList::Stream { +public: + StdIStream(std::istream &istr) + : istr(istr) + { + init(); + } + + virtual int get() { + return istr.get(); + } + virtual int peek() { + return istr.peek(); + } + virtual void unget() { + istr.unget(); + } + virtual bool good() { + return istr.good(); + } + +private: + std::istream &istr; +}; + +class FileStream : public simplecpp::TokenList::Stream { +public: + FileStream(const std::string &filename) + : file(fopen(filename.c_str(), "rb")) + , lastCh(0) + , lastStatus(0) + { + init(); + } + + ~FileStream() { + fclose(file); + file = nullptr; + } + + virtual int get() { + lastStatus = lastCh = fgetc(file); + return lastCh; + } + virtual int peek() { + // keep lastCh intact + const int ch = fgetc(file); + unget_internal(ch); + return ch; + } + virtual void unget() { + unget_internal(lastCh); + } + virtual bool good() { + return lastStatus != EOF; + } + +private: + void unget_internal(int ch) { + if (isUtf16) { + // TODO: use ungetc() as well + // UTF-16 has subsequent unget() calls + fseek(file, -1, SEEK_CUR); + } + else + ungetc(ch, file); + } + + FILE *file; + int lastCh; + int lastStatus; +}; + simplecpp::TokenList::TokenList(std::vector &filenames) : frontToken(nullptr), backToken(nullptr), files(filenames) {} simplecpp::TokenList::TokenList(std::istream &istr, std::vector &filenames, const std::string &filename, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { - readfile(istr,filename,outputList); + StdIStream stream(istr); + readfile(stream,filename,outputList); +} + +simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + FileStream stream(filename); + readfile(stream,filename,outputList); } simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), backToken(nullptr), files(other.files) @@ -335,92 +532,6 @@ std::string simplecpp::TokenList::stringify() const return ret.str(); } -static unsigned char readChar(std::istream &istr, unsigned int bom) -{ - unsigned char ch = static_cast(istr.get()); - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (bom == 0xfeff || bom == 0xfffe) { - const unsigned char ch2 = static_cast(istr.get()); - const int ch16 = (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); - ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); - } - - // Handling of newlines.. - if (ch == '\r') { - ch = '\n'; - if (bom == 0 && static_cast(istr.peek()) == '\n') - (void)istr.get(); - else if (bom == 0xfeff || bom == 0xfffe) { - const int c1 = istr.get(); - const int c2 = istr.get(); - const int ch16 = (bom == 0xfeff) ? (c1<<8 | c2) : (c2<<8 | c1); - if (ch16 != '\n') { - istr.unget(); - istr.unget(); - } - } - } - - return ch; -} - -static unsigned char peekChar(std::istream &istr, unsigned int bom) -{ - unsigned char ch = static_cast(istr.peek()); - - // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the - // character is non-ASCII character then replace it with 0xff - if (bom == 0xfeff || bom == 0xfffe) { - (void)istr.get(); - const unsigned char ch2 = static_cast(istr.peek()); - istr.unget(); - const int ch16 = (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); - ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); - } - - // Handling of newlines.. - if (ch == '\r') - ch = '\n'; - - return ch; -} - -static void ungetChar(std::istream &istr, unsigned int bom) -{ - istr.unget(); - if (bom == 0xfeff || bom == 0xfffe) - istr.unget(); -} - -static unsigned short getAndSkipBOM(std::istream &istr) -{ - const int ch1 = istr.peek(); - - // The UTF-16 BOM is 0xfffe or 0xfeff. - if (ch1 >= 0xfe) { - const unsigned short bom = (static_cast(istr.get()) << 8); - if (istr.peek() >= 0xfe) - return bom | static_cast(istr.get()); - istr.unget(); - return 0; - } - - // Skip UTF-8 BOM 0xefbbbf - if (ch1 == 0xef) { - (void)istr.get(); - if (istr.get() == 0xbb && istr.peek() == 0xbf) { - (void)istr.get(); - } else { - istr.unget(); - istr.unget(); - } - } - - return 0; -} - static bool isNameChar(unsigned char ch) { return std::isalnum(ch) || ch == '_' || ch == '$'; @@ -476,7 +587,7 @@ void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int li static const std::string COMMENT_END("*/"); -void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filename, OutputList *outputList) +void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, OutputList *outputList) { std::stack loc; @@ -484,15 +595,13 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen const Token *oldLastToken = nullptr; - const unsigned short bom = getAndSkipBOM(istr); - Location location(files); location.fileIndex = fileIndex(filename); location.line = 1U; location.col = 1U; - while (istr.good()) { - unsigned char ch = readChar(istr,bom); - if (!istr.good()) + while (stream.good()) { + unsigned char ch = stream.readChar(); + if (!stream.good()) break; if (ch < ' ' && ch != '\t' && ch != '\n' && ch != '\r') ch = ' '; @@ -570,12 +679,12 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && isLastLinePreprocessor() && (lastLine() == "# error" || lastLine() == "# warning")) { char prev = ' '; - while (istr.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { + while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; prev = ch; - ch = readChar(istr, bom); + ch = stream.readChar(); } - ungetChar(istr, bom); + stream.ungetChar(); push_back(new Token(currentToken, location)); location.adjust(currentToken); continue; @@ -584,21 +693,21 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen // number or name if (isNameChar(ch)) { const bool num = std::isdigit(ch); - while (istr.good() && isNameChar(ch)) { + while (stream.good() && isNameChar(ch)) { currentToken += ch; - ch = readChar(istr,bom); - if (num && ch=='\'' && isNameChar(peekChar(istr,bom))) - ch = readChar(istr,bom); + ch = stream.readChar(); + if (num && ch=='\'' && isNameChar(stream.peekChar())) + ch = stream.readChar(); } - ungetChar(istr,bom); + stream.ungetChar(); } // comment - else if (ch == '/' && peekChar(istr,bom) == '/') { - while (istr.good() && ch != '\r' && ch != '\n') { + else if (ch == '/' && stream.peekChar() == '/') { + while (stream.good() && ch != '\r' && ch != '\n') { currentToken += ch; - ch = readChar(istr, bom); + ch = stream.readChar(); } const std::string::size_type pos = currentToken.find_last_not_of(" \t"); if (pos < currentToken.size() - 1U && currentToken[pos] == '\\') @@ -607,20 +716,20 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen ++multiline; currentToken.erase(currentToken.size() - 1U); } else { - ungetChar(istr, bom); + stream.ungetChar(); } } // comment - else if (ch == '/' && peekChar(istr,bom) == '*') { + else if (ch == '/' && stream.peekChar() == '*') { currentToken = "/*"; - (void)readChar(istr,bom); - ch = readChar(istr,bom); - while (istr.good()) { + (void)stream.readChar(); + ch = stream.readChar(); + while (stream.good()) { currentToken += ch; if (currentToken.size() >= 4U && endsWith(currentToken, COMMENT_END)) break; - ch = readChar(istr,bom); + ch = stream.readChar(); } // multiline.. @@ -651,12 +760,12 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen std::string delim; currentToken = ch; prefix.resize(prefix.size() - 1); - ch = readChar(istr,bom); - while (istr.good() && ch != '(' && ch != '\n') { + ch = stream.readChar(); + while (stream.good() && ch != '(' && ch != '\n') { delim += ch; - ch = readChar(istr,bom); + ch = stream.readChar(); } - if (!istr.good() || ch == '\n') { + if (!stream.good() || ch == '\n') { if (outputList) { Output err(files); err.type = Output::SYNTAX_ERROR; @@ -667,8 +776,8 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen return; } const std::string endOfRawString(')' + delim + currentToken); - while (istr.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) - currentToken += readChar(istr,bom); + while (stream.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) + currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { Output err(files); @@ -692,7 +801,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen continue; } - currentToken = readUntil(istr,location,ch,ch,outputList,bom); + currentToken = readUntil(stream,location,ch,ch,outputList); if (currentToken.size() < 2U) // Error is reported by readUntil() return; @@ -724,7 +833,7 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen } if (*currentToken.begin() == '<' && isLastLinePreprocessor() && lastLine() == "# include") { - currentToken = readUntil(istr, location, '<', '>', outputList, bom); + currentToken = readUntil(stream, location, '<', '>', outputList); if (currentToken.size() < 2U) return; } @@ -1169,15 +1278,15 @@ void simplecpp::TokenList::removeComments() } } -std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location &location, const char start, const char end, OutputList *outputList, unsigned int bom) +std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &location, const char start, const char end, OutputList *outputList) { std::string ret; ret += start; bool backslash = false; char ch = 0; - while (ch != end && ch != '\r' && ch != '\n' && istr.good()) { - ch = readChar(istr, bom); + while (ch != end && ch != '\r' && ch != '\n' && stream.good()) { + ch = stream.readChar(); if (backslash && ch == '\n') { ch = 0; backslash = false; @@ -1189,7 +1298,7 @@ std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location & bool update_ch = false; char next = 0; do { - next = readChar(istr, bom); + next = stream.readChar(); if (next == '\r' || next == '\n') { ret.erase(ret.size()-1U); backslash = (next == '\r'); @@ -1203,7 +1312,7 @@ std::string simplecpp::TokenList::readUntil(std::istream &istr, const Location & } } - if (!istr.good() || ch != end) { + if (!stream.good() || ch != end) { clear(); if (outputList) { Output err(files); @@ -1300,7 +1409,8 @@ namespace simplecpp { Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); std::istringstream istr(def); - tokenListDefine.readfile(istr); + StdIStream stream(istr); + tokenListDefine.readfile(stream); if (!parseDefine(tokenListDefine.cfront())) throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); } @@ -2933,8 +3043,9 @@ std::map simplecpp::load(const simplecpp::To } continue; } + fin.close(); - TokenList *tokenlist = new TokenList(fin, filenames, filename, outputList); + TokenList *tokenlist = new TokenList(filename, filenames, outputList); if (!tokenlist->front()) { delete tokenlist; continue; @@ -2972,8 +3083,9 @@ std::map simplecpp::load(const simplecpp::To const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); if (!f.is_open()) continue; + f.close(); - TokenList *tokens = new TokenList(f, filenames, header2, outputList); + TokenList *tokens = new TokenList(header2, filenames, outputList); ret[header2] = tokens; if (tokens->front()) filelist.push_back(tokens->front()); diff --git a/simplecpp.h b/simplecpp.h index 3412da59..5b918a98 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -193,8 +193,13 @@ namespace simplecpp { /** List of tokens. */ class SIMPLECPP_LIB TokenList { public: + class Stream; + explicit TokenList(std::vector &filenames); + /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given filename parameter */ + TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); #if __cplusplus >= 201103L TokenList(TokenList &&other); @@ -214,7 +219,7 @@ namespace simplecpp { void dump() const; std::string stringify() const; - void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = nullptr); + void readfile(Stream &stream, const std::string &filename=std::string(), OutputList *outputList = nullptr); void constFold(); void removeComments(); @@ -279,7 +284,7 @@ namespace simplecpp { void constFoldLogicalOp(Token *tok); void constFoldQuestionOp(Token **tok1); - std::string readUntil(std::istream &istr, const Location &location, char start, char end, OutputList *outputList, unsigned int bom); + std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); std::string lastLine(int maxsize=100000) const; diff --git a/test.cpp b/test.cpp index 41fbc972..b0ba751a 100644 --- a/test.cpp +++ b/test.cpp @@ -77,17 +77,15 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) - -static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) +static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(code); + std::istringstream istr(std::string(code, size)); return simplecpp::TokenList(istr,filenames,filename,outputList); } -static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) +static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(std::string(code, size)); - return simplecpp::TokenList(istr,filenames,filename,outputList); + return makeTokenList(code, strlen(code), filenames, filename, outputList); } static std::string readfile(const char code[], simplecpp::OutputList *outputList=nullptr) @@ -2235,6 +2233,12 @@ static void utf8() ASSERT_EQUALS("123", readfile("\xEF\xBB\xBF 123")); } +static void utf8_invalid() +{ + ASSERT_EQUALS("", readfile("\xEF 123")); + ASSERT_EQUALS("", readfile("\xEF\xBB 123")); +} + static void unicode() { { @@ -2267,6 +2271,42 @@ static void unicode() } } +static void unicode_invalid() +{ + { + const char code[] = "\xFF"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE\xFF\x31"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x31"; + ASSERT_EQUALS("1", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE\xFF\x31\x32"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x31\x32"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFE\xFF\x00\x31\x00\x32\x33"; + ASSERT_EQUALS("", readfile(code, sizeof(code))); + } + { + const char code[] = "\xFF\xFE\x31\x00\x32\x00\x33"; + ASSERT_EQUALS("123", readfile(code, sizeof(code))); + } +} + static void warning() { const char code[] = "#warning MSG\n1"; @@ -2689,7 +2729,9 @@ int main(int argc, char **argv) // utf/unicode TEST_CASE(utf8); + TEST_CASE(utf8_invalid); TEST_CASE(unicode); + TEST_CASE(unicode_invalid); TEST_CASE(warning); From db1f61af5dde75eb84087b581dad4b3a19d55b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 12 Mar 2023 11:13:06 +0100 Subject: [PATCH 187/381] added selfcheck / simplecpp: added `-e` to report errors only (#290) --- .github/workflows/CI-unixish.yml | 17 ++++++++++------ .github/workflows/CI-windows.yml | 4 ++++ Makefile | 3 +++ appveyor.yml | 1 + main.cpp | 33 +++++++++++++++++++++++++++++--- selfcheck.sh | 6 ++++++ 6 files changed, 55 insertions(+), 9 deletions(-) create mode 100755 selfcheck.sh diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 7a64ee14..3a604900 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -1,4 +1,4 @@ -name: CI Unixish +name: CI-unixish on: [push, pull_request] @@ -28,6 +28,10 @@ jobs: - name: make test run: make -j$(nproc) test CXX=${{ matrix.compiler }} + - name: selfcheck + run: | + make -j$(nproc) selfcheck CXX=${{ matrix.compiler }} + - name: Run valgrind if: matrix.os == 'ubuntu-22.04' run: | @@ -35,24 +39,25 @@ jobs: # this valgrind version doesn't support DWARF 5 yet make -j$(nproc) CXX=${{ matrix.compiler }} CXXFLAGS="-gdwarf-4" valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner + valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./simplecpp simplecpp.cpp -e - name: Run with libstdc++ debug mode if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'g++' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" + make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" - name: Run with libc++ debug mode if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" + make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" - name: Run AddressSanitizer if: matrix.os == 'ubuntu-22.04' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" + make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" env: ASAN_OPTIONS: detect_stack_use_after_return=1 @@ -60,11 +65,11 @@ jobs: if: matrix.os == 'ubuntu-22.04' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" # TODO: requires instrumented libc++ - name: Run MemorySanitizer if: false && matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" + make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index aca68c93..1c4e3404 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -46,4 +46,8 @@ jobs: - name: Test run: | .\${{ matrix.config }}\testrunner.exe || exit /b !errorlevel! + + - name: Selfcheck + run: | + .\${{ matrix.config }}\simplecpp.exe simplecpp.cpp -e || exit /b !errorlevel! diff --git a/Makefile b/Makefile index a209666e..db1ca257 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ test: testrunner simplecpp ./testrunner python3 run-tests.py +selfcheck: simplecpp + ./selfcheck.sh + simplecpp: main.o simplecpp.o $(CXX) $(LDFLAGS) main.o simplecpp.o -o simplecpp diff --git a/appveyor.yml b/appveyor.yml index 00136602..ea8dd1df 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,3 +17,4 @@ build_script: test_script: - debug\testrunner.exe + - debug\simplecpp.exe simplecpp.cpp -e diff --git a/main.cpp b/main.cpp index 6592bf6c..7516085f 100644 --- a/main.cpp +++ b/main.cpp @@ -31,25 +31,34 @@ int main(int argc, char **argv) // Settings.. simplecpp::DUI dui; bool quiet = false; + bool error_only = false; for (int i = 1; i < argc; i++) { const char * const arg = argv[i]; if (*arg == '-') { bool found = false; const char c = arg[1]; - const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; switch (c) { case 'D': // define symbol + { + const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.defines.push_back(value); found = true; break; + } case 'U': // undefine symbol + { + const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.undefined.insert(value); found = true; break; + } case 'I': // include path + { + const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.includePaths.push_back(value); found = true; break; + } case 'i': if (std::strncmp(arg, "-include=",9)==0) { dui.includes.push_back(arg+9); @@ -70,11 +79,18 @@ int main(int argc, char **argv) quiet = true; found = true; break; + case 'e': + error_only = true; + found = true; + break; } if (!found) { - std::cout << "Option '" << arg << "' is unknown." << std::endl; + std::cout << "error: option '" << arg << "' is unknown." << std::endl; error = true; } + } else if (filename) { + std::cout << "error: multiple filenames specified" << std::endl; + std::exit(1); } else { filename = arg; } @@ -83,6 +99,11 @@ int main(int argc, char **argv) if (error) std::exit(1); + if (quiet && error_only) { + std::cout << "error: -e cannot be used in conjunction with -q" << std::endl; + std::exit(1); + } + if (!filename) { std::cout << "Syntax:" << std::endl; std::cout << "simplecpp [options] filename" << std::endl; @@ -93,6 +114,7 @@ int main(int argc, char **argv) std::cout << " -std=STD Specify standard." << std::endl; std::cout << " -q Quiet mode (no output)." << std::endl; std::cout << " -is Use std::istream interface." << std::endl; + std::cout << " -e Output errors only." << std::endl; std::exit(0); } @@ -102,6 +124,10 @@ int main(int argc, char **argv) simplecpp::TokenList *rawtokens; if (use_istream) { std::ifstream f(filename); + if (!f.is_open()) { + std::cout << "error: could not open file '" << filename << "'" << std::endl; + std::exit(1); + } rawtokens = new simplecpp::TokenList(f, files,filename,&outputList); } else { @@ -118,7 +144,8 @@ int main(int argc, char **argv) // Output if (!quiet) { - std::cout << outputTokens.stringify() << std::endl; + if (!error_only) + std::cout << outputTokens.stringify() << std::endl; for (const simplecpp::Output &output : outputList) { std::cerr << output.location.file() << ':' << output.location.line << ": "; diff --git a/selfcheck.sh b/selfcheck.sh new file mode 100755 index 00000000..3518c654 --- /dev/null +++ b/selfcheck.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +output=$(./simplecpp simplecpp.cpp -e 2>&1) +ec=$? +echo "$output" | grep -v 'Header not found: <' +exit $ec \ No newline at end of file From a4964e8a95174b78e46960e09a39e94bfad69420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 12 Mar 2023 14:17:10 +0100 Subject: [PATCH 188/381] updated CI to Clang 16 (#291) --- .clang-tidy | 2 +- .github/workflows/clang-tidy.yml | 8 ++++---- CMakeLists.txt | 11 +++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 1005f909..ad644244 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-core.NullDereference,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move,-modernize-use-override' +Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-*,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move,-modernize-use-override,-misc-use-anonymous-namespace,-modernize-use-nodiscard' HeaderFilterRegex: '.*' WarningsAsErrors: '*' CheckOptions: diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 1202f5df..84a7c338 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,15 +21,15 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 15 - sudo apt-get install clang-tidy-15 + sudo ./llvm.sh 16 + sudo apt-get install clang-tidy-16 - name: Prepare CMake run: | cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-15 + CXX: clang-16 - name: Clang-Tidy run: | - run-clang-tidy-15 -q -j $(nproc) -p=cmake.output + run-clang-tidy-16 -q -j $(nproc) -p=cmake.output diff --git a/CMakeLists.txt b/CMakeLists.txt index e88afe5b..8341f6b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,16 @@ project (simplecpp LANGUAGES CXX) option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF) +include(CheckCXXCompilerFlag) + +function(add_compile_options_safe FLAG) + string(MAKE_C_IDENTIFIER "HAS_CXX_FLAG${FLAG}" mangled_flag) + check_cxx_compiler_flag(${FLAG} ${mangled_flag}) + if (${mangled_flag}) + add_compile_options(${FLAG}) + endif() +endfunction() + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast -Wno-multichar) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -11,6 +21,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-c++98-compat-pedantic) # these are not really fixable add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors -Wno-weak-vtables) + add_compile_options_safe(-Wno-unsafe-buffer-usage) # we are not interested in these add_compile_options(-Wno-multichar -Wno-four-char-constants) # ignore C++11-specific warning From dd3d09751c9e87eb5a1406d81db121e9aeec8b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 13 Mar 2023 16:09:53 +0100 Subject: [PATCH 189/381] Fix hashhash bug when there is null statement on the next line --- simplecpp.cpp | 2 +- test.cpp | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2adfa898..4f39b087 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2132,7 +2132,7 @@ namespace simplecpp { for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; output->takeTokens(tokensB); - } else if (nextTok->op == '#' && nextTok->next->op == '#') { + } else if (sameline(B, nextTok) && sameline(B, nextTok->next) && nextTok->op == '#' && nextTok->next->op == '#') { TokenList output2(files); output2.push_back(new Token(strAB, tok->location)); nextTok = expandHashHash(&output2, loc, nextTok, macros, expandedmacros, parametertokens); diff --git a/test.cpp b/test.cpp index b0ba751a..e73ed456 100644 --- a/test.cpp +++ b/test.cpp @@ -1207,6 +1207,18 @@ static void hashhash_invalid_missing_args() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'BAD', Invalid ## usage when expanding 'BAD': Missing first argument\n", toString(outputList)); } +static void hashhash_null_stmt() +{ + const char code[] = + "# define B(x) C ## x\n" + "#\n" + "# define C0 1\n" + "\n" + "B(0);\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n\n\n\n1 ;", preprocess(code, &outputList)); +} + static void hashhash_universal_character() { const char code[] = @@ -2636,6 +2648,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash_invalid_2); TEST_CASE(hashhash_invalid_string_number); TEST_CASE(hashhash_invalid_missing_args); + TEST_CASE(hashhash_null_stmt); // C standard, 5.1.1.2, paragraph 4: // If a character sequence that matches the syntax of a universal // character name is produced by token concatenation (6.10.3.3), From 2828bb1e2e0956b21520ab5e6d2db14ed816230c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 27 Mar 2023 17:51:21 +0200 Subject: [PATCH 190/381] added asserts to streams to make sure the input is valid (#294) --- simplecpp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4f39b087..d757c574 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -23,6 +23,7 @@ #include "simplecpp.h" #include +#include #include #include #include @@ -351,6 +352,7 @@ class StdIStream : public simplecpp::TokenList::Stream { StdIStream(std::istream &istr) : istr(istr) { + assert(istr.good()); init(); } @@ -378,6 +380,7 @@ class FileStream : public simplecpp::TokenList::Stream { , lastCh(0) , lastStatus(0) { + assert(file != nullptr); init(); } From 179bf9f135aedba414aaac8fea5e6b30cf710160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 5 Apr 2023 21:13:27 +0200 Subject: [PATCH 191/381] Do not use TRUE symbol name, that is a macro in some systems --- simplecpp.cpp | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d757c574..96e1295b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -44,7 +44,6 @@ #ifdef SIMPLECPP_WINDOWS #include #undef ERROR -#undef TRUE #endif #if (__cplusplus < 201103L) && !defined(__APPLE__) @@ -3212,12 +3211,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } - // TRUE => code in current #if block should be kept - // ELSE_IS_TRUE => code in current #if block should be dropped. the code in the #else should be kept. - // ALWAYS_FALSE => drop all code in #if and #else - enum IfState { TRUE, ELSE_IS_TRUE, ALWAYS_FALSE }; + // True => code in current #if block should be kept + // ElseIsTrue => code in current #if block should be dropped. the code in the #else should be kept. + // AlwaysFalse => drop all code in #if and #else + enum IfState { True, ElseIsTrue, AlwaysFalse }; std::stack ifstates; - ifstates.push(TRUE); + ifstates.push(True); std::stack includetokenstack; @@ -3262,7 +3261,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL return; } - if (ifstates.top() == TRUE && (rawtok->str() == ERROR || rawtok->str() == WARNING)) { + if (ifstates.top() == True && (rawtok->str() == ERROR || rawtok->str() == WARNING)) { if (outputList) { simplecpp::Output err(rawtok->location.files); err.type = rawtok->str() == ERROR ? Output::ERROR : Output::WARNING; @@ -3282,7 +3281,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } if (rawtok->str() == DEFINE) { - if (ifstates.top() != TRUE) + if (ifstates.top() != True) continue; try { const Macro ¯o = Macro(rawtok->previous, files); @@ -3304,7 +3303,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL output.clear(); return; } - } else if (ifstates.top() == TRUE && rawtok->str() == INCLUDE) { + } else if (ifstates.top() == True && rawtok->str() == INCLUDE) { TokenList inc1(files); for (const Token *inctok = rawtok->next; sameline(rawtok,inctok); inctok = inctok->next) { if (!inctok->comment) @@ -3395,7 +3394,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } bool conditionIsTrue; - if (ifstates.top() == ALWAYS_FALSE || (ifstates.top() == ELSE_IS_TRUE && rawtok->str() != ELIF)) + if (ifstates.top() == AlwaysFalse || (ifstates.top() == ElseIsTrue && rawtok->str() != ELIF)) conditionIsTrue = false; else if (rawtok->str() == IFDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end() || (hasInclude && rawtok->next->str() == HAS_INCLUDE)); @@ -3523,35 +3522,35 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (rawtok->str() != ELIF) { // push a new ifstate.. - if (ifstates.top() != TRUE) - ifstates.push(ALWAYS_FALSE); + if (ifstates.top() != True) + ifstates.push(AlwaysFalse); else - ifstates.push(conditionIsTrue ? TRUE : ELSE_IS_TRUE); - } else if (ifstates.top() == TRUE) { - ifstates.top() = ALWAYS_FALSE; - } else if (ifstates.top() == ELSE_IS_TRUE && conditionIsTrue) { - ifstates.top() = TRUE; + ifstates.push(conditionIsTrue ? True : ElseIsTrue); + } else if (ifstates.top() == True) { + ifstates.top() = AlwaysFalse; + } else if (ifstates.top() == ElseIsTrue && conditionIsTrue) { + ifstates.top() = True; } } else if (rawtok->str() == ELSE) { - ifstates.top() = (ifstates.top() == ELSE_IS_TRUE) ? TRUE : ALWAYS_FALSE; + ifstates.top() = (ifstates.top() == ElseIsTrue) ? True : AlwaysFalse; } else if (rawtok->str() == ENDIF) { ifstates.pop(); } else if (rawtok->str() == UNDEF) { - if (ifstates.top() == TRUE) { + if (ifstates.top() == True) { const Token *tok = rawtok->next; while (sameline(rawtok,tok) && tok->comment) tok = tok->next; if (sameline(rawtok, tok)) macros.erase(tok->str()); } - } else if (ifstates.top() == TRUE && rawtok->str() == PRAGMA && rawtok->next && rawtok->next->str() == ONCE && sameline(rawtok,rawtok->next)) { + } else if (ifstates.top() == True && rawtok->str() == PRAGMA && rawtok->next && rawtok->next->str() == ONCE && sameline(rawtok,rawtok->next)) { pragmaOnce.insert(rawtok->location.file()); } rawtok = gotoNextLine(rawtok); continue; } - if (ifstates.top() != TRUE) { + if (ifstates.top() != True) { // drop code rawtok = gotoNextLine(rawtok); continue; From 8cc8e683977b9bddef9ddca3c04f5aad0a07bb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 18 Apr 2023 20:10:42 +0200 Subject: [PATCH 192/381] fix special case when comment before hash was not ignored --- simplecpp.cpp | 4 ++-- test.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 96e1295b..3ef91d59 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1393,7 +1393,7 @@ namespace simplecpp { explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), valueDefinedInCode_(false) {} Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { - if (sameline(tok->previous, tok)) + if (sameline(tok->previousSkipComments(), tok)) throw std::runtime_error("bad macro syntax"); if (tok->op != '#') throw std::runtime_error("bad macro syntax"); @@ -3238,7 +3238,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } - if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) { + if (rawtok->op == '#' && !sameline(rawtok->previousSkipComments(), rawtok)) { if (!sameline(rawtok, rawtok->next)) { rawtok = rawtok->next; continue; diff --git a/test.cpp b/test.cpp index e73ed456..73bb2794 100644 --- a/test.cpp +++ b/test.cpp @@ -1967,6 +1967,34 @@ static void include8() // #include MACRO(X) ASSERT_EQUALS("file0,3,missing_header,Header not found: <../somewhere/header.h>\n", toString(outputList)); } +static void include9() +{ + const char code_c[] = "#define HDR \"1.h\"\n" + "#include HDR\n"; + const char code_h[] = "/**/ #define X 1\n" // <- comment before hash should be ignored + "x=X;"; + + std::vector files; + + simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "1.c"); + simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "1.h"); + + ASSERT_EQUALS(2U, files.size()); + ASSERT_EQUALS("1.c", files[0]); + ASSERT_EQUALS("1.h", files[1]); + + std::map filedata; + filedata["1.c"] = &rawtokens_c; + filedata["1.h"] = &rawtokens_h; + + simplecpp::TokenList out(files); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + + ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify()); +} + static void readfile_nullbyte() { const char code[] = "ab\0cd"; @@ -2707,6 +2735,7 @@ int main(int argc, char **argv) TEST_CASE(include6); // invalid code: #include MACRO(,) TEST_CASE(include7); // #include MACRO TEST_CASE(include8); // #include MACRO(X) + TEST_CASE(include9); // #include MACRO TEST_CASE(multiline1); TEST_CASE(multiline2); From 5164ec6ed8865d973c9ce51ebefeb1934f0ea6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 2 Jun 2023 15:58:50 +0200 Subject: [PATCH 193/381] msvc compliant handling of ',__VA_ARGS__)' when __VA_ARGS__ is empty --- simplecpp.cpp | 6 +++++- test.cpp | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3ef91d59..77c80fa5 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1925,8 +1925,12 @@ namespace simplecpp { // Macro parameter.. { TokenList temp(files); - if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) + if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) { + if (tok->str() == "__VA_ARGS__" && temp.empty() && output->cback() && output->cback()->str() == "," && + tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") + output->deleteToken(output->back()); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); + } } // Macro.. diff --git a/test.cpp b/test.cpp index 73bb2794..eb18246a 100644 --- a/test.cpp +++ b/test.cpp @@ -765,6 +765,13 @@ static void define_va_args_3() // min number of arguments ASSERT_EQUALS("\n1", preprocess(code)); } +static void define_va_args_4() // cppcheck trac #9754 +{ + const char code[] = "#define A(x, y, ...) printf(x, y, __VA_ARGS__)\n" + "A(1, 2)\n"; + ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); +} + static void define_ifdef() { const char code[] = "#define A(X) X\n" @@ -2633,6 +2640,7 @@ int main(int argc, char **argv) TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); + TEST_CASE(define_va_args_4); // UB: #ifdef as macro parameter TEST_CASE(define_ifdef); From a60f13334087a9b3e474295510396fe6ec80dc54 Mon Sep 17 00:00:00 2001 From: Edo Date: Tue, 27 Jun 2023 14:29:53 +0200 Subject: [PATCH 194/381] suppress msvc warnings about totally safe functions (#298) --- CMakeLists.txt | 2 ++ simplecpp.h | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8341f6b6..5954ac87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,8 @@ endfunction() if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast -Wno-multichar) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Weverything) # no need for c++98 compatibility diff --git a/simplecpp.h b/simplecpp.h index 5b918a98..1857908d 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -44,6 +44,12 @@ #define nullptr NULL #endif +#if defined(_MSC_VER) +// suppress warnings about "conversion from 'type1' to 'type2', possible loss of data" +# pragma warning(disable : 4267) +# pragma warning(disable : 4244) +#endif + namespace simplecpp { typedef std::string TokenString; From 62c61a6d5bbc5911c5cf3f6113be78acaae5088b Mon Sep 17 00:00:00 2001 From: Anton Lindqvist Date: Mon, 7 Aug 2023 20:31:40 +0200 Subject: [PATCH 195/381] Do not confuse defines and designated initializers as float literals (#299) Should hopefully also fix #297. --- simplecpp.cpp | 2 +- test.cpp | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 77c80fa5..5444238c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -926,7 +926,7 @@ void simplecpp::TokenList::combineOperators() continue; } // float literals.. - if (tok->previous && tok->previous->number) { + if (tok->previous && tok->previous->number && sameline(tok->previous, tok)) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); if (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp"))) { diff --git a/test.cpp b/test.cpp index eb18246a..4d38db37 100644 --- a/test.cpp +++ b/test.cpp @@ -568,6 +568,23 @@ static void define11() // location of expanded argument ASSERT_EQUALS("\n#line 10 \"cppcheck.cpp\"\n1 ;", preprocess(code)); } +static void define12() +{ + const char code[] = "struct foo x = {\n" + " #define V 0\n" + " .x = V,\n" + "};\n"; + ASSERT_EQUALS("struct foo x = {\n" + "# define V 0\n" + ". x = V ,\n" + "} ;", readfile(code)); + ASSERT_EQUALS("struct foo x = {\n" + "\n" + ". x = 0 ,\n" + "} ;", preprocess(code)); +} + + static void define_invalid_1() { @@ -2617,6 +2634,7 @@ int main(int argc, char **argv) TEST_CASE(define9); TEST_CASE(define10); TEST_CASE(define11); + TEST_CASE(define12); TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_define_1); From 03a3613cfd6474a0d3d796cc956518d1de30f736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 18 Aug 2023 11:53:02 +0200 Subject: [PATCH 196/381] cleaned up includes based on `include-what-you-use` (#301) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5444238c..2a91c3b3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include From f5a567908a2782dcab8004cc008e1c13a5705099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 22 Aug 2023 14:51:19 +0200 Subject: [PATCH 197/381] added preliminary C++26 `-std=` support and improved comments (#303) --- simplecpp.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2a91c3b3..ced7ab93 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3631,7 +3631,8 @@ std::string simplecpp::getCStdString(const std::string &std) if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") return "201710L"; if (std == "c2x" || std == "gnu2x") { - // Clang 11, 12, 13 return "201710L" - correct in 14 + // supported by GCC 9+ and Clang 9+ + // Clang 9, 10, 11, 12, 13 return "201710L" return "202000L"; } return ""; @@ -3653,8 +3654,14 @@ std::string simplecpp::getCppStdString(const std::string &std) } if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") { // supported by GCC 11+ and Clang 12+ - // Clang 12, 13, 14 do not support "c++23" and "gnu++23" - return "202100L"; + // GCC 11, 12, 13 return "202100L" + // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" + // Clang 17, 18 return "202302L" + return "202100L"; // TODO: update value? + } + if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") { + // supported by Clang 17+ + return "202400L"; } return ""; } From ff94646b884502a388c31e87290d9c65706b4511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 23 Aug 2023 11:29:21 +0200 Subject: [PATCH 198/381] simplecpp.h: push and pop the MSVC warning disabling `#pragma` so they don't spill into user code (#306) --- simplecpp.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/simplecpp.h b/simplecpp.h index 1857908d..b6124953 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -45,6 +45,7 @@ #endif #if defined(_MSC_VER) +# pragma warning(push) // suppress warnings about "conversion from 'type1' to 'type2', possible loss of data" # pragma warning(disable : 4267) # pragma warning(disable : 4244) @@ -370,6 +371,10 @@ namespace simplecpp { SIMPLECPP_LIB std::string getCppStdString(const std::string &std); } +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + #if (__cplusplus < 201103L) && !defined(__APPLE__) #undef nullptr #endif From ce5f06b5857f3a2c138ee48dc20c5f70a7667740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 23 Aug 2023 11:34:30 +0200 Subject: [PATCH 199/381] clang-tidy.yml: updated to Clang 17 (#302) --- .clang-tidy | 54 +++++++++++++++++++++++++++++++- .github/workflows/clang-tidy.yml | 12 ++++--- main.cpp | 7 ++++- simplecpp.cpp | 9 ++++++ test.cpp | 11 +++++-- 5 files changed, 85 insertions(+), 8 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index ad644244..518490b0 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,57 @@ --- -Checks: '*,-abseil-*,-altera-*,-android-*,-cert-*,-cppcoreguidelines-*,-fuchsia-*,-google-*,-hicpp-*,-linuxkernel-*,-llvm-*,-llvmlibc-*,-mpi-*,-objc-*,-openmp-*,-zircon-*,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-use-default-member-init,-modernize-use-using,-readability-braces-around-statements,-readability-function-size,-readability-implicit-bool-conversion,-readability-isolate-declaration,-readability-magic-numbers,-readability-simplify-boolean-expr,-readability-uppercase-literal-suffix,-modernize-use-auto,-modernize-use-trailing-return-type,-bugprone-branch-clone,-modernize-pass-by-value,-modernize-loop-convert,-modernize-use-emplace,-modernize-use-equals-default,-performance-noexcept-move-constructor,-modernize-use-equals-delete,-readability-identifier-length,-readability-function-cognitive-complexity,-modernize-return-braced-init-list,-misc-no-recursion,-bugprone-easily-swappable-parameters,-bugprone-narrowing-conversions,-concurrency-mt-unsafe,-modernize-loop-convert,-clang-analyzer-*,-performance-move-constructor-init,-performance-inefficient-string-concatenation,-performance-no-automatic-move,-modernize-use-override,-misc-use-anonymous-namespace,-modernize-use-nodiscard' +Checks: > + *, + -abseil-*, + -altera-*, + -android-*, + -cert-*, + -clang-analyzer-*, + -cppcoreguidelines-*, + -fuchsia-*, + -google-*, + -hicpp-*, + -linuxkernel-*, + -llvm-*, + -llvmlibc-*, + -mpi-*, + -objc-*, + -openmp-*, + -zircon-*, + -bugprone-branch-clone, + -bugprone-easily-swappable-parameters, + -bugprone-narrowing-conversions, + -bugprone-switch-missing-default-case, + -concurrency-mt-unsafe, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -misc-use-anonymous-namespace, + -modernize-avoid-c-arrays, + -modernize-loop-convert, + -modernize-pass-by-value, + -modernize-return-braced-init-list, + -modernize-use-auto, + -modernize-use-emplace, + -modernize-use-equals-default, + -modernize-use-equals-delete, + -modernize-use-default-member-init, + -modernize-use-nodiscard, + -modernize-use-override, + -modernize-use-trailing-return-type, + -modernize-use-using, + -readability-braces-around-statements, + -readability-function-cognitive-complexity, + -readability-function-size, + -readability-implicit-bool-conversion, + -readability-identifier-length, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-simplify-boolean-expr, + -readability-uppercase-literal-suffix, + -performance-avoid-endl, + -performance-inefficient-string-concatenation, + -performance-move-constructor-init, + -performance-no-automatic-move, + -performance-noexcept-move-constructor HeaderFilterRegex: '.*' WarningsAsErrors: '*' CheckOptions: diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 84a7c338..4fb98b0f 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,15 +21,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 - sudo apt-get install clang-tidy-16 + sudo ./llvm.sh 17 + sudo apt-get install clang-tidy-17 + + - name: Verify clang-tidy configuration + run: | + clang-tidy-17 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-16 + CXX: clang-17 - name: Clang-Tidy run: | - run-clang-tidy-16 -q -j $(nproc) -p=cmake.output + run-clang-tidy-17 -q -j $(nproc) -p=cmake.output diff --git a/main.cpp b/main.cpp index 7516085f..da66f59e 100644 --- a/main.cpp +++ b/main.cpp @@ -18,9 +18,14 @@ #include "simplecpp.h" +#include +#include #include #include -#include +#include +#include +#include +#include int main(int argc, char **argv) { diff --git a/simplecpp.cpp b/simplecpp.cpp index ced7ab93..48b8c6f1 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -20,11 +20,14 @@ #define SIMPLECPP_WINDOWS #define NOMINMAX #endif + #include "simplecpp.h" #include #include +#include #include +#include #include #include #include @@ -33,13 +36,18 @@ #include // IWYU pragma: keep #include #include +#include +#include +#include #include // IWYU pragma: keep #include #include +#include #if __cplusplus >= 201103L #include #endif #include +#include #ifdef SIMPLECPP_WINDOWS #include @@ -3132,6 +3140,7 @@ static void getLocaltime(struct tm <ime) time_t t; time(&t); #ifndef _WIN32 + // NOLINTNEXTLINE(misc-include-cleaner) - false positive localtime_r(&t, <ime); #else localtime_s(<ime, &t); diff --git a/test.cpp b/test.cpp index 4d38db37..5c7142c0 100644 --- a/test.cpp +++ b/test.cpp @@ -16,11 +16,18 @@ * License along with this library. If not, see . */ +#include "simplecpp.h" + +#include +#include +#include +#include #include -#include +#include #include +#include +#include #include -#include "simplecpp.h" static int numberOfFailedAssertions = 0; From df40677aa0eca0b21a417643269166e29fd6139e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 11 Sep 2023 19:38:55 +0200 Subject: [PATCH 200/381] some platform-dependent code cleanups (#310) --- simplecpp.cpp | 115 +++++++++++++++++++++++++++++--------------------- test.cpp | 4 ++ 2 files changed, 70 insertions(+), 49 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 48b8c6f1..c3f688bd 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -44,6 +44,9 @@ #include #include #if __cplusplus >= 201103L +#ifdef SIMPLECPP_WINDOWS +#include +#endif #include #endif #include @@ -139,11 +142,6 @@ static unsigned long long stringToULL(const std::string &s) return ret; } -static bool startsWith(const std::string &str, const std::string &s) -{ - return (str.size() >= s.size() && str.compare(0, s.size(), s) == 0); -} - static bool endsWith(const std::string &s, const std::string &e) { return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); @@ -2215,6 +2213,12 @@ namespace simplecpp { namespace simplecpp { +#ifdef __CYGWIN__ + bool startsWith(const std::string &str, const std::string &s) + { + return (str.size() >= s.size() && str.compare(0, s.size(), s) == 0); + } + std::string convertCygwinToWindowsPath(const std::string &cygwinPath) { std::string windowsPath; @@ -2244,67 +2248,86 @@ namespace simplecpp { return windowsPath; } +#endif } #ifdef SIMPLECPP_WINDOWS -class ScopedLock { +#if __cplusplus >= 201103L +using MyMutex = std::mutex; +template +using MyLock = std::lock_guard; +#else +class MyMutex { public: - explicit ScopedLock(CRITICAL_SECTION& criticalSection) - : m_criticalSection(criticalSection) { - EnterCriticalSection(&m_criticalSection); + MyMutex() { + InitializeCriticalSection(&m_criticalSection); } - ~ScopedLock() { - LeaveCriticalSection(&m_criticalSection); + ~MyMutex() { + DeleteCriticalSection(&m_criticalSection); } + CRITICAL_SECTION* lock() { + return &m_criticalSection; + } private: - ScopedLock& operator=(const ScopedLock&); - ScopedLock(const ScopedLock&); - - CRITICAL_SECTION& m_criticalSection; + CRITICAL_SECTION m_criticalSection; }; -class RealFileNameMap { +template +class MyLock { public: - RealFileNameMap() { - InitializeCriticalSection(&m_criticalSection); + explicit MyLock(T& m) + : m_mutex(m) { + EnterCriticalSection(m_mutex.lock()); } - ~RealFileNameMap() { - DeleteCriticalSection(&m_criticalSection); + ~MyLock() { + LeaveCriticalSection(m_mutex.lock()); } - bool getCacheEntry(const std::string& path, std::string* returnPath) { - ScopedLock lock(m_criticalSection); +private: + MyLock& operator=(const MyLock&); + MyLock(const MyLock&); + + T& m_mutex; +}; +#endif + +class RealFileNameMap { +public: + RealFileNameMap() {} + + bool getCacheEntry(const std::string& path, std::string& returnPath) { + MyLock lock(m_mutex); - std::map::iterator it = m_fileMap.find(path); + const std::map::iterator it = m_fileMap.find(path); if (it != m_fileMap.end()) { - *returnPath = it->second; + returnPath = it->second; return true; } return false; } void addToCache(const std::string& path, const std::string& actualPath) { - ScopedLock lock(m_criticalSection); + MyLock lock(m_mutex); m_fileMap[path] = actualPath; } private: std::map m_fileMap; - CRITICAL_SECTION m_criticalSection; + MyMutex m_mutex; }; static RealFileNameMap realFileNameMap; -static bool realFileName(const std::string &f, std::string *result) +static bool realFileName(const std::string &f, std::string &result) { // are there alpha characters in last subpath? bool alpha = false; for (std::string::size_type pos = 1; pos <= f.size(); ++pos) { - unsigned char c = f[f.size() - pos]; + const unsigned char c = f[f.size() - pos]; if (c == '/' || c == '\\') break; if (std::isalpha(c)) { @@ -2323,16 +2346,16 @@ static bool realFileName(const std::string &f, std::string *result) WIN32_FIND_DATAA FindFileData; #ifdef __CYGWIN__ - std::string fConverted = simplecpp::convertCygwinToWindowsPath(f); - HANDLE hFind = FindFirstFileExA(fConverted.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); + const std::string fConverted = simplecpp::convertCygwinToWindowsPath(f); + const HANDLE hFind = FindFirstFileExA(fConverted.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); #else HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); #endif if (INVALID_HANDLE_VALUE == hFind) return false; - *result = FindFileData.cFileName; - realFileNameMap.addToCache(f, *result); + result = FindFileData.cFileName; + realFileNameMap.addToCache(f, result); FindClose(hFind); } return true; @@ -2345,14 +2368,14 @@ static std::string realFilename(const std::string &f) { std::string ret; ret.reserve(f.size()); // this will be the final size - if (realFilePathMap.getCacheEntry(f, &ret)) + if (realFilePathMap.getCacheEntry(f, ret)) return ret; // Current subpath std::string subpath; for (std::string::size_type pos = 0; pos < f.size(); ++pos) { - unsigned char c = f[pos]; + const unsigned char c = f[pos]; // Separator.. add subpath and separator if (c == '/' || c == '\\') { @@ -2362,12 +2385,12 @@ static std::string realFilename(const std::string &f) continue; } - bool isDriveSpecification = + const bool isDriveSpecification = (pos == 2 && subpath.size() == 2 && std::isalpha(subpath[0]) && subpath[1] == ':'); // Append real filename (proper case) std::string f2; - if (!isDriveSpecification && realFileName(f.substr(0, pos), &f2)) + if (!isDriveSpecification && realFileName(f.substr(0, pos), f2)) ret += f2; else ret += subpath; @@ -2383,7 +2406,7 @@ static std::string realFilename(const std::string &f) if (!subpath.empty()) { std::string f2; - if (realFileName(f,&f2)) + if (realFileName(f,f2)) ret += f2; else ret += subpath; @@ -2902,32 +2925,26 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) class NonExistingFilesCache { public: - NonExistingFilesCache() { - InitializeCriticalSection(&m_criticalSection); - } - - ~NonExistingFilesCache() { - DeleteCriticalSection(&m_criticalSection); - } + NonExistingFilesCache() {} bool contains(const std::string& path) { - ScopedLock lock(m_criticalSection); + MyLock lock(m_mutex); return (m_pathSet.find(path) != m_pathSet.end()); } void add(const std::string& path) { - ScopedLock lock(m_criticalSection); + MyLock lock(m_mutex); m_pathSet.insert(path); } void clear() { - ScopedLock lock(m_criticalSection); + MyLock lock(m_mutex); m_pathSet.clear(); } private: std::set m_pathSet; - CRITICAL_SECTION m_criticalSection; + MyMutex m_mutex; }; static NonExistingFilesCache nonExistingFilesCache; @@ -3032,7 +3049,7 @@ std::map simplecpp::load(const simplecpp::To { #ifdef SIMPLECPP_WINDOWS if (dui.clearIncludeCache) - nonExistingFilesCache .clear(); + nonExistingFilesCache.clear(); #endif std::map ret; diff --git a/test.cpp b/test.cpp index 5c7142c0..368c9c31 100644 --- a/test.cpp +++ b/test.cpp @@ -455,6 +455,7 @@ static void constFold() ASSERT_EQUALS("exception", testConstFold("?2:3")); } +#ifdef __CYGWIN__ static void convertCygwinPath() { // absolute paths @@ -472,6 +473,7 @@ static void convertCygwinPath() ASSERT_EQUALS("\\cygdrive", simplecpp::convertCygwinToWindowsPath("/cygdrive")); ASSERT_EQUALS("\\cygdrive\\", simplecpp::convertCygwinToWindowsPath("/cygdrive/")); } +#endif static void define1() { @@ -2628,7 +2630,9 @@ int main(int argc, char **argv) TEST_CASE(constFold); +#ifdef __CYGWIN__ TEST_CASE(convertCygwinPath); +#endif TEST_CASE(define1); TEST_CASE(define2); From adeff2a1d57b348990c8bcd387f28154d58e5caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 11 Sep 2023 20:21:00 +0200 Subject: [PATCH 201/381] .clang-tidy: enabled `performance-move-constructor-init` (#311) --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 518490b0..376c3848 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -49,7 +49,6 @@ Checks: > -readability-uppercase-literal-suffix, -performance-avoid-endl, -performance-inefficient-string-concatenation, - -performance-move-constructor-init, -performance-no-automatic-move, -performance-noexcept-move-constructor HeaderFilterRegex: '.*' From 62a8c7b16fa6fcbe17c8e1a24bffb53bf87a0f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 11 Sep 2023 20:23:27 +0200 Subject: [PATCH 202/381] fixed Cppcheck warnings (#315) * fixed `missingOverride` Cppcheck warnings * fixed `operatorEqVarError` Cppcheck warning * fixed `noCopyConstructor` Cppcheck warning * fixed `noOperatorEq` Cppcheck warning * fixed `noExplicitConstructor` Cppcheck warnings * suppressed `noConstructor` Cppcheck warning * fixed `constParameterPointer` Cppcheck warnings * suppressed `selfAssignment` Cppcheck warnings * suppressed `duplicateExpressionTernary` Cppcheck warning * suppressed `uninitDerivedMemberVar` Cppcheck warnings * fixed `-Winconsistent-missing-destructor-override` Clang warning --- simplecpp.cpp | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c3f688bd..7c4a6c4d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -57,6 +57,14 @@ #undef ERROR #endif +#if __cplusplus >= 201103L +#define OVERRIDE override +#define EXPLICIT explicit +#else +#define OVERRIDE +#define EXPLICIT +#endif + #if (__cplusplus < 201103L) && !defined(__APPLE__) #define nullptr NULL #endif @@ -236,6 +244,7 @@ void simplecpp::Token::printOut() const std::cout << std::endl; } +// cppcheck-suppress noConstructor - we call init() in the inherited to initialize the private members class simplecpp::TokenList::Stream { public: virtual ~Stream() {} @@ -354,23 +363,24 @@ class simplecpp::TokenList::Stream { class StdIStream : public simplecpp::TokenList::Stream { public: - StdIStream(std::istream &istr) + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + EXPLICIT StdIStream(std::istream &istr) : istr(istr) { assert(istr.good()); init(); } - virtual int get() { + virtual int get() OVERRIDE { return istr.get(); } - virtual int peek() { + virtual int peek() OVERRIDE { return istr.peek(); } - virtual void unget() { + virtual void unget() OVERRIDE { istr.unget(); } - virtual bool good() { + virtual bool good() OVERRIDE { return istr.good(); } @@ -380,7 +390,8 @@ class StdIStream : public simplecpp::TokenList::Stream { class FileStream : public simplecpp::TokenList::Stream { public: - FileStream(const std::string &filename) + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + EXPLICIT FileStream(const std::string &filename) : file(fopen(filename.c_str(), "rb")) , lastCh(0) , lastStatus(0) @@ -389,25 +400,25 @@ class FileStream : public simplecpp::TokenList::Stream { init(); } - ~FileStream() { + ~FileStream() OVERRIDE { fclose(file); file = nullptr; } - virtual int get() { + virtual int get() OVERRIDE { lastStatus = lastCh = fgetc(file); return lastCh; } - virtual int peek() { + virtual int peek() OVERRIDE{ // keep lastCh intact const int ch = fgetc(file); unget_internal(ch); return ch; } - virtual void unget() { + virtual void unget() OVERRIDE { unget_internal(lastCh); } - virtual bool good() { + virtual bool good() OVERRIDE { return lastStatus != EOF; } @@ -422,6 +433,9 @@ class FileStream : public simplecpp::TokenList::Stream { ungetc(ch, file); } + FileStream(const FileStream&); + FileStream &operator=(const FileStream&); + FILE *file; int lastCh; int lastStatus; @@ -1437,6 +1451,7 @@ namespace simplecpp { tokenListDefine = other.tokenListDefine; parseDefine(tokenListDefine.cfront()); } + usageList = other.usageList; } return *this; } @@ -2502,6 +2517,7 @@ namespace simplecpp { if (unc) path = '/' + path; + // cppcheck-suppress duplicateExpressionTernary - platform-dependent implementation return strpbrk(path.c_str(), "*?") == nullptr ? realFilename(path) : path; } } @@ -2595,6 +2611,7 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI for (simplecpp::Token *headerToken = tok1->next; headerToken != tok3; headerToken = headerToken->next) header += headerToken->str(); + // cppcheck-suppress selfAssignment - platform-dependent implementation header = realFilename(header); } else { @@ -3164,14 +3181,14 @@ static void getLocaltime(struct tm <ime) #endif } -static std::string getDateDefine(struct tm *timep) +static std::string getDateDefine(const struct tm *timep) { char buf[] = "??? ?? ????"; strftime(buf, sizeof(buf), "%b %d %Y", timep); return std::string("\"").append(buf).append("\""); } -static std::string getTimeDefine(struct tm *timep) +static std::string getTimeDefine(const struct tm *timep) { char buf[] = "??:??:??"; strftime(buf, sizeof(buf), "%T", timep); @@ -3484,6 +3501,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (systemheader) { while ((tok = tok->next) && tok->op != '>') header += tok->str(); + // cppcheck-suppress selfAssignment - platform-dependent implementation header = realFilename(header); if (tok && tok->op == '>') closingAngularBracket = true; From da449d16ff4ab262ba67847866609f621df771af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 12 Sep 2023 13:10:25 +0200 Subject: [PATCH 203/381] return proper `__cplusplus` value for C++23 / improved preliminary C23 support (#318) --- simplecpp.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7c4a6c4d..e98ecc61 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3674,10 +3674,12 @@ std::string simplecpp::getCStdString(const std::string &std) return "201112L"; if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") return "201710L"; - if (std == "c2x" || std == "gnu2x") { + if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") { // supported by GCC 9+ and Clang 9+ // Clang 9, 10, 11, 12, 13 return "201710L" - return "202000L"; + // Clang 14, 15, 16, 17 return "202000L" + // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" + return "202311L"; } return ""; } @@ -3701,7 +3703,7 @@ std::string simplecpp::getCppStdString(const std::string &std) // GCC 11, 12, 13 return "202100L" // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" // Clang 17, 18 return "202302L" - return "202100L"; // TODO: update value? + return "202302L"; } if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") { // supported by Clang 17+ From 42090bd446da5d90d7135da7e590ee03b1bf086a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 12 Sep 2023 13:11:00 +0200 Subject: [PATCH 204/381] CI-unixish.yml: run CMake build (#317) --- .github/workflows/CI-unixish.yml | 34 +++++++++++++++++++++++--------- .github/workflows/CI-windows.yml | 4 +--- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 3a604900..2acd8d93 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -13,6 +13,9 @@ jobs: runs-on: ${{ matrix.os }} + env: + CXX: ${{ matrix.compiler }} + steps: - uses: actions/checkout@v3 @@ -23,21 +26,34 @@ jobs: sudo apt-get install valgrind - name: make simplecpp - run: make -j$(nproc) CXX=${{ matrix.compiler }} + run: make -j$(nproc) - name: make test - run: make -j$(nproc) test CXX=${{ matrix.compiler }} + run: make -j$(nproc) test - name: selfcheck run: | - make -j$(nproc) selfcheck CXX=${{ matrix.compiler }} + make -j$(nproc) selfcheck + + - name: Run CMake + run: | + cmake -S . -B cmake.output + + - name: CMake simplecpp + run: | + cmake --build cmake.output --target simplecpp -- -j $(nproc) + + - name: CMake testrunner + run: | + cmake --build cmake.output --target testrunner -- -j $(nproc) + ./cmake.output/testrunner - name: Run valgrind if: matrix.os == 'ubuntu-22.04' run: | make clean # this valgrind version doesn't support DWARF 5 yet - make -j$(nproc) CXX=${{ matrix.compiler }} CXXFLAGS="-gdwarf-4" + make -j$(nproc) CXXFLAGS="-gdwarf-4" valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./simplecpp simplecpp.cpp -e @@ -45,19 +61,19 @@ jobs: if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'g++' run: | make clean - make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" + make -j$(nproc) test selfcheck CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" - name: Run with libc++ debug mode if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" + make -j$(nproc) test selfcheck CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" - name: Run AddressSanitizer if: matrix.os == 'ubuntu-22.04' run: | make clean - make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" + make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" env: ASAN_OPTIONS: detect_stack_use_after_return=1 @@ -65,11 +81,11 @@ jobs: if: matrix.os == 'ubuntu-22.04' run: | make clean - make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" # TODO: requires instrumented libc++ - name: Run MemorySanitizer if: false && matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" + make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 1c4e3404..ed88f4bb 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -31,14 +31,12 @@ jobs: if: matrix.os == 'windows-2019' run: | cmake -G "Visual Studio 16 2019" -A x64 . || exit /b !errorlevel! - dir - name: Run cmake if: matrix.os == 'windows-2022' run: | cmake -G "Visual Studio 17 2022" -A x64 . || exit /b !errorlevel! - dir - + - name: Build run: | msbuild -m simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! From b5d1112072c4dec74751fdf4d40a0dba96524770 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:21:51 +0200 Subject: [PATCH 205/381] Fix #314 TokenList::lastLine() is slow for long lines (#319) --- simplecpp.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e98ecc61..1f572fda 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1362,14 +1362,18 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const if (++count > maxsize) return ""; if (!ret.empty()) - ret.insert(0, 1, ' '); + ret += ' '; + // add tokens in reverse for performance reasons if (tok->str()[0] == '\"') - ret.insert(0, "%str%"); + ret += "%rts%"; // %str% else if (tok->number) - ret.insert(0, "%num%"); - else - ret.insert(0, tok->str()); + ret += "%mun%"; // %num% + else { + ret += tok->str(); + std::reverse(ret.end() - tok->str().length(), ret.end()); + } } + std::reverse(ret.begin(), ret.end()); return ret; } From 977220f577c26dbb199bd411c0f1092db39d0880 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Tue, 12 Sep 2023 22:32:32 +0200 Subject: [PATCH 206/381] Limit maxsize to 1000 tokens (#320) --- simplecpp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.h b/simplecpp.h index b6124953..afd7d92c 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -294,7 +294,7 @@ namespace simplecpp { std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); - std::string lastLine(int maxsize=100000) const; + std::string lastLine(int maxsize=1000) const; bool isLastLinePreprocessor(int maxsize=100000) const; unsigned int fileIndex(const std::string &filename); From b9bfbaa4a98f9c2f9d8c41547488b146f0584837 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Tue, 19 Sep 2023 10:43:52 +0200 Subject: [PATCH 207/381] Fix #321 End of double #define not recognized (#322) --- simplecpp.cpp | 2 +- simplecpp.h | 6 +++++- test.cpp | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1f572fda..6bad9e30 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -949,7 +949,7 @@ void simplecpp::TokenList::combineOperators() if (tok->previous && tok->previous->number && sameline(tok->previous, tok)) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); - if (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp"))) { + if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp")))) { tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } diff --git a/simplecpp.h b/simplecpp.h index afd7d92c..18999311 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -120,7 +120,7 @@ namespace simplecpp { name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); - number = std::isdigit(static_cast(string[0])) || (string.size() > 1U && (string[0] == '-' || string[0] == '+') && std::isdigit(static_cast(string[1]))); + number = isNumberLike(string); op = (string.size() == 1U && !name && !comment && !number) ? string[0] : '\0'; } @@ -135,6 +135,10 @@ namespace simplecpp { bool isOneOf(const char ops[]) const; bool startsWithOneOf(const char c[]) const; bool endsWithOneOf(const char c[]) const; + static bool SIMPLECPP_LIB isNumberLike(const std::string& str) { + return std::isdigit(static_cast(str[0])) || + (str.size() > 1U && (str[0] == '-' || str[0] == '+') && std::isdigit(static_cast(str[1]))); + } TokenString macro; char op; diff --git a/test.cpp b/test.cpp index 368c9c31..151bb825 100644 --- a/test.cpp +++ b/test.cpp @@ -593,6 +593,19 @@ static void define12() "} ;", preprocess(code)); } +static void define13() +{ + const char code[] = "#define M 180.\n" + "extern void g();\n" + "void f(double d) {\n" + " if (d > M) {}\n" + "}\n"; + ASSERT_EQUALS("\nextern void g ( ) ;\n" + "void f ( double d ) {\n" + "if ( d > 180. ) { }\n" + "}", preprocess(code)); +} + static void define_invalid_1() @@ -2646,6 +2659,7 @@ int main(int argc, char **argv) TEST_CASE(define10); TEST_CASE(define11); TEST_CASE(define12); + TEST_CASE(define13); TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_define_1); From 2f44b77d9690e0150390520bfa1cbc5f94e5b904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 5 Oct 2023 15:12:27 +0200 Subject: [PATCH 208/381] Fix #292 (Fail to expand macro, comma in inner macro) (#326) --- simplecpp.cpp | 11 +++++++++++ test.cpp | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6bad9e30..80458339 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -114,6 +114,8 @@ static const simplecpp::TokenString ONCE("once"); static const simplecpp::TokenString HAS_INCLUDE("__has_include"); +static const simplecpp::TokenString INNER_COMMA(",,"); + template static std::string toString(T t) { // NOLINTNEXTLINE(misc-const-correctness) - false positive @@ -1559,6 +1561,10 @@ namespace simplecpp { rawtok = rawtok2->next; } output->takeTokens(output2); + for (Token* tok = output->front(); tok; tok = tok->next) { + if (tok->str() == INNER_COMMA) + tok->setstr(","); + } return rawtok; } @@ -1733,7 +1739,12 @@ namespace simplecpp { if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { const Macro &m = it->second; if (!m.functionLike()) { + Token* mtok = tokens->back(); m.expand(tokens, rawloc, tok, macros, expandedmacros); + for (mtok = mtok->next; mtok; mtok = mtok->next) { + if (mtok->op == ',') + mtok->setstr(INNER_COMMA); + } expanded = true; } } diff --git a/test.cpp b/test.cpp index 151bb825..b786987a 100644 --- a/test.cpp +++ b/test.cpp @@ -783,6 +783,15 @@ static void define_define_18() ASSERT_EQUALS("\n\n\n( ( p -> var ) ) ;", preprocess(code)); } +static void define_define_19() // #292 +{ + const char code[] = "#define X 1,2,3\n" + "#define Foo(A, B) A\n" + "#define Bar Foo(X, 0)\n" + "Bar\n"; + ASSERT_EQUALS("\n\n\n1 , 2 , 3", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -1294,7 +1303,7 @@ static void has_include_1() static void has_include_2() { const char code[] = "#if defined( __has_include)\n" - " #if /*commant*/ __has_include /*comment*/(\"simplecpp.h\") // comment\n" + " #if /*comment*/ __has_include /*comment*/(\"simplecpp.h\") // comment\n" " A\n" " #else\n" " B\n" @@ -2680,6 +2689,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_16); TEST_CASE(define_define_17); TEST_CASE(define_define_18); + TEST_CASE(define_define_19); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From dbae338e26e5e198a6d1fa098f3d264c2109d737 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Thu, 5 Oct 2023 18:16:52 +0200 Subject: [PATCH 209/381] Remove redundant declaration (#323) --- simplecpp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.h b/simplecpp.h index 18999311..55a78b45 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -135,7 +135,7 @@ namespace simplecpp { bool isOneOf(const char ops[]) const; bool startsWithOneOf(const char c[]) const; bool endsWithOneOf(const char c[]) const; - static bool SIMPLECPP_LIB isNumberLike(const std::string& str) { + static bool isNumberLike(const std::string& str) { return std::isdigit(static_cast(str[0])) || (str.size() > 1U && (str[0] == '-' || str[0] == '+') && std::isdigit(static_cast(str[1]))); } From edb7b86837f553f32929eddccb3ade6a4060d43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 8 Oct 2023 08:45:05 +0200 Subject: [PATCH 210/381] avoid some redundant checks in `TokenList::readfile()` (#324) --- simplecpp.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 80458339..881b7c7f 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -627,8 +627,6 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, unsigned char ch = stream.readChar(); if (!stream.good()) break; - if (ch < ' ' && ch != '\t' && ch != '\n' && ch != '\r') - ch = ' '; if (ch >= 0x80) { if (outputList) { @@ -694,7 +692,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, continue; } - if (std::isspace(ch)) { + if (ch <= ' ') { location.col++; continue; } From e941a2ec85b3761c1593d97823741594ad4feba1 Mon Sep 17 00:00:00 2001 From: John Siegel <36650083+JohnSiegel@users.noreply.github.com> Date: Mon, 6 Nov 2023 14:54:22 -0500 Subject: [PATCH 211/381] Added support for `__VA_OPT__` expansion (#329) --- simplecpp.cpp | 18 +++++++++++ test.cpp | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 881b7c7f..672ccf21 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1959,6 +1959,24 @@ namespace simplecpp { // Macro parameter.. { TokenList temp(files); + if (tok->str() == "__VA_OPT__") { + if (sameline(tok, tok->next) && tok->next->str() == "(") { + tok = tok->next; + int paren = 1; + while (sameline(tok, tok->next)) { + if (tok->next->str() == "(") + ++paren; + else if (tok->next->str() == ")") + --paren; + if (paren == 0) + return tok->next->next; + tok = tok->next; + if (parametertokens.front()->next->str() != ")" && parametertokens.size() > args.size()) + tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens)->previous; + } + } + throw Error(tok->location, "Missing parenthesis for __VA_OPT__(content)"); + } if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) { if (tok->str() == "__VA_ARGS__" && temp.empty() && output->cback() && output->cback()->str() == "," && tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") diff --git a/test.cpp b/test.cpp index b786987a..3120b574 100644 --- a/test.cpp +++ b/test.cpp @@ -820,6 +820,88 @@ static void define_va_args_4() // cppcheck trac #9754 ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); } +static void define_va_opt_1() +{ + const char code[] = "#define p1(fmt, args...) printf(fmt __VA_OPT__(,) args)\n" + "p1(\"hello\");\n" + "p1(\"%s\", \"hello\");\n"; + + ASSERT_EQUALS("\nprintf ( \"hello\" ) ;\n" + "printf ( \"%s\" , \"hello\" ) ;", + preprocess(code)); +} + +static void define_va_opt_2() +{ + const char code[] = "#define err(...)\\\n" + "__VA_OPT__(\\\n" + "printf(__VA_ARGS__);\\\n" + ")\n" + "#define err2(something, ...) __VA_OPT__(err(__VA_ARGS__))\n" + "err2(test)\n" + "err2(test, \"%d\", 2)\n"; + + ASSERT_EQUALS("\n\n\n\n\n\nprintf ( \"%d\" , 2 ) ;", preprocess(code)); +} + +static void define_va_opt_3() +{ + // non-escaped newline without closing parenthesis + const char code1[] = "#define err(...) __VA_OPT__(printf( __VA_ARGS__);\n" + ")\n" + "err()"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code1, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + toString(outputList)); + + outputList.clear(); + + // non-escaped newline without open parenthesis + const char code2[] = "#define err(...) __VA_OPT__\n" + "(something)\n" + "err()"; + + ASSERT_EQUALS("", preprocess(code2, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + toString(outputList)); +} + +static void define_va_opt_4() +{ + // missing parenthesis + const char code1[] = "#define err(...) __VA_OPT__ something\n" + "err()"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code1, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + toString(outputList)); + + outputList.clear(); + + // missing open parenthesis + const char code2[] = "#define err(...) __VA_OPT__ something)\n" + "err()"; + + ASSERT_EQUALS("", preprocess(code2, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + toString(outputList)); +} + +static void define_va_opt_5() +{ + // parenthesis not directly proceeding __VA_OPT__ + const char code[] = "#define err(...) __VA_OPT__ something (something)\n" + "err()"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + toString(outputList)); +} + static void define_ifdef() { const char code[] = "#define A(X) X\n" @@ -2694,6 +2776,11 @@ int main(int argc, char **argv) TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); TEST_CASE(define_va_args_4); + TEST_CASE(define_va_opt_1); + TEST_CASE(define_va_opt_2); + TEST_CASE(define_va_opt_3); + TEST_CASE(define_va_opt_4); + TEST_CASE(define_va_opt_5); // UB: #ifdef as macro parameter TEST_CASE(define_ifdef); From 7876b815eb420a75ce35a8475464a09d9eebdb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 Nov 2023 21:25:30 +0100 Subject: [PATCH 212/381] optimized `lastLine()` usage in `readfile()` (#325) --- simplecpp.cpp | 60 ++++++++++++++++++++++++++++++++------------------- simplecpp.h | 3 ++- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 672ccf21..c61c3ce1 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -699,17 +699,20 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, TokenString currentToken; - if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#' && isLastLinePreprocessor() && (lastLine() == "# error" || lastLine() == "# warning")) { - char prev = ' '; - while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { - currentToken += ch; - prev = ch; - ch = stream.readChar(); + if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#') { + const Token* const llTok = lastLineTok(); + if (llTok && llTok->op == '#' && llTok->next && (llTok->next->str() == "error" || llTok->next->str() == "warning")) { + char prev = ' '; + while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { + currentToken += ch; + prev = ch; + ch = stream.readChar(); + } + stream.ungetChar(); + push_back(new Token(currentToken, location)); + location.adjust(currentToken); + continue; } - stream.ungetChar(); - push_back(new Token(currentToken, location)); - location.adjust(currentToken); - continue; } // number or name @@ -841,12 +844,16 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, else back()->setstr(prefix + s); - if (newlines > 0 && isLastLinePreprocessor() && lastLine().compare(0,9,"# define ") == 0) { - multiline += newlines; - location.adjust(s); - } else { - location.adjust(currentToken); + if (newlines > 0 ) { + const Token * const llTok = lastLineTok(); + if (llTok && llTok->op == '#' && llTok->next && llTok->next->str() == "define" && llTok->next->next) { + multiline += newlines; + location.adjust(s); + continue; + } } + + location.adjust(currentToken); continue; } @@ -854,10 +861,13 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, currentToken += ch; } - if (*currentToken.begin() == '<' && isLastLinePreprocessor() && lastLine() == "# include") { - currentToken = readUntil(stream, location, '<', '>', outputList); - if (currentToken.size() < 2U) - return; + if (*currentToken.begin() == '<') { + const Token * const llTok = lastLineTok(); + if (llTok && llTok->op == '#' && llTok->next && llTok->next->str() == "include") { + currentToken = readUntil(stream, location, '<', '>', outputList); + if (currentToken.size() < 2U) + return; + } } push_back(new Token(currentToken, location)); @@ -1377,7 +1387,7 @@ std::string simplecpp::TokenList::lastLine(int maxsize) const return ret; } -bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const +const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const { const Token* prevTok = nullptr; int count = 0; @@ -1387,10 +1397,16 @@ bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const if (tok->comment) continue; if (++count > maxsize) - return false; + return nullptr; prevTok = tok; } - return prevTok && prevTok->str()[0] == '#'; + return prevTok; +} + +bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const +{ + const Token * const prevTok = lastLineTok(maxsize); + return prevTok && prevTok->op == '#'; } unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) diff --git a/simplecpp.h b/simplecpp.h index 55a78b45..163c8497 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -299,7 +299,8 @@ namespace simplecpp { void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); std::string lastLine(int maxsize=1000) const; - bool isLastLinePreprocessor(int maxsize=100000) const; + const Token* lastLineTok(int maxsize=1000) const; + bool isLastLinePreprocessor(int maxsize=1000) const; unsigned int fileIndex(const std::string &filename); From cfef803a3615322dcf289e791d8123a0e24cae82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 Nov 2023 21:26:09 +0100 Subject: [PATCH 213/381] test.cpp: also check the message of the exception (#313) --- test.cpp | 84 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/test.cpp b/test.cpp index 3120b574..9233c964 100644 --- a/test.cpp +++ b/test.cpp @@ -32,7 +32,7 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) -#define ASSERT_THROW(stmt, e) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e&) {} } while(false) +#define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while(false) static std::string pprint(const std::string &in) { @@ -289,7 +289,7 @@ static void characterLiteral() // END Implementation-specific results #endif - ASSERT_THROW(simplecpp::characterLiteralToLL("'\\9'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'\\9'"), std::runtime_error, "invalid escape sequence"); // Input is manually encoded to (escaped) UTF-8 byte sequences // to avoid dependence on source encoding used for this file @@ -309,18 +309,18 @@ static void characterLiteral() ASSERT_EQUALS(0x157, simplecpp::characterLiteralToLL("u'\305\227'")); ASSERT_EQUALS(0xff0f, simplecpp::characterLiteralToLL("u'\357\274\217'")); ASSERT_EQUALS(0x3042, simplecpp::characterLiteralToLL("u'\343\201\202'")); - ASSERT_THROW(simplecpp::characterLiteralToLL("u'\360\223\200\200'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u'\360\223\200\200'"), std::runtime_error, "code point too large"); - ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\302\265'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\305\227'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\357\274\217'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\343\201\202'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("u8'\360\223\200\200'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\302\265'"), std::runtime_error, "code point too large"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\305\227'"), std::runtime_error, "code point too large"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\357\274\217'"), std::runtime_error, "code point too large"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\343\201\202'"), std::runtime_error, "code point too large"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\360\223\200\200'"), std::runtime_error, "code point too large"); ASSERT_EQUALS('\x89', simplecpp::characterLiteralToLL("'\x89'")); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf4\x90\x80\x80'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf4\x90\x80\x80'"), std::runtime_error, "code point too large"); // following examples based on https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt ASSERT_EQUALS(0x80, simplecpp::characterLiteralToLL("U'\xc2\x80'")); @@ -336,38 +336,38 @@ static void characterLiteral() ASSERT_EQUALS(0xfffd, simplecpp::characterLiteralToLL("U'\xef\xbf\xbd'")); ASSERT_EQUALS(0x10ffff, simplecpp::characterLiteralToLL("U'\xf4\x8f\xbf\xbf'")); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f\x8f'"), std::runtime_error); - - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f\x8f'"), std::runtime_error); - - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0 '"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x8f '"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f '"), std::runtime_error); - - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf8'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xff'"), std::runtime_error); - - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0\xaf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x80\xaf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\xaf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc1\xbf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x9f\xbf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x8f\xbf\xbf'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xc0\x80'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xe0\x80\x80'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\x80'"), std::runtime_error); - - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xed\xa0\x80'"), std::runtime_error); - ASSERT_THROW(simplecpp::characterLiteralToLL("U'\xed\xbf\xbf'"), std::runtime_error); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x80\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x80\x8f\x8f\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xbf\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xbf\x8f\x8f\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xc0'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xc0 '"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xe0\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xe0\x8f '"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf0\x8f\x8f '"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf8'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xff'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xc0\xaf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xe0\x80\xaf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\xaf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xc1\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xe0\x9f\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf0\x8f\xbf\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xc0\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xe0\x80\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xf0\x80\x80\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xed\xa0\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xed\xbf\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); } static void combineOperators_floatliteral() From e096d6b3f15f134d37abaa20f12bfbf84b8dd552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 22 Dec 2023 18:16:20 +0100 Subject: [PATCH 214/381] Fix #173 (Change license to 0BSD) (#332) --- LICENSE | 174 ++++---------------------------------------------- main.cpp | 15 +---- simplecpp.cpp | 15 +---- simplecpp.h | 15 +---- test.cpp | 15 +---- 5 files changed, 15 insertions(+), 219 deletions(-) diff --git a/LICENSE b/LICENSE index 341c30bd..b1f013e9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,166 +1,14 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 +BSD Zero Clause License - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +Copyright (c) 2023 simplecpp team +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. - +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/main.cpp b/main.cpp index da66f59e..9e9c4b1d 100644 --- a/main.cpp +++ b/main.cpp @@ -1,19 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016-2022 Daniel Marjamäki. - * - * This library is free software: you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation, either - * version 3 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see . + * Copyright (C) 2016-2023 simplecpp team */ #include "simplecpp.h" diff --git a/simplecpp.cpp b/simplecpp.cpp index c61c3ce1..e8636aaa 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1,19 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016-2022 Daniel Marjamäki. - * - * This library is free software: you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation, either - * version 3 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see . + * Copyright (C) 2016-2023 simplecpp team */ #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) diff --git a/simplecpp.h b/simplecpp.h index 163c8497..e5745a1a 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -1,19 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016-2022 Daniel Marjamäki. - * - * This library is free software: you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation, either - * version 3 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see . + * Copyright (C) 2016-2023 simplecpp team */ #ifndef simplecppH diff --git a/test.cpp b/test.cpp index 9233c964..2bbf861d 100644 --- a/test.cpp +++ b/test.cpp @@ -1,19 +1,6 @@ /* * simplecpp - A simple and high-fidelity C/C++ preprocessor library - * Copyright (C) 2016-2022 Daniel Marjamäki. - * - * This library is free software: you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation, either - * version 3 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see . + * Copyright (C) 2016-2023 simplecpp team */ #include "simplecpp.h" From ea78f9a0e5f0056b287fbbca031b15bebf9cfcfe Mon Sep 17 00:00:00 2001 From: olabetskyi <153490942+olabetskyi@users.noreply.github.com> Date: Fri, 2 Feb 2024 14:42:11 +0200 Subject: [PATCH 215/381] Fix #334 Do not assert when source file is missing (#333) --- main.cpp | 3 +++ simplecpp.cpp | 16 +++++++++++++--- simplecpp.h | 4 +++- test.cpp | 13 +++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/main.cpp b/main.cpp index 9e9c4b1d..ecbe062f 100644 --- a/main.cpp +++ b/main.cpp @@ -166,6 +166,9 @@ int main(int argc, char **argv) case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: std::cerr << "explicit include not found: "; break; + case simplecpp::Output::FILE_NOT_FOUND: + std::cerr << "file not found: "; + break; } std::cerr << output.msg << std::endl; } diff --git a/simplecpp.cpp b/simplecpp.cpp index e8636aaa..d9fd0087 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -385,7 +385,10 @@ class FileStream : public simplecpp::TokenList::Stream { , lastCh(0) , lastStatus(0) { - assert(file != nullptr); + if (!file) { + const std::vector location; + throw simplecpp::Output(location, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); + } init(); } @@ -442,8 +445,15 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector &fi simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { - FileStream stream(filename); - readfile(stream,filename,outputList); + try + { + FileStream stream(filename); + readfile(stream,filename,outputList); + } + catch(const simplecpp::Output & e) // TODO handle extra type of errors + { + outputList->push_back(e); + } } simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), backToken(nullptr), files(other.files) diff --git a/simplecpp.h b/simplecpp.h index e5745a1a..5ad44a09 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -180,8 +180,10 @@ namespace simplecpp { SYNTAX_ERROR, PORTABILITY_BACKSLASH, UNHANDLED_CHAR_ERROR, - EXPLICIT_INCLUDE_NOT_FOUND + EXPLICIT_INCLUDE_NOT_FOUND, + FILE_NOT_FOUND } type; + explicit Output(const std::vector &files, Output::Type id, const std::string & errMsg ) : type(id), location(files), msg(errMsg) {} Location location; std::string msg; }; diff --git a/test.cpp b/test.cpp index 2bbf861d..9588441c 100644 --- a/test.cpp +++ b/test.cpp @@ -150,6 +150,10 @@ static std::string toString(const simplecpp::OutputList &outputList) break; case simplecpp::Output::Type::EXPLICIT_INCLUDE_NOT_FOUND: ostr << "explicit_include_not_found,"; + break; + case simplecpp::Output::Type::FILE_NOT_FOUND: + ostr << "file_not_found,"; + break; } ostr << output.msg << '\n'; @@ -2266,6 +2270,14 @@ static void readfile_error() "X",readfile("#if !A\n#error\n#endif\nX\n")); } +static void readfile_file_not_found() +{ + simplecpp::OutputList outputList; + std::vector files; + (void)simplecpp::TokenList("NotAFile", files, &outputList); + ASSERT_EQUALS("file0,1,file_not_found,File is missing: NotAFile\n", toString(outputList)); +} + static void stringify1() { const char code_c[] = "#include \"A.h\"\n" @@ -2891,6 +2903,7 @@ int main(int argc, char **argv) TEST_CASE(readfile_cpp14_number); TEST_CASE(readfile_unhandled_chars); TEST_CASE(readfile_error); + TEST_CASE(readfile_file_not_found); TEST_CASE(stringify1); From 1b293697ba80e82c381a3f6096575aa6c4ee73a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 2 Feb 2024 15:11:20 +0100 Subject: [PATCH 216/381] Fix #335: object lifetime issue when reporting error (#336) --- simplecpp.cpp | 8 ++++---- simplecpp.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d9fd0087..d8363267 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -380,14 +380,14 @@ class StdIStream : public simplecpp::TokenList::Stream { class FileStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - EXPLICIT FileStream(const std::string &filename) + EXPLICIT FileStream(const std::string &filename, std::vector &files) : file(fopen(filename.c_str(), "rb")) , lastCh(0) , lastStatus(0) { if (!file) { - const std::vector location; - throw simplecpp::Output(location, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); + files.push_back(filename); + throw simplecpp::Output(files, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); } init(); } @@ -447,7 +447,7 @@ simplecpp::TokenList::TokenList(const std::string &filename, std::vector &files, Output::Type id, const std::string & errMsg ) : type(id), location(files), msg(errMsg) {} + explicit Output(const std::vector& files, Type type, const std::string& msg) : type(type), location(files), msg(msg) {} Location location; std::string msg; }; From 03dee1b6635666f875207fc6010c0588b11af11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 27 Feb 2024 12:35:05 +0100 Subject: [PATCH 217/381] reduced scope of function call in `getFileName()` (#338) --- simplecpp.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d8363267..7dad4672 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3085,9 +3085,11 @@ static std::string getFileName(const std::map::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { std::string s = simplecpp::simplifyPath(getIncludePathFileName(*it, header)); From 6547bf749c6da868348ae621e208915f23639658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 27 Feb 2024 12:36:03 +0100 Subject: [PATCH 218/381] clang-tidy.yml: updated to Clang 18 (#307) * clang-tidy.yml: updated to Clang 18 * .clang-tidy: disabled `performance-enum-size` clang-tidy check * disabled `-Wswitch-default` Clang compiler warning * .clang-tidy: disabled `readability-redundant-inline-specifier` clang-tidy check * .clang-tidy: disabled `readability-avoid-nested-conditional-operator` clang-tidy check for now * CI-unixish.yml: added TODO about updated libc++ hardening mode --- .clang-tidy | 3 +++ .github/workflows/CI-unixish.yml | 1 + .github/workflows/clang-tidy.yml | 10 +++++----- CMakeLists.txt | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 376c3848..752ae48f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -38,6 +38,7 @@ Checks: > -modernize-use-override, -modernize-use-trailing-return-type, -modernize-use-using, + -readability-avoid-nested-conditional-operator, -readability-braces-around-statements, -readability-function-cognitive-complexity, -readability-function-size, @@ -45,9 +46,11 @@ Checks: > -readability-identifier-length, -readability-isolate-declaration, -readability-magic-numbers, + -readability-redundant-inline-specifier, -readability-simplify-boolean-expr, -readability-uppercase-literal-suffix, -performance-avoid-endl, + -performance-enum-size, -performance-inefficient-string-concatenation, -performance-no-automatic-move, -performance-noexcept-move-constructor diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 2acd8d93..ea6ed37b 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -63,6 +63,7 @@ jobs: make clean make -j$(nproc) test selfcheck CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" + # TODO: change it to -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG when the compiler is at least Clang 18 - name: Run with libc++ debug mode if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' run: | diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 4fb98b0f..a57f874b 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,19 +21,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 17 - sudo apt-get install clang-tidy-17 + sudo ./llvm.sh 18 + sudo apt-get install clang-tidy-18 - name: Verify clang-tidy configuration run: | - clang-tidy-17 --verify-config + clang-tidy-18 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-17 + CXX: clang-18 - name: Clang-Tidy run: | - run-clang-tidy-17 -q -j $(nproc) -p=cmake.output + run-clang-tidy-18 -q -j $(nproc) -p=cmake.output diff --git a/CMakeLists.txt b/CMakeLists.txt index 5954ac87..30ee1431 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-multichar -Wno-four-char-constants) # ignore C++11-specific warning add_compile_options(-Wno-suggest-override -Wno-suggest-destructor-override) + # contradicts -Wcovered-switch-default + add_compile_options(-Wno-switch-default) # TODO: fix these? add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32 -Wno-shadow-field-in-constructor) From c5c02ff331c461bfb4f5e8628d7d71bd64ba7f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 27 Feb 2024 15:58:33 +0100 Subject: [PATCH 219/381] fixed fuzzing crash in `simplecpp::Macro::expandToken()` (#345) --- simplecpp.cpp | 2 +- test.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7dad4672..43d8ac4c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1984,7 +1984,7 @@ namespace simplecpp { if (paren == 0) return tok->next->next; tok = tok->next; - if (parametertokens.front()->next->str() != ")" && parametertokens.size() > args.size()) + if (parametertokens.size() > args.size() && parametertokens.front()->next->str() != ")") tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens)->previous; } } diff --git a/test.cpp b/test.cpp index 9588441c..322d1876 100644 --- a/test.cpp +++ b/test.cpp @@ -2714,6 +2714,15 @@ static void token() ASSERT_TOKEN("+22", false, true, false); } +static void fuzz_crash() +{ + { + const char code[] = "#define n __VA_OPT__(u\n" + "n\n"; + (void)preprocess(code, simplecpp::DUI()); // do not crash + } +} + int main(int argc, char **argv) { TEST_CASE(backslash); @@ -2940,5 +2949,7 @@ int main(int argc, char **argv) TEST_CASE(token); + TEST_CASE(fuzz_crash); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From ad9b49d7a613c8da4dbe5c2bf757aba01f90e7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 8 Mar 2024 08:46:03 +0100 Subject: [PATCH 220/381] refs #342 - do not load included files twice in CLI application / added `DUI::removeComment` (#340) --- main.cpp | 13 +++++-------- simplecpp.cpp | 6 ++++++ simplecpp.h | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/main.cpp b/main.cpp index ecbe062f..6b5b0d0e 100644 --- a/main.cpp +++ b/main.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include int main(int argc, char **argv) @@ -110,6 +109,8 @@ int main(int argc, char **argv) std::exit(0); } + dui.removeComments = true; + // Perform preprocessing simplecpp::OutputList outputList; std::vector files; @@ -126,11 +127,10 @@ int main(int argc, char **argv) rawtokens = new simplecpp::TokenList(filename,files,&outputList); } rawtokens->removeComments(); - std::map included = simplecpp::load(*rawtokens, files, dui, &outputList); - for (std::pair i : included) - i.second->removeComments(); simplecpp::TokenList outputTokens(files); - simplecpp::preprocess(outputTokens, *rawtokens, files, included, dui, &outputList); + std::map filedata; + simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList); + simplecpp::cleanup(filedata); delete rawtokens; rawtokens = nullptr; @@ -174,8 +174,5 @@ int main(int argc, char **argv) } } - // cleanup included tokenlists - simplecpp::cleanup(included); - return 0; } diff --git a/simplecpp.cpp b/simplecpp.cpp index 43d8ac4c..1de36a52 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3145,6 +3145,8 @@ std::map simplecpp::load(const simplecpp::To continue; } + if (dui.removeComments) + tokenlist->removeComments(); ret[filename] = tokenlist; filelist.push_back(tokenlist->front()); } @@ -3180,6 +3182,8 @@ std::map simplecpp::load(const simplecpp::To f.close(); TokenList *tokens = new TokenList(header2, filenames, outputList); + if (dui.removeComments) + tokens->removeComments(); ret[header2] = tokens; if (tokens->front()) filelist.push_back(tokens->front()); @@ -3448,6 +3452,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); if (f.is_open()) { TokenList * const tokens = new TokenList(f, files, header2, outputList); + if (dui.removeComments) + tokens->removeComments(); filedata[header2] = tokens; } } diff --git a/simplecpp.h b/simplecpp.h index 7ef0740c..87238378 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -320,13 +320,14 @@ namespace simplecpp { * On the command line these are configured by -D, -U, -I, --include, -std */ struct SIMPLECPP_LIB DUI { - DUI() : clearIncludeCache(false) {} + DUI() : clearIncludeCache(false), removeComments(false) {} std::list defines; std::set undefined; std::list includePaths; std::list includes; std::string std; bool clearIncludeCache; + bool removeComments; /** remove comment tokens from included files */ }; SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); From 25d95cc14e2dc8cf8a5f6b17eb725e4ddab233bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Apr 2024 14:26:05 +0200 Subject: [PATCH 221/381] added accessor for `TokenList::files` (#349) --- simplecpp.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/simplecpp.h b/simplecpp.h index 87238378..c72e4c65 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -272,6 +272,10 @@ namespace simplecpp { /** sizeof(T) */ std::map sizeOfType; + const std::vector& getFiles() const { + return files; + } + private: void combineOperators(); From 19df0b1ae68f872821f7e1502fde84b46586466f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Apr 2024 14:27:21 +0200 Subject: [PATCH 222/381] do not add empty filename to `files` in `preprocess()` (#350) --- simplecpp.cpp | 19 +++++++++++-------- test.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1de36a52..812600bf 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3274,6 +3274,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL sizeOfType.insert(std::make_pair("double *", sizeof(double *))); sizeOfType.insert(std::make_pair("long double *", sizeof(long double *))); + // use a dummy vector for the macros because as this is not part of the file and would add an empty entry - e.g. /usr/include/poll.h + std::vector dummy; + const bool hasInclude = (dui.std.size() == 5 && dui.std.compare(0,3,"c++") == 0 && dui.std >= "c++17"); MacroMap macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { @@ -3285,26 +3288,26 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; const std::string lhs(macrostr.substr(0,eq)); const std::string rhs(eq==std::string::npos ? std::string("1") : macrostr.substr(eq+1)); - const Macro macro(lhs, rhs, files); + const Macro macro(lhs, rhs, dummy); macros.insert(std::pair(macro.name(), macro)); } - macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", files))); - macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", files))); - macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", files))); + macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", dummy))); + macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", dummy))); + macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", dummy))); struct tm ltime = {}; getLocaltime(ltime); - macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(<ime), files))); - macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), files))); + macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(<ime), dummy))); + macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), dummy))); if (!dui.std.empty()) { std::string std_def = simplecpp::getCStdString(dui.std); if (!std_def.empty()) { - macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, files))); + macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); } else { std_def = simplecpp::getCppStdString(dui.std); if (!std_def.empty()) - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, files))); + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); } } diff --git a/test.cpp b/test.cpp index 322d1876..16c5ce06 100644 --- a/test.cpp +++ b/test.cpp @@ -2714,6 +2714,44 @@ static void token() ASSERT_TOKEN("+22", false, true, false); } +static void preprocess_files() +{ + { + const char code[] = "#define A"; + std::vector files; + + const simplecpp::TokenList tokens = makeTokenList(code, files); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("", *files.cbegin()); + + simplecpp::TokenList tokens2(files); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("", *files.cbegin()); + + std::map filedata; + simplecpp::preprocess(tokens2, tokens, files, filedata, simplecpp::DUI(), nullptr); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("", *files.cbegin()); + } + { + const char code[] = "#define A"; + std::vector files; + + const simplecpp::TokenList tokens = makeTokenList(code, files, "test.cpp"); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("test.cpp", *files.cbegin()); + + simplecpp::TokenList tokens2(files); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("test.cpp", *files.cbegin()); + + std::map filedata; + simplecpp::preprocess(tokens2, tokens, files, filedata, simplecpp::DUI(), nullptr); + ASSERT_EQUALS(1, files.size()); + ASSERT_EQUALS("test.cpp", *files.cbegin()); + } +} + static void fuzz_crash() { { @@ -2949,6 +2987,8 @@ int main(int argc, char **argv) TEST_CASE(token); + TEST_CASE(preprocess_files); + TEST_CASE(fuzz_crash); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; From 3678cd18bd4f8ad8519ac0677f137e741e6c2b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 24 Apr 2024 20:12:15 +0200 Subject: [PATCH 223/381] mitigated `include-what-you-use` warnings (#330) --- simplecpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 812600bf..2be16c1b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -14,19 +14,19 @@ #include #include #include -#include +#include // IWYU pragma: keep #include #include #include #include #include -#include // IWYU pragma: keep +#include #include #include #include #include #include -#include // IWYU pragma: keep +#include #include #include #include From e040047b6bf538fdcdb80d51f0a444d9d0728661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 26 Apr 2024 15:42:21 +0200 Subject: [PATCH 224/381] improved prerequisite checks for `__has_include` handling (#286) --- simplecpp.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2be16c1b..47bb78f8 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1980,7 +1980,7 @@ namespace simplecpp { if (tok->next->str() == "(") ++paren; else if (tok->next->str() == ")") - --paren; + --paren; if (paren == 0) return tok->next->next; tok = tok->next; @@ -2614,11 +2614,20 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::map= "201703L"); +} + static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { + if (!isCpp17OrLater(dui)) + return; + for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { - if (tok->str() != "__has_include") + if (tok->str() != HAS_INCLUDE) continue; simplecpp::Token *tok1 = tok->next; if (!tok1) { @@ -3277,7 +3286,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL // use a dummy vector for the macros because as this is not part of the file and would add an empty entry - e.g. /usr/include/poll.h std::vector dummy; - const bool hasInclude = (dui.std.size() == 5 && dui.std.compare(0,3,"c++") == 0 && dui.std >= "c++17"); + const bool hasInclude = isCpp17OrLater(dui); MacroMap macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; From 8d8926bab3b21f0bbf0b84e7259bf91f99d3cc22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 22 May 2024 06:55:53 +0200 Subject: [PATCH 225/381] read headers via `FileStream` (#353) --- simplecpp.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 47bb78f8..fb5a047f 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3463,7 +3463,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::ifstream f; header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); if (f.is_open()) { - TokenList * const tokens = new TokenList(f, files, header2, outputList); + f.close(); + TokenList * const tokens = new TokenList(header2, files, outputList); if (dui.removeComments) tokens->removeComments(); filedata[header2] = tokens; From 59f376ca6d0f85f1cc8e2700912b71806c67cf9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 22 May 2024 09:08:59 +0200 Subject: [PATCH 226/381] bail out on unknown `DUI::std` value (#312) --- main.cpp | 3 +++ simplecpp.cpp | 13 +++++++++++-- simplecpp.h | 3 ++- test.cpp | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/main.cpp b/main.cpp index 6b5b0d0e..3f02773f 100644 --- a/main.cpp +++ b/main.cpp @@ -169,6 +169,9 @@ int main(int argc, char **argv) case simplecpp::Output::FILE_NOT_FOUND: std::cerr << "file not found: "; break; + case simplecpp::Output::DUI_ERROR: + std::cerr << "dui error: "; + break; } std::cerr << output.msg << std::endl; } diff --git a/simplecpp.cpp b/simplecpp.cpp index fb5a047f..af9a8ced 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3315,8 +3315,17 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); } else { std_def = simplecpp::getCppStdString(dui.std); - if (!std_def.empty()) - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); + if (std_def.empty()) { + if (outputList) { + simplecpp::Output err(files); + err.type = Output::DUI_ERROR; + err.msg = "unknown standard specified: '" + dui.std + "'"; + outputList->push_back(err); + } + output.clear(); + return; + } + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); } } diff --git a/simplecpp.h b/simplecpp.h index c72e4c65..f495effd 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -181,7 +181,8 @@ namespace simplecpp { PORTABILITY_BACKSLASH, UNHANDLED_CHAR_ERROR, EXPLICIT_INCLUDE_NOT_FOUND, - FILE_NOT_FOUND + FILE_NOT_FOUND, + DUI_ERROR } type; explicit Output(const std::vector& files, Type type, const std::string& msg) : type(type), location(files), msg(msg) {} Location location; diff --git a/test.cpp b/test.cpp index 16c5ce06..d00658ad 100644 --- a/test.cpp +++ b/test.cpp @@ -154,6 +154,9 @@ static std::string toString(const simplecpp::OutputList &outputList) case simplecpp::Output::Type::FILE_NOT_FOUND: ostr << "file_not_found,"; break; + case simplecpp::Output::Type::DUI_ERROR: + ostr << "dui_error,"; + break; } ostr << output.msg << '\n'; @@ -2656,6 +2659,48 @@ static void cpluscplusDefine() ASSERT_EQUALS("\n201103L", preprocess(code, dui)); } +static void invalidStd() +{ + const char code[] = ""; + simplecpp::DUI dui; + simplecpp::OutputList outputList; + + dui.std = "c88"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("unknown standard specified: 'c88'", outputList.cbegin()->msg); + outputList.clear(); + + dui.std = "gnu88"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("unknown standard specified: 'gnu88'", outputList.cbegin()->msg); + outputList.clear(); + + dui.std = "d99"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("unknown standard specified: 'd99'", outputList.cbegin()->msg); + outputList.clear(); + + dui.std = "c++77"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("unknown standard specified: 'c++77'", outputList.cbegin()->msg); + outputList.clear(); + + dui.std = "gnu++33"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("unknown standard specified: 'gnu++33'", outputList.cbegin()->msg); + outputList.clear(); +} + static void assertToken(const std::string& s, bool name, bool number, bool comment, char op, int line) { const std::vector f; @@ -2984,6 +3029,7 @@ int main(int argc, char **argv) TEST_CASE(stdcVersionDefine); TEST_CASE(cpluscplusDefine); + TEST_CASE(invalidStd); TEST_CASE(token); From fdc947f520bc47100ba2d6fbe097044f0a7a76c7 Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 23 May 2024 10:36:46 +0200 Subject: [PATCH 227/381] added Emacs C++ marker to header comment --- simplecpp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.h b/simplecpp.h index f495effd..df0f2858 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -1,4 +1,4 @@ -/* +/* -*- C++ -*- * simplecpp - A simple and high-fidelity C/C++ preprocessor library * Copyright (C) 2016-2023 simplecpp team */ From d90401601af4d7a747ce2b157db1f2d517467333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 24 May 2024 08:53:47 +0200 Subject: [PATCH 228/381] made it possible to create a `TokenList` from a buffer (#347) * added `StdCharBufStream` which reads from a buffer * use `StdCharBufStream` in `Macro` * added `TokenList` constructors which take a buffer --- simplecpp.cpp | 53 +++++++++++++++++++++++++++++++++++++++++++++++++-- simplecpp.h | 4 ++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index af9a8ced..9c3f3a01 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -377,6 +377,42 @@ class StdIStream : public simplecpp::TokenList::Stream { std::istream &istr; }; +class StdCharBufStream : public simplecpp::TokenList::Stream { +public: + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + StdCharBufStream(const unsigned char* str, std::size_t size) + : str(str) + , size(size) + , pos(0) + , lastStatus(0) + { + init(); + } + + virtual int get() OVERRIDE { + if (pos >= size) + return lastStatus = EOF; + return str[pos++]; + } + virtual int peek() OVERRIDE { + if (pos >= size) + return lastStatus = EOF; + return str[pos]; + } + virtual void unget() OVERRIDE { + --pos; + } + virtual bool good() OVERRIDE { + return lastStatus != EOF; + } + +private: + const unsigned char *str; + const std::size_t size; + std::size_t pos; + int lastStatus; +}; + class FileStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members @@ -442,6 +478,20 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector &fi readfile(stream,filename,outputList); } +simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(data, size); + readfile(stream,filename,outputList); +} + +simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(reinterpret_cast(data), size); + readfile(stream,filename,outputList); +} + simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { @@ -1447,8 +1497,7 @@ namespace simplecpp { Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); - std::istringstream istr(def); - StdIStream stream(istr); + StdCharBufStream stream(reinterpret_cast(def.data()), def.size()); tokenListDefine.readfile(stream); if (!parseDefine(tokenListDefine.cfront())) throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); diff --git a/simplecpp.h b/simplecpp.h index df0f2858..398024d1 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -199,6 +199,10 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); From 9f55d0eadfd16b1ccdd5e758a64a930e29444a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Jul 2024 16:45:09 +0200 Subject: [PATCH 229/381] CI-unixish.yml: removed `macos-11` / added `macos-13` and `macos-14` (#358) `macos-11` is no longer available --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index ea6ed37b..f07b179e 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] + os: [ubuntu-20.04, ubuntu-22.04, macos-12, macos-13, macos-14] fail-fast: false runs-on: ${{ matrix.os }} From 8df5a54e4939b2662f3231ec2f636d46f2949988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 20 Jul 2024 09:40:39 +0200 Subject: [PATCH 230/381] use `` in header (#357) --- simplecpp.cpp | 1 + simplecpp.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9c3f3a01..3cce780e 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/simplecpp.h b/simplecpp.h index 398024d1..88a14014 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include From 74273cd5a9e192d2ddb48ca69d5cbbfc26f0bb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 20 Jul 2024 14:00:06 +0200 Subject: [PATCH 231/381] updated GitHub actions to address Node 16 deprecation (#360) * updated `actions/checkout` to `v4` * updated `microsoft/setup-msbuild` to `v2` --- .github/workflows/CI-unixish.yml | 2 +- .github/workflows/CI-windows.yml | 4 ++-- .github/workflows/clang-tidy.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index f07b179e..48e9aa97 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -17,7 +17,7 @@ jobs: CXX: ${{ matrix.compiler }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install missing software on ubuntu if: matrix.os == 'ubuntu-22.04' diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index ed88f4bb..1f78876d 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -22,10 +22,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup msbuild.exe - uses: microsoft/setup-msbuild@v1.1 + uses: microsoft/setup-msbuild@v2 - name: Run cmake if: matrix.os == 'windows-2019' diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index a57f874b..f1337b72 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install missing software run: | From 76368ec7ca769358d6eb0bab5b47bf000cb6fe26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 7 Aug 2024 21:07:11 +0200 Subject: [PATCH 232/381] clang-tidy.yml: updated to Clang 19 (#359) * clang-tidy.yml: updated to Clang 19 * .clang-tidy: disabled `boost-use-ranges` check --- .clang-tidy | 1 + .github/workflows/clang-tidy.yml | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 752ae48f..c03da523 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -17,6 +17,7 @@ Checks: > -objc-*, -openmp-*, -zircon-*, + -boost-use-ranges, -bugprone-branch-clone, -bugprone-easily-swappable-parameters, -bugprone-narrowing-conversions, diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index f1337b72..2c67e10e 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,19 +21,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 18 - sudo apt-get install clang-tidy-18 + sudo ./llvm.sh 19 + sudo apt-get install clang-tidy-19 - name: Verify clang-tidy configuration run: | - clang-tidy-18 --verify-config + clang-tidy-19 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-18 + CXX: clang-19 - name: Clang-Tidy run: | - run-clang-tidy-18 -q -j $(nproc) -p=cmake.output + run-clang-tidy-19 -q -j $(nproc) -p=cmake.output From 7b88c1356298793667ea03c0627d564a0a900e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 9 Sep 2024 15:08:15 +0200 Subject: [PATCH 233/381] introduced enum for C/C++ standards and added functions to get them from strings (#366) --- simplecpp.cpp | 115 +++++++++++++++++++++++++++++++++++--------------- simplecpp.h | 13 ++++++ test.cpp | 18 ++++++++ 3 files changed, 113 insertions(+), 33 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3cce780e..bc6ca411 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3786,56 +3786,105 @@ void simplecpp::cleanup(std::map &filedata) filedata.clear(); } -std::string simplecpp::getCStdString(const std::string &std) +simplecpp::cstd_t simplecpp::getCStd(const std::string &std) { - if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") { - // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments - return ""; - } + if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") + return C89; if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99"|| std == "gnu9x") - return "199901L"; + return C99; if (std == "c11" || std == "c1x" || std == "iso9899:2011" || std == "gnu11" || std == "gnu1x") - return "201112L"; + return C11; if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") - return "201710L"; - if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") { - // supported by GCC 9+ and Clang 9+ - // Clang 9, 10, 11, 12, 13 return "201710L" - // Clang 14, 15, 16, 17 return "202000L" - // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" - return "202311L"; + return C17; + if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") + return C23; + return CUnknown; +} + +std::string simplecpp::getCStdString(cstd_t std) +{ + switch (std) + { + case C89: + // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments + return ""; + case C99: + return "199901L"; + case C11: + return "201112L"; + case C17: + return "201710L"; + case C23: + // supported by GCC 9+ and Clang 9+ + // Clang 9, 10, 11, 12, 13 return "201710L" + // Clang 14, 15, 16, 17 return "202000L" + // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" + return "202311L"; + case CUnknown: + return ""; } return ""; } -std::string simplecpp::getCppStdString(const std::string &std) +std::string simplecpp::getCStdString(const std::string &std) +{ + return getCStdString(getCStd(std)); +} + +simplecpp::cppstd_t simplecpp::getCppStd(const std::string &std) { if (std == "c++98" || std == "c++03" || std == "gnu++98" || std == "gnu++03") - return "199711L"; + return CPP03; if (std == "c++11" || std == "gnu++11" || std == "c++0x" || std == "gnu++0x") - return "201103L"; + return CPP11; if (std == "c++14" || std == "c++1y" || std == "gnu++14" || std == "gnu++1y") - return "201402L"; + return CPP14; if (std == "c++17" || std == "c++1z" || std == "gnu++17" || std == "gnu++1z") - return "201703L"; - if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") { - // GCC 10 returns "201703L" - correct in 11+ - return "202002L"; - } - if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") { - // supported by GCC 11+ and Clang 12+ - // GCC 11, 12, 13 return "202100L" - // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" - // Clang 17, 18 return "202302L" - return "202302L"; - } - if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") { - // supported by Clang 17+ - return "202400L"; + return CPP17; + if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") + return CPP20; + if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") + return CPP23; + if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") + return CPP26; + return CPPUnknown; +} + +std::string simplecpp::getCppStdString(cppstd_t std) +{ + switch (std) + { + case CPP03: + return "199711L"; + case CPP11: + return "201103L"; + case CPP14: + return "201402L"; + case CPP17: + return "201703L"; + case CPP20: + // GCC 10 returns "201703L" - correct in 11+ + return "202002L"; + case CPP23: + // supported by GCC 11+ and Clang 12+ + // GCC 11, 12, 13 return "202100L" + // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" + // Clang 17, 18 return "202302L" + return "202302L"; + case CPP26: + // supported by Clang 17+ + return "202400L"; + case CPPUnknown: + return ""; } return ""; } +std::string simplecpp::getCppStdString(const std::string &std) +{ + return getCppStdString(getCppStd(std)); +} + #if (__cplusplus < 201103L) && !defined(__APPLE__) #undef nullptr #endif diff --git a/simplecpp.h b/simplecpp.h index 88a14014..d6641136 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -39,6 +39,11 @@ #endif namespace simplecpp { + /** C code standard */ + enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 }; + + /** C++ code standard */ + enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; typedef std::string TokenString; class Macro; @@ -368,11 +373,19 @@ namespace simplecpp { /** Convert Cygwin path to Windows path */ SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); + /** Returns the C version a given standard */ + SIMPLECPP_LIB cstd_t getCStd(const std::string &std); + + /** Returns the C++ version a given standard */ + SIMPLECPP_LIB cppstd_t getCppStd(const std::string &std); + /** Returns the __STDC_VERSION__ value for a given standard */ SIMPLECPP_LIB std::string getCStdString(const std::string &std); + SIMPLECPP_LIB std::string getCStdString(cstd_t std); /** Returns the __cplusplus value for a given standard */ SIMPLECPP_LIB std::string getCppStdString(const std::string &std); + SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); } #if defined(_MSC_VER) diff --git a/test.cpp b/test.cpp index d00658ad..6b7cdc96 100644 --- a/test.cpp +++ b/test.cpp @@ -2701,6 +2701,23 @@ static void invalidStd() outputList.clear(); } +static void stdEnum() +{ + ASSERT_EQUALS(simplecpp::cstd_t::C89, simplecpp::getCStd("c89")); + ASSERT_EQUALS(simplecpp::cstd_t::C89, simplecpp::getCStd("c90")); + ASSERT_EQUALS(simplecpp::cstd_t::C11, simplecpp::getCStd("iso9899:2011")); + ASSERT_EQUALS(simplecpp::cstd_t::C23, simplecpp::getCStd("gnu23")); + ASSERT_EQUALS(simplecpp::cstd_t::CUnknown, simplecpp::getCStd("gnu77")); + ASSERT_EQUALS(simplecpp::cstd_t::CUnknown, simplecpp::getCStd("c++11")); + + ASSERT_EQUALS(simplecpp::cppstd_t::CPP03, simplecpp::getCppStd("c++03")); + ASSERT_EQUALS(simplecpp::cppstd_t::CPP03, simplecpp::getCppStd("c++98")); + ASSERT_EQUALS(simplecpp::cppstd_t::CPP17, simplecpp::getCppStd("c++1z")); + ASSERT_EQUALS(simplecpp::cppstd_t::CPP26, simplecpp::getCppStd("gnu++26")); + ASSERT_EQUALS(simplecpp::cppstd_t::CPPUnknown, simplecpp::getCppStd("gnu++77")); + ASSERT_EQUALS(simplecpp::cppstd_t::CPPUnknown, simplecpp::getCppStd("c11")); +} + static void assertToken(const std::string& s, bool name, bool number, bool comment, char op, int line) { const std::vector f; @@ -3030,6 +3047,7 @@ int main(int argc, char **argv) TEST_CASE(stdcVersionDefine); TEST_CASE(cpluscplusDefine); TEST_CASE(invalidStd); + TEST_CASE(stdEnum); TEST_CASE(token); From 2c66d0895ec0b9f34a611376b2225d480e368868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 9 Sep 2024 18:37:13 +0200 Subject: [PATCH 234/381] fixed #365 - do not treat standards with empty define string as unknown (#367) --- simplecpp.cpp | 16 ++++++++++------ test.cpp | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index bc6ca411..d2fa6ee3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3360,12 +3360,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), dummy))); if (!dui.std.empty()) { - std::string std_def = simplecpp::getCStdString(dui.std); - if (!std_def.empty()) { - macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); + const cstd_t c_std = simplecpp::getCStd(dui.std); + if (c_std != CUnknown) { + const std::string std_def = simplecpp::getCStdString(c_std); + if (!std_def.empty()) + macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); } else { - std_def = simplecpp::getCppStdString(dui.std); - if (std_def.empty()) { + const cppstd_t cpp_std = simplecpp::getCppStd(dui.std); + if (cpp_std == CPPUnknown) { if (outputList) { simplecpp::Output err(files); err.type = Output::DUI_ERROR; @@ -3375,7 +3377,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL output.clear(); return; } - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); + const std::string std_def = simplecpp::getCppStdString(cpp_std); + if (!std_def.empty()) + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); } } diff --git a/test.cpp b/test.cpp index 6b7cdc96..3a4999a0 100644 --- a/test.cpp +++ b/test.cpp @@ -2718,6 +2718,33 @@ static void stdEnum() ASSERT_EQUALS(simplecpp::cppstd_t::CPPUnknown, simplecpp::getCppStd("c11")); } +static void stdValid() +{ + const char code[] = ""; + simplecpp::DUI dui; + simplecpp::OutputList outputList; + + dui.std = "c89"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(0, outputList.size()); + outputList.clear(); + + dui.std = "gnu23"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(0, outputList.size()); + outputList.clear(); + + dui.std = "c++03"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(0, outputList.size()); + outputList.clear(); + + dui.std = "gnu++26"; + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); + ASSERT_EQUALS(0, outputList.size()); + outputList.clear(); +} + static void assertToken(const std::string& s, bool name, bool number, bool comment, char op, int line) { const std::vector f; @@ -3048,6 +3075,7 @@ int main(int argc, char **argv) TEST_CASE(cpluscplusDefine); TEST_CASE(invalidStd); TEST_CASE(stdEnum); + TEST_CASE(stdValid); TEST_CASE(token); From 0442161c1cc6455ca9cb82cf6ae85c124ca841c9 Mon Sep 17 00:00:00 2001 From: datadiode Date: Sat, 28 Sep 2024 11:00:19 +0200 Subject: [PATCH 235/381] Allow reexpansion of currently expanding macros during argument evaluation (#364) --- CMakeLists.txt | 5 +++ run-tests.py | 1 - simplecpp.cpp | 82 +++++++++++++++++++++----------------------------- simplecpp.h | 9 ++++-- test.cpp | 43 ++++++++++++++++++++++++-- 5 files changed, 86 insertions(+), 54 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30ee1431..88c46b9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF) include(CheckCXXCompilerFlag) +if (WIN32) + # prevent simplifyPath_cppcheck() from wasting time on looking for a hypothetical network host + add_definitions(-DUNCHOST=$ENV{COMPUTERNAME}) +endif() + function(add_compile_options_safe FLAG) string(MAKE_C_IDENTIFIER "HAS_CXX_FLAG${FLAG}" mangled_flag) check_cxx_compiler_flag(${FLAG} ${mangled_flag}) diff --git a/run-tests.py b/run-tests.py index c053ed32..2f28bf0f 100644 --- a/run-tests.py +++ b/run-tests.py @@ -71,7 +71,6 @@ def cleanup(out): 'c99-6_10_3_4_p6.c', 'expr_usual_conversions.c', # condition is true: 4U - 30 >= 0 'stdint.c', - 'stringize_misc.c', # GCC.. 'diagnostic-pragma-1.c', diff --git a/simplecpp.cpp b/simplecpp.cpp index d2fa6ee3..37fdc61b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -102,8 +102,6 @@ static const simplecpp::TokenString ONCE("once"); static const simplecpp::TokenString HAS_INCLUDE("__has_include"); -static const simplecpp::TokenString INNER_COMMA(",,"); - template static std::string toString(T t) { // NOLINTNEXTLINE(misc-const-correctness) - false positive @@ -888,7 +886,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } if (prefix.empty()) - push_back(new Token(s, location)); // push string without newlines + push_back(new Token(s, location, isspace(stream.peekChar()))); // push string without newlines else back()->setstr(prefix + s); @@ -918,7 +916,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } } - push_back(new Token(currentToken, location)); + push_back(new Token(currentToken, location, isspace(stream.peekChar()))); if (multiline) location.col += currentToken.size(); @@ -1548,9 +1546,9 @@ namespace simplecpp { // Copy macro call to a new tokenlist with no linebreaks const Token * const rawtok1 = rawtok; TokenList rawtokens2(inputFiles); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); rawtok = rawtok->next; - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); rawtok = rawtok->next; int par = 1; while (rawtok && par > 0) { @@ -1560,13 +1558,10 @@ namespace simplecpp { --par; else if (rawtok->op == '#' && !sameline(rawtok->previous, rawtok)) throw Error(rawtok->location, "it is invalid to use a preprocessor directive as macro parameter"); - rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location)); + rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); rawtok = rawtok->next; } - bool first = true; - if (valueToken && valueToken->str() == rawtok1->str()) - first = false; - if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros, first)) + if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros)) rawtok = rawtok1->next; } else { rawtok = expand(&output2, rawtok->location, rawtok, macros, expandedmacros); @@ -1622,10 +1617,6 @@ namespace simplecpp { rawtok = rawtok2->next; } output->takeTokens(output2); - for (Token* tok = output->front(); tok; tok = tok->next) { - if (tok->str() == INNER_COMMA) - tok->setstr(","); - } return rawtok; } @@ -1792,28 +1783,12 @@ namespace simplecpp { // A##B => AB tok = expandHashHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens); } else if (tok->op == '#' && sameline(tok, tok->next) && tok->next->op != '#') { - tok = expandHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens); + tok = expandHash(tokens, rawloc, tok, expandedmacros, parametertokens); } else { if (!expandArg(tokens, tok, rawloc, macros, expandedmacros, parametertokens)) { - bool expanded = false; - const MacroMap::const_iterator it = macros.find(tok->str()); - if (it != macros.end() && expandedmacros.find(tok->str()) == expandedmacros.end()) { - const Macro &m = it->second; - if (!m.functionLike()) { - Token* mtok = tokens->back(); - m.expand(tokens, rawloc, tok, macros, expandedmacros); - for (mtok = mtok->next; mtok; mtok = mtok->next) { - if (mtok->op == ',') - mtok->setstr(INNER_COMMA); - } - expanded = true; - } - } - if (!expanded) { - tokens->push_back(new Token(*tok)); - if (tok->macro.empty() && (par > 0 || tok->str() != "(")) - tokens->back()->macro = name(); - } + tokens->push_back(new Token(*tok)); + if (tok->macro.empty() && (par > 0 || tok->str() != "(")) + tokens->back()->macro = name(); } if (tok->op == '(') @@ -1831,10 +1806,8 @@ namespace simplecpp { return sameline(lpar,tok) ? tok : nullptr; } - const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros, bool first=false) const { - - if (!first) - expandedmacros.insert(nameTokInst->str()); + const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros) const { + expandedmacros.insert(nameTokInst->str()); usageList.push_back(loc); @@ -1917,6 +1890,14 @@ namespace simplecpp { if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { if (!sameline(tok, tok->next->next->next)) throw invalidHashHash::unexpectedNewline(tok->location, name()); + if (variadic && tok->op == ',' && tok->next->next->next->str() == args.back()) { + Token *const comma = newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok); + output->push_back(comma); + tok = expandToken(output, loc, tok->next->next->next, macros, expandedmacros, parametertokens2); + if (output->back() == comma) + output->deleteToken(comma); + continue; + } TokenList new_output(files); if (!expandArg(&new_output, tok, parametertokens2)) output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok)); @@ -1961,7 +1942,7 @@ namespace simplecpp { tok = expandHashHash(output, loc, tok->previous, macros, expandedmacros, parametertokens2); } else { // #123 => "123" - tok = expandHash(output, loc, tok->previous, macros, expandedmacros, parametertokens2); + tok = expandHash(output, loc, tok->previous, expandedmacros, parametertokens2); } } @@ -2138,14 +2119,17 @@ namespace simplecpp { return true; for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) { const MacroMap::const_iterator it = macros.find(partok->str()); - if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) - partok = it->second.expand(output, loc, partok, macros, expandedmacros); - else { + if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { + const std::set expandedmacros2; // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation + partok = it->second.expand(output, loc, partok, macros, expandedmacros2); + } else { output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); output->back()->macro = partok->macro; partok = partok->next; } } + if (tok->whitespaceahead && output->back()) + output->back()->whitespaceahead = true; return true; } @@ -2154,18 +2138,22 @@ namespace simplecpp { * @param output destination tokenlist * @param loc location for expanded token * @param tok The # token - * @param macros all macros * @param expandedmacros set with expanded macros, with this macro * @param parametertokens parameters given when expanding this macro * @return token after the X */ - const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const std::set &expandedmacros, const std::vector ¶metertokens) const { TokenList tokenListHash(files); - tok = expandToken(&tokenListHash, loc, tok->next, macros, expandedmacros, parametertokens); + const MacroMap macros2; // temporarily bypass macro expansion + tok = expandToken(&tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens); std::ostringstream ostr; ostr << '\"'; - for (const Token *hashtok = tokenListHash.cfront(); hashtok; hashtok = hashtok->next) + for (const Token *hashtok = tokenListHash.cfront(), *next; hashtok; hashtok = next) { + next = hashtok->next; ostr << hashtok->str(); + if (next && hashtok->whitespaceahead) + ostr << ' '; + } ostr << '\"'; output->push_back(newMacroToken(escapeString(ostr.str()), loc, isReplaced(expandedmacros))); return tok; diff --git a/simplecpp.h b/simplecpp.h index d6641136..f5c69593 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -99,13 +99,13 @@ namespace simplecpp { */ class SIMPLECPP_LIB Token { public: - Token(const TokenString &s, const Location &loc) : - location(loc), previous(nullptr), next(nullptr), string(s) { + Token(const TokenString &s, const Location &loc, bool wsahead = false) : + whitespaceahead(wsahead), location(loc), previous(nullptr), next(nullptr), string(s) { flags(); } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { } void flags() { @@ -137,6 +137,7 @@ namespace simplecpp { bool comment; bool name; bool number; + bool whitespaceahead; Location location; Token *previous; Token *next; @@ -158,6 +159,8 @@ namespace simplecpp { void setExpandedFrom(const Token *tok, const Macro* m) { mExpandedFrom = tok->mExpandedFrom; mExpandedFrom.insert(m); + if (tok->whitespaceahead) + whitespaceahead = true; } bool isExpandedFrom(const Macro* m) const { return mExpandedFrom.find(m) != mExpandedFrom.end(); diff --git a/test.cpp b/test.cpp index 3a4999a0..82df5a65 100644 --- a/test.cpp +++ b/test.cpp @@ -16,6 +16,9 @@ #include #include +#define STRINGIZE_(x) #x +#define STRINGIZE(x) STRINGIZE_(x) + static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) @@ -44,7 +47,7 @@ static int assertEquals(const std::string &expected, const std::string &actual, return (expected == actual); } -static int assertEquals(const unsigned int &expected, const unsigned int &actual, int line) +static int assertEquals(const long long &expected, const long long &actual, int line) { return assertEquals(std::to_string(expected), std::to_string(actual), line); } @@ -717,6 +720,17 @@ static void define_define_11() ASSERT_EQUALS("\n\n\n\nP2DIR ;", preprocess(code)); } +static void define_define_11a() +{ + const char code[] = "#define A_B_C 0x1\n" + "#define A_ADDRESS 0x00001000U\n" + "#define A ((uint32_t ) A_ADDRESS)\n" + "#define CONCAT(x, y, z) x ## _ ## y ## _ ## z\n" + "#define TEST_MACRO CONCAT(A, B, C)\n" + "TEST_MACRO\n"; + ASSERT_EQUALS("\n\n\n\n\n0x1", preprocess(code)); +} + static void define_define_12() { const char code[] = "#define XY(Z) Z\n" @@ -1019,6 +1033,17 @@ static void hash() preprocess("#define A(x) (x)\n" "#define B(x) A(#x)\n" "B(123)")); + + ASSERT_EQUALS("\n\nprintf ( \"bar(3)\" \"\\n\" ) ;", + preprocess("#define bar(x) x % 2\n" + "#define foo(x) printf(#x \"\\n\")\n" + "foo(bar(3));")); + + ASSERT_EQUALS("\n\n\n\"Y Y\"", + preprocess("#define X(x,y) x y\n" + "#define STR_(x) #x\n" + "#define STR(x) STR_(x)\n" + "STR(X(Y,Y))")); } static void hashhash1() // #4703 @@ -1058,6 +1083,16 @@ static void hashhash4() // nonstandard gcc/clang extension for empty varargs ASSERT_EQUALS("\n\na ( 1 ) ;", preprocess(code)); } +static void hashhash4a() +{ + const char code[] = "#define GETMYID(a) ((a))+1\n" + "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" + "#define FIGHT_BAR(c, args...) bar(c, ##args)\n" + "FIGHT_FOO(1, GETMYID(a));\n" + "FIGHT_BAR(1, GETMYID(b));"; + ASSERT_EQUALS("\n\n\nfoo ( 1 , ( ( a ) ) + 1 ) ;\nbar ( 1 , ( ( b ) ) + 1 ) ;", preprocess(code)); +} + static void hashhash5() { ASSERT_EQUALS("x1", preprocess("x##__LINE__")); @@ -2567,8 +2602,8 @@ static void simplifyPath_cppcheck() ASSERT_EQUALS("src/", simplecpp::simplifyPath("src/abc/../")); // Handling of UNC paths on Windows - ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("//src/test.cpp")); - ASSERT_EQUALS("//src/test.cpp", simplecpp::simplifyPath("///src/test.cpp")); + ASSERT_EQUALS("//" STRINGIZE(UNCHOST) "/test.cpp", simplecpp::simplifyPath("//" STRINGIZE(UNCHOST) "/test.cpp")); + ASSERT_EQUALS("//" STRINGIZE(UNCHOST) "/test.cpp", simplecpp::simplifyPath("///" STRINGIZE(UNCHOST) "/test.cpp")); } static void simplifyPath_New() @@ -2899,6 +2934,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_9); // line break in nested macro call TEST_CASE(define_define_10); TEST_CASE(define_define_11); + TEST_CASE(define_define_11a); TEST_CASE(define_define_12); // expand result of ## TEST_CASE(define_define_13); TEST_CASE(define_define_14); @@ -2936,6 +2972,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash2); TEST_CASE(hashhash3); TEST_CASE(hashhash4); + TEST_CASE(hashhash4a); // #66, #130 TEST_CASE(hashhash5); TEST_CASE(hashhash6); TEST_CASE(hashhash7); // # ## # (C standard; 6.10.3.3.p4) From 26b64318e740ebf3aa9a296b59ba754c8efbc490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 28 Sep 2024 11:15:50 +0200 Subject: [PATCH 236/381] replace 'isspace' with 'std::isspace' (#370) --- simplecpp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 37fdc61b..0201f6cb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -886,7 +886,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } if (prefix.empty()) - push_back(new Token(s, location, isspace(stream.peekChar()))); // push string without newlines + push_back(new Token(s, location, std::isspace(stream.peekChar()))); // push string without newlines else back()->setstr(prefix + s); @@ -916,7 +916,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } } - push_back(new Token(currentToken, location, isspace(stream.peekChar()))); + push_back(new Token(currentToken, location, std::isspace(stream.peekChar()))); if (multiline) location.col += currentToken.size(); From 1c18356c487a035a93543af964e2d0259bb219c2 Mon Sep 17 00:00:00 2001 From: "Justin C. Nelson" Date: Sat, 28 Sep 2024 04:17:23 -0500 Subject: [PATCH 237/381] Multiline Pragma Directive False Positive (#369) When #pragma is a multiline directive, simplecpp doesn't mark the directive as a multiline resulting in a false positive of the end parenthesis ')'. --- simplecpp.cpp | 2 +- test.cpp | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0201f6cb..2dae2f20 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -892,7 +892,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (newlines > 0 ) { const Token * const llTok = lastLineTok(); - if (llTok && llTok->op == '#' && llTok->next && llTok->next->str() == "define" && llTok->next->next) { + if (llTok && llTok->op == '#' && llTok->next && (llTok->next->str() == "define" || llTok->next->str() == "pragma") && llTok->next->next) { multiline += newlines; location.adjust(s); continue; diff --git a/test.cpp b/test.cpp index 82df5a65..1f979dd3 100644 --- a/test.cpp +++ b/test.cpp @@ -924,6 +924,21 @@ static void define_ifdef() } +static void pragma_backslash() +{ + const char code[] = "#pragma comment (longstring, \\\n" + "\"HEADER\\\n" + "This is a very long string that is\\\n" + "a multi-line string.\\\n" + "How much more do I have to say?\\\n" + "Well, be prepared, because the\\\n" + "story is just beginning. This is a test\\\n" + "string for demonstration purposes. \")\n"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); +} + static void dollar() { ASSERT_EQUALS("$ab", readfile("$ab")); @@ -2953,6 +2968,8 @@ int main(int argc, char **argv) TEST_CASE(define_va_opt_4); TEST_CASE(define_va_opt_5); + TEST_CASE(pragma_backslash); // multiline pragma directive + // UB: #ifdef as macro parameter TEST_CASE(define_ifdef); From 3588ca804f80e1f0abbcaaf15afdf65c7c9d630e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 30 Sep 2024 13:02:25 +0200 Subject: [PATCH 238/381] CI-unixish.yml: removed unsupported `macos-12` (#371) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 48e9aa97..64fc6ca5 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-20.04, ubuntu-22.04, macos-12, macos-13, macos-14] + os: [ubuntu-20.04, ubuntu-22.04, macos-13, macos-14] fail-fast: false runs-on: ${{ matrix.os }} From 6aa3ea1443a6f5c83f20ada5eeacbe9bdc2cda4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 4 Oct 2024 00:11:28 +0200 Subject: [PATCH 239/381] added `ubuntu-24.04` to CI and made it the primary platform (#372) * added `ubuntu-24.04` to CI and made it the primary platform * CI-unixish.yml: added missing dependency for Clang hardening mode * CI-unixish.yml: removed DWARF workaround for valgrind * CI-unixish.yml: use hardening mode for libc++ --- .github/workflows/CI-unixish.yml | 30 +++++++++++++++++------------- .github/workflows/clang-tidy.yml | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 64fc6ca5..050e1d74 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-20.04, ubuntu-22.04, macos-13, macos-14] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, macos-13, macos-14] fail-fast: false runs-on: ${{ matrix.os }} @@ -20,10 +20,16 @@ jobs: - uses: actions/checkout@v4 - name: Install missing software on ubuntu - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' run: | sudo apt-get update sudo apt-get install valgrind + + - name: Install missing software on ubuntu (clang++) + if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' + run: | + sudo apt-get update + sudo apt-get install libc++-18-dev - name: make simplecpp run: make -j$(nproc) @@ -49,29 +55,27 @@ jobs: ./cmake.output/testrunner - name: Run valgrind - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' run: | make clean - # this valgrind version doesn't support DWARF 5 yet - make -j$(nproc) CXXFLAGS="-gdwarf-4" + make -j$(nproc) valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./simplecpp simplecpp.cpp -e - name: Run with libstdc++ debug mode - if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'g++' + if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++' run: | make clean make -j$(nproc) test selfcheck CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" - # TODO: change it to -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG when the compiler is at least Clang 18 - - name: Run with libc++ debug mode - if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' + - name: Run with libc++ hardening mode + if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_ENABLE_ASSERTIONS=1" LDFLAGS="-lc++" + make -j$(nproc) test selfcheck CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDFLAGS="-lc++" - name: Run AddressSanitizer - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' run: | make clean make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" @@ -79,14 +83,14 @@ jobs: ASAN_OPTIONS: detect_stack_use_after_return=1 - name: Run UndefinedBehaviorSanitizer - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' run: | make clean make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" # TODO: requires instrumented libc++ - name: Run MemorySanitizer - if: false && matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++' + if: false && matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 2c67e10e..22602075 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -7,7 +7,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 From d245956b473797908a8bd4e9d072faaf8100b734 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Sat, 5 Oct 2024 12:37:46 +0200 Subject: [PATCH 240/381] Fix #344 fuzzing crash in simplecpp::preprocess() (#375) --- simplecpp.cpp | 2 +- test.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2dae2f20..5bd5caca 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3480,7 +3480,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL inc2.takeTokens(inc1); } - if (!inc2.empty() && inc2.cfront()->op == '<' && inc2.cback()->op == '>') { + if (!inc1.empty() && !inc2.empty() && inc2.cfront()->op == '<' && inc2.cback()->op == '>') { TokenString hdr; // TODO: Sometimes spaces must be added in the string // Somehow preprocessToken etc must be told that the location should be source location not destination location diff --git a/test.cpp b/test.cpp index 1f979dd3..d8e359e5 100644 --- a/test.cpp +++ b/test.cpp @@ -1829,6 +1829,14 @@ static void missingHeader3() ASSERT_EQUALS("", toString(outputList)); } +static void missingHeader4() +{ + const char code[] = "#/**/include <>\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,No header in #include\n", toString(outputList)); +} + static void nestedInclude() { const char code[] = "#include \"test.h\"\n"; @@ -3057,6 +3065,7 @@ int main(int argc, char **argv) TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); TEST_CASE(missingHeader3); + TEST_CASE(missingHeader4); TEST_CASE(nestedInclude); TEST_CASE(systemInclude); From d9edfb0ac8a2525e903d38639355b5b9ab77ca32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20Gunne=20Lindstr=C3=B6m?= Date: Mon, 21 Oct 2024 14:48:27 +0200 Subject: [PATCH 241/381] Fix #376: Member access on user-defined literal tokenized incorrectly (#379) --- simplecpp.cpp | 2 +- test.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5bd5caca..67e59abb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1002,7 +1002,7 @@ void simplecpp::TokenList::combineOperators() continue; } // float literals.. - if (tok->previous && tok->previous->number && sameline(tok->previous, tok)) { + if (tok->previous && tok->previous->number && sameline(tok->previous, tok) && tok->previous->str().find_first_of("._") == std::string::npos) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp")))) { diff --git a/test.cpp b/test.cpp index d8e359e5..5b889789 100644 --- a/test.cpp +++ b/test.cpp @@ -390,6 +390,8 @@ static void combineOperators_floatliteral() ASSERT_EQUALS("0x1p+3f", preprocess("0x1p+3f")); ASSERT_EQUALS("0x1p+3L", preprocess("0x1p+3L")); ASSERT_EQUALS("1p + 3", preprocess("1p+3")); + ASSERT_EQUALS("1.0_a . b", preprocess("1.0_a.b")); + ASSERT_EQUALS("1_a . b", preprocess("1_a.b")); } static void combineOperators_increment() From 0ab783c7d6c208ac9f83550fc1f1ab9de3280d03 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:58:55 +0200 Subject: [PATCH 242/381] Add test for #374 (#380) --- test.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test.cpp b/test.cpp index 5b889789..8ae2fa25 100644 --- a/test.cpp +++ b/test.cpp @@ -731,6 +731,13 @@ static void define_define_11a() "#define TEST_MACRO CONCAT(A, B, C)\n" "TEST_MACRO\n"; ASSERT_EQUALS("\n\n\n\n\n0x1", preprocess(code)); + + const char code2[] = "#define ADDER_S(a, b) a + b\n" // #374 + "#define ADDER(x) ADDER_S(x)\n" + "#define ARGUMENTS 1, 2\n" + "#define RUN ADDER(ARGUMENTS)\n" + "void f() { RUN; }\n"; + ASSERT_EQUALS("\n\n\n\nvoid f ( ) { 1 + 2 ; }", preprocess(code2)); } static void define_define_12() From 78c34de6790607c17ce1841470f22f49902df01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 10 Dec 2024 14:23:26 +0000 Subject: [PATCH 243/381] Fix #384 (fails to expand inner macro with a parameter that contains a comma) (#392) --- simplecpp.cpp | 10 +++++++--- test.cpp | 13 +++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 67e59abb..798fdeba 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1781,7 +1781,7 @@ namespace simplecpp { while (sameline(lpar, tok)) { if (tok->op == '#' && sameline(tok,tok->next) && tok->next->op == '#' && sameline(tok,tok->next->next)) { // A##B => AB - tok = expandHashHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens); + tok = expandHashHash(tokens, rawloc, tok, macros, expandedmacros, parametertokens, false); } else if (tok->op == '#' && sameline(tok, tok->next) && tok->next->op != '#') { tok = expandHash(tokens, rawloc, tok, expandedmacros, parametertokens); } else { @@ -2168,9 +2168,10 @@ namespace simplecpp { * @param macros all macros * @param expandedmacros set with expanded macros, with this macro * @param parametertokens parameters given when expanding this macro + * @param expandResult expand ## result i.e. "AB"? * @return token after B */ - const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, bool expandResult=true) const { Token *A = output->back(); if (!A) throw invalidHashHash(tok->location, name(), "Missing first argument"); @@ -2260,7 +2261,10 @@ namespace simplecpp { nextTok = tok2->next; } } - expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens); + if (expandResult) + expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens); + else + output->takeTokens(tokens); for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; output->takeTokens(tokensB); diff --git a/test.cpp b/test.cpp index 8ae2fa25..3e8c11a2 100644 --- a/test.cpp +++ b/test.cpp @@ -809,6 +809,18 @@ static void define_define_19() // #292 ASSERT_EQUALS("\n\n\n1 , 2 , 3", preprocess(code)); } +static void define_define_20() // #384 arg contains comma +{ + const char code[] = "#define Z_IS_ENABLED1(config_macro) Z_IS_ENABLED2(_XXXX##config_macro)\n" + "#define _XXXX1 _YYYY,\n" + "#define Z_IS_ENABLED2(one_or_two_args) Z_IS_ENABLED3(one_or_two_args 1, 0)\n" + "#define Z_IS_ENABLED3(ignore_this, val, ...) val\n" + "#define IS_ENABLED(config_macro) Z_IS_ENABLED1(config_macro)\n" + "#define FEATURE 1\n" + "a = IS_ENABLED(FEATURE)\n"; + ASSERT_EQUALS("\n\n\n\n\n\na = 1", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -2975,6 +2987,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_17); TEST_CASE(define_define_18); TEST_CASE(define_define_19); + TEST_CASE(define_define_20); // 384 arg contains comma TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From bacd5bd1ff0ce8863c9382b5ef91fc1f21ce240a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 11 Dec 2024 16:05:53 +0100 Subject: [PATCH 244/381] runastyle --- main.cpp | 15 ++---- simplecpp.cpp | 138 ++++++++++++++++++++++---------------------------- test.cpp | 38 +++++++------- 3 files changed, 85 insertions(+), 106 deletions(-) diff --git a/main.cpp b/main.cpp index 3f02773f..f05cf793 100644 --- a/main.cpp +++ b/main.cpp @@ -29,22 +29,19 @@ int main(int argc, char **argv) bool found = false; const char c = arg[1]; switch (c) { - case 'D': // define symbol - { + case 'D': { // define symbol const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.defines.push_back(value); found = true; break; } - case 'U': // undefine symbol - { + case 'U': { // undefine symbol const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.undefined.insert(value); found = true; break; } - case 'I': // include path - { + case 'I': { // include path const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; dui.includePaths.push_back(value); found = true; @@ -54,8 +51,7 @@ int main(int argc, char **argv) if (std::strncmp(arg, "-include=",9)==0) { dui.includes.push_back(arg+9); found = true; - } - else if (std::strncmp(arg, "-is",3)==0) { + } else if (std::strncmp(arg, "-is",3)==0) { use_istream = true; found = true; } @@ -122,8 +118,7 @@ int main(int argc, char **argv) std::exit(1); } rawtokens = new simplecpp::TokenList(f, files,filename,&outputList); - } - else { + } else { rawtokens = new simplecpp::TokenList(filename,files,&outputList); } rawtokens->removeComments(); diff --git a/simplecpp.cpp b/simplecpp.cpp index 798fdeba..0b791945 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -242,8 +242,7 @@ class simplecpp::TokenList::Stream { virtual void unget() = 0; virtual bool good() = 0; - unsigned char readChar() - { + unsigned char readChar() { unsigned char ch = static_cast(get()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the @@ -271,8 +270,7 @@ class simplecpp::TokenList::Stream { return ch; } - unsigned char peekChar() - { + unsigned char peekChar() { unsigned char ch = static_cast(peek()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the @@ -292,8 +290,7 @@ class simplecpp::TokenList::Stream { return ch; } - void ungetChar() - { + void ungetChar() { unget(); if (isUtf16) unget(); @@ -308,13 +305,11 @@ class simplecpp::TokenList::Stream { } private: - inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const - { + inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const { return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch); } - unsigned short getAndSkipBOM() - { + unsigned short getAndSkipBOM() { const int ch1 = peek(); // The UTF-16 BOM is 0xfffe or 0xfeff. @@ -353,8 +348,7 @@ class StdIStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members EXPLICIT StdIStream(std::istream &istr) - : istr(istr) - { + : istr(istr) { assert(istr.good()); init(); } @@ -383,8 +377,7 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { : str(str) , size(size) , pos(0) - , lastStatus(0) - { + , lastStatus(0) { init(); } @@ -418,8 +411,7 @@ class FileStream : public simplecpp::TokenList::Stream { EXPLICIT FileStream(const std::string &filename, std::vector &files) : file(fopen(filename.c_str(), "rb")) , lastCh(0) - , lastStatus(0) - { + , lastStatus(0) { if (!file) { files.push_back(filename); throw simplecpp::Output(files, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); @@ -455,8 +447,7 @@ class FileStream : public simplecpp::TokenList::Stream { // TODO: use ungetc() as well // UTF-16 has subsequent unget() calls fseek(file, -1, SEEK_CUR); - } - else + } else ungetc(ch, file); } @@ -485,22 +476,19 @@ simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std } simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) - : frontToken(nullptr), backToken(nullptr), files(filenames) + : frontToken(nullptr), backToken(nullptr), files(filenames) { StdCharBufStream stream(reinterpret_cast(data), size); readfile(stream,filename,outputList); } simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) - : frontToken(nullptr), backToken(nullptr), files(filenames) + : frontToken(nullptr), backToken(nullptr), files(filenames) { - try - { + try { FileStream stream(filename, filenames); readfile(stream,filename,outputList); - } - catch(const simplecpp::Output & e) // TODO handle extra type of errors - { + } catch (const simplecpp::Output & e) { // TODO handle extra type of errors outputList->push_back(e); } } @@ -890,7 +878,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, else back()->setstr(prefix + s); - if (newlines > 0 ) { + if (newlines > 0) { const Token * const llTok = lastLineTok(); if (llTok && llTok->op == '#' && llTok->next && (llTok->next->str() == "define" || llTok->next->str() == "pragma") && llTok->next->next) { multiline += newlines; @@ -2009,14 +1997,14 @@ namespace simplecpp { int paren = 1; while (sameline(tok, tok->next)) { if (tok->next->str() == "(") - ++paren; + ++paren; else if (tok->next->str() == ")") - --paren; + --paren; if (paren == 0) - return tok->next->next; + return tok->next->next; tok = tok->next; if (parametertokens.size() > args.size() && parametertokens.front()->next->str() != ")") - tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens)->previous; + tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens)->previous; } } throw Error(tok->location, "Missing parenthesis for __VA_OPT__(content)"); @@ -2708,8 +2696,7 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI header += headerToken->str(); // cppcheck-suppress selfAssignment - platform-dependent implementation header = realFilename(header); - } - else { + } else { header = realFilename(tok1->str().substr(1U, tok1->str().size() - 2U)); } std::ifstream f; @@ -3625,8 +3612,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL header = realFilename(header); if (tok && tok->op == '>') closingAngularBracket = true; - } - else { + } else { header = realFilename(tok->str().substr(1U, tok->str().size() - 2U)); closingAngularBracket = true; } @@ -3799,25 +3785,24 @@ simplecpp::cstd_t simplecpp::getCStd(const std::string &std) std::string simplecpp::getCStdString(cstd_t std) { - switch (std) - { - case C89: - // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments - return ""; - case C99: - return "199901L"; - case C11: - return "201112L"; - case C17: - return "201710L"; - case C23: - // supported by GCC 9+ and Clang 9+ - // Clang 9, 10, 11, 12, 13 return "201710L" - // Clang 14, 15, 16, 17 return "202000L" - // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" - return "202311L"; - case CUnknown: - return ""; + switch (std) { + case C89: + // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments + return ""; + case C99: + return "199901L"; + case C11: + return "201112L"; + case C17: + return "201710L"; + case C23: + // supported by GCC 9+ and Clang 9+ + // Clang 9, 10, 11, 12, 13 return "201710L" + // Clang 14, 15, 16, 17 return "202000L" + // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" + return "202311L"; + case CUnknown: + return ""; } return ""; } @@ -3848,30 +3833,29 @@ simplecpp::cppstd_t simplecpp::getCppStd(const std::string &std) std::string simplecpp::getCppStdString(cppstd_t std) { - switch (std) - { - case CPP03: - return "199711L"; - case CPP11: - return "201103L"; - case CPP14: - return "201402L"; - case CPP17: - return "201703L"; - case CPP20: - // GCC 10 returns "201703L" - correct in 11+ - return "202002L"; - case CPP23: - // supported by GCC 11+ and Clang 12+ - // GCC 11, 12, 13 return "202100L" - // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" - // Clang 17, 18 return "202302L" - return "202302L"; - case CPP26: - // supported by Clang 17+ - return "202400L"; - case CPPUnknown: - return ""; + switch (std) { + case CPP03: + return "199711L"; + case CPP11: + return "201103L"; + case CPP14: + return "201402L"; + case CPP17: + return "201703L"; + case CPP20: + // GCC 10 returns "201703L" - correct in 11+ + return "202002L"; + case CPP23: + // supported by GCC 11+ and Clang 12+ + // GCC 11, 12, 13 return "202100L" + // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" + // Clang 17, 18 return "202302L" + return "202302L"; + case CPP26: + // supported by Clang 17+ + return "202400L"; + case CPPUnknown: + return ""; } return ""; } diff --git a/test.cpp b/test.cpp index 3e8c11a2..783e0175 100644 --- a/test.cpp +++ b/test.cpp @@ -731,7 +731,7 @@ static void define_define_11a() "#define TEST_MACRO CONCAT(A, B, C)\n" "TEST_MACRO\n"; ASSERT_EQUALS("\n\n\n\n\n0x1", preprocess(code)); - + const char code2[] = "#define ADDER_S(a, b) a + b\n" // #374 "#define ADDER(x) ADDER_S(x)\n" "#define ARGUMENTS 1, 2\n" @@ -849,18 +849,18 @@ static void define_va_args_4() // cppcheck trac #9754 ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); } -static void define_va_opt_1() +static void define_va_opt_1() { const char code[] = "#define p1(fmt, args...) printf(fmt __VA_OPT__(,) args)\n" "p1(\"hello\");\n" "p1(\"%s\", \"hello\");\n"; ASSERT_EQUALS("\nprintf ( \"hello\" ) ;\n" - "printf ( \"%s\" , \"hello\" ) ;", - preprocess(code)); + "printf ( \"%s\" , \"hello\" ) ;", + preprocess(code)); } -static void define_va_opt_2() +static void define_va_opt_2() { const char code[] = "#define err(...)\\\n" "__VA_OPT__(\\\n" @@ -883,7 +883,7 @@ static void define_va_opt_3() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code1, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", - toString(outputList)); + toString(outputList)); outputList.clear(); @@ -894,10 +894,10 @@ static void define_va_opt_3() ASSERT_EQUALS("", preprocess(code2, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", - toString(outputList)); + toString(outputList)); } -static void define_va_opt_4() +static void define_va_opt_4() { // missing parenthesis const char code1[] = "#define err(...) __VA_OPT__ something\n" @@ -906,7 +906,7 @@ static void define_va_opt_4() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code1, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", - toString(outputList)); + toString(outputList)); outputList.clear(); @@ -916,7 +916,7 @@ static void define_va_opt_4() ASSERT_EQUALS("", preprocess(code2, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", - toString(outputList)); + toString(outputList)); } static void define_va_opt_5() @@ -928,7 +928,7 @@ static void define_va_opt_5() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", - toString(outputList)); + toString(outputList)); } static void define_ifdef() @@ -948,14 +948,14 @@ static void define_ifdef() static void pragma_backslash() { const char code[] = "#pragma comment (longstring, \\\n" - "\"HEADER\\\n" - "This is a very long string that is\\\n" - "a multi-line string.\\\n" - "How much more do I have to say?\\\n" - "Well, be prepared, because the\\\n" - "story is just beginning. This is a test\\\n" - "string for demonstration purposes. \")\n"; - + "\"HEADER\\\n" + "This is a very long string that is\\\n" + "a multi-line string.\\\n" + "How much more do I have to say?\\\n" + "Well, be prepared, because the\\\n" + "story is just beginning. This is a test\\\n" + "string for demonstration purposes. \")\n"; + simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); } From 0b6feabdd97f2556b17a83c92a5b022b57617502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 11 Dec 2024 16:04:42 +0100 Subject: [PATCH 245/381] Fix #395 (hash hash with an empty __VA_ARGS__ in a macro) --- simplecpp.cpp | 31 +++++++++++++++++++++++++++---- test.cpp | 12 ++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0b791945..476f42ff 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -110,6 +110,15 @@ template static std::string toString(T t) return ostr.str(); } +#ifdef SIMPLECPP_DEBUG_MACRO_EXPANSION +static std::string locstring(const simplecpp::Location &loc) +{ + std::ostringstream ostr; + ostr << '[' << loc.file() << ':' << loc.line << ':' << loc.col << ']'; + return ostr.str(); +} +#endif + static long long stringToLL(const std::string &s) { long long ret; @@ -1528,6 +1537,10 @@ namespace simplecpp { std::vector &inputFiles) const { std::set expandedmacros; +#ifdef SIMPLECPP_DEBUG_MACRO_EXPANSION + std::cout << "expand " << name() << " " << locstring(rawtok->location) << std::endl; +#endif + TokenList output2(inputFiles); if (functionLike() && rawtok->next && rawtok->next->op == '(') { @@ -1797,6 +1810,10 @@ namespace simplecpp { const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros) const { expandedmacros.insert(nameTokInst->str()); +#ifdef SIMPLECPP_DEBUG_MACRO_EXPANSION + std::cout << " expand " << name() << " " << locstring(defineLocation()) << std::endl; +#endif + usageList.push_back(loc); if (nameTokInst->str() == "__FILE__") { @@ -2168,8 +2185,7 @@ namespace simplecpp { const bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; const bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); - if (!A->name && !A->number && A->op != ',' && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar) - throw invalidHashHash::unexpectedToken(tok->location, name(), A); + const bool unexpectedA = (!A->name && !A->number && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar); Token * const B = tok->next->next; if (!B->name && !B->number && B->op && !B->isOneOf("#=")) @@ -2187,6 +2203,9 @@ namespace simplecpp { const Token *nextTok = B->next; if (canBeConcatenatedStringOrChar) { + if (unexpectedA) + throw invalidHashHash::unexpectedToken(tok->location, name(), A); + // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. // TODO The question is whether the ## or varargs may still apply, and how to provoke? if (expandArg(&tokensB, B, parametertokens)) { @@ -2205,13 +2224,17 @@ namespace simplecpp { if (expandArg(&tokensB, B, parametertokens)) { if (tokensB.empty()) strAB = A->str(); - else if (varargs && A->op == ',') { + else if (varargs && A->op == ',') strAB = ","; - } else { + else if (varargs && unexpectedA) + throw invalidHashHash::unexpectedToken(tok->location, name(), A); + else { strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); } } else { + if (unexpectedA) + throw invalidHashHash::unexpectedToken(tok->location, name(), A); strAB = A->str() + B->str(); } diff --git a/test.cpp b/test.cpp index 783e0175..0f7ebe77 100644 --- a/test.cpp +++ b/test.cpp @@ -1421,6 +1421,17 @@ static void hashhash_null_stmt() ASSERT_EQUALS("\n\n\n\n1 ;", preprocess(code, &outputList)); } +static void hashhash_empty_va_args() +{ + // #395 hash hash with an empty __VA_ARGS__ in a macro + const char code[] = + "#define CAT(a, ...) a##__VA_ARGS__\n" + "#define X(a, ...) CAT(a)\n" + "#define LEVEL_2 (2)\n" + "X(LEVEL_2)\n"; + ASSERT_EQUALS("\n\n\n( 2 )", preprocess(code)); +} + static void hashhash_universal_character() { const char code[] = @@ -3044,6 +3055,7 @@ int main(int argc, char **argv) TEST_CASE(hashhash_invalid_string_number); TEST_CASE(hashhash_invalid_missing_args); TEST_CASE(hashhash_null_stmt); + TEST_CASE(hashhash_empty_va_args); // C standard, 5.1.1.2, paragraph 4: // If a character sequence that matches the syntax of a universal // character name is produced by token concatenation (6.10.3.3), From 4bb6eba874efd4258a516ab2233f58a20d0de40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 13 Dec 2024 10:26:40 +0000 Subject: [PATCH 246/381] Fix #397 (Debracket macro not expanded) (#398) --- simplecpp.cpp | 10 ++++++++-- test.cpp | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 476f42ff..050a5d08 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2046,13 +2046,19 @@ namespace simplecpp { calledMacro.expand(&temp, loc, tok, macros, expandedmacros); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); } - if (!sameline(tok, tok->next) || tok->next->op != '(') { + if (!sameline(tok, tok->next)) { output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } TokenList tokens(files); tokens.push_back(new Token(*tok)); - const Token * const tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); + const Token * tok2 = nullptr; + if (tok->next->op == '(') + tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); + else if (expandArg(&tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { + if (tokens.cfront()->next && tokens.cfront()->next->op == '(') + tok2 = tok->next; + } if (!tok2) { output->push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; diff --git a/test.cpp b/test.cpp index 0f7ebe77..bf83763e 100644 --- a/test.cpp +++ b/test.cpp @@ -821,6 +821,20 @@ static void define_define_20() // #384 arg contains comma ASSERT_EQUALS("\n\n\n\n\n\na = 1", preprocess(code)); } +static void define_define_21() // #397 DEBRACKET macro +{ + const char code1[] = "#define A(val) B val\n" + "#define B(val) val\n" + "A((2))\n"; + ASSERT_EQUALS("\n\n2", preprocess(code1)); + + const char code2[] = "#define x (2)\n" + "#define A B x\n" + "#define B(val) val\n" + "A\n"; + ASSERT_EQUALS("\n\n\nB ( 2 )", preprocess(code2)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -2999,6 +3013,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_18); TEST_CASE(define_define_19); TEST_CASE(define_define_20); // 384 arg contains comma + TEST_CASE(define_define_21); TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From 6a6865d3971ea5cc17db5244b9a5caf9fd6c39ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Fri, 13 Dec 2024 17:12:12 +0000 Subject: [PATCH 247/381] Fixup #397 (Debracket macro not expanded) (#399) --- simplecpp.cpp | 1 + test.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 050a5d08..604bdd19 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2056,6 +2056,7 @@ namespace simplecpp { if (tok->next->op == '(') tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); else if (expandArg(&tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { + tokens.front()->location = loc; if (tokens.cfront()->next && tokens.cfront()->next->op == '(') tok2 = tok->next; } diff --git a/test.cpp b/test.cpp index bf83763e..97686aec 100644 --- a/test.cpp +++ b/test.cpp @@ -833,6 +833,12 @@ static void define_define_21() // #397 DEBRACKET macro "#define B(val) val\n" "A\n"; ASSERT_EQUALS("\n\n\nB ( 2 )", preprocess(code2)); + + const char code3[] = "#define __GET_ARG2_DEBRACKET(ignore_this, val, ...) __DEBRACKET val\n" + "#define __DEBRACKET(...) __VA_ARGS__\n" + "#5 \"a.c\"\n" + "__GET_ARG2_DEBRACKET(432 (33), (B))\n"; + ASSERT_EQUALS("\n#line 5 \"a.c\"\nB", preprocess(code3)); } static void define_va_args_1() From a853253fc1958000ecad8543d5fd15b6a6b34959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sat, 14 Dec 2024 12:52:44 +0000 Subject: [PATCH 248/381] Fix #400 (inner macro not expanded after hash hash) (#401) --- simplecpp.cpp | 2 +- test.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 604bdd19..3e9dda6c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2066,7 +2066,7 @@ namespace simplecpp { } TokenList temp(files); calledMacro.expand(&temp, loc, tokens.cfront(), macros, expandedmacros); - return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros2, parametertokens); + return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens); } if (tok->str() == DEFINED) { diff --git a/test.cpp b/test.cpp index 97686aec..3ff00b33 100644 --- a/test.cpp +++ b/test.cpp @@ -841,6 +841,15 @@ static void define_define_21() // #397 DEBRACKET macro ASSERT_EQUALS("\n#line 5 \"a.c\"\nB", preprocess(code3)); } +static void define_define_22() // #400 inner macro not expanded after hash hash +{ + const char code[] = "#define FOO(a) CAT(DO, STUFF)(1,2)\n" + "#define DOSTUFF(a, b) CAT(3, 4)\n" + "#define CAT(a, b) a##b\n" + "FOO(1)\n"; + ASSERT_EQUALS("\n\n\n34", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -3020,6 +3029,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_19); TEST_CASE(define_define_20); // 384 arg contains comma TEST_CASE(define_define_21); + TEST_CASE(define_define_22); // #400 TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From c4b6e37a55128e0cdc7ba1145bee62bb1d880f65 Mon Sep 17 00:00:00 2001 From: Tal500 Date: Wed, 1 Jan 2025 23:47:16 +0200 Subject: [PATCH 249/381] fix: use both absolute and relative header paths in header matching (#362) Co-authored-by: Tal Hadad --- .github/workflows/CI-unixish.yml | 17 ++++- .github/workflows/CI-windows.yml | 18 ++++- .gitignore | 3 + integration_test.py | 94 ++++++++++++++++++++++++ simplecpp.cpp | 121 +++++++++++++++++++++++++------ testutils.py | 57 +++++++++++++++ 6 files changed, 286 insertions(+), 24 deletions(-) create mode 100644 integration_test.py create mode 100644 testutils.py diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 050e1d74..9a0e8d8e 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -30,7 +30,18 @@ jobs: run: | sudo apt-get update sudo apt-get install libc++-18-dev - + + - name: Install missing software on macos + if: contains(matrix.os, 'macos') + run: | + brew install python3 + + - name: Install missing Python packages + run: | + python3 -m pip config set global.break-system-packages true + python3 -m pip install pip --upgrade + python3 -m pip install pytest + - name: make simplecpp run: make -j$(nproc) @@ -41,6 +52,10 @@ jobs: run: | make -j$(nproc) selfcheck + - name: integration test + run: | + python3 -m pytest integration_test.py + - name: Run CMake run: | cmake -S . -B cmake.output diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 1f78876d..50d5a84e 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -26,7 +26,18 @@ jobs: - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 - + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + check-latest: true + + - name: Install missing Python packages + run: | + python -m pip install pip --upgrade || exit /b !errorlevel! + python -m pip install pytest || exit /b !errorlevel! + - name: Run cmake if: matrix.os == 'windows-2019' run: | @@ -48,4 +59,9 @@ jobs: - name: Selfcheck run: | .\${{ matrix.config }}\simplecpp.exe simplecpp.cpp -e || exit /b !errorlevel! + + - name: integration test + run: | + set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe + python -m pytest integration_test.py || exit /b !errorlevel! diff --git a/.gitignore b/.gitignore index 183545f1..113cf360 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ testrunner # CLion /.idea /cmake-build-* + +# python +__pycache__/ diff --git a/integration_test.py b/integration_test.py new file mode 100644 index 00000000..0b2b0b38 --- /dev/null +++ b/integration_test.py @@ -0,0 +1,94 @@ +## test with python -m pytest integration_test.py + +import os +import pytest +from testutils import simplecpp, format_include_path_arg, format_include + +def __test_relative_header_create_header(dir, with_pragma_once=True): + header_file = os.path.join(dir, 'test.h') + with open(header_file, 'wt') as f: + f.write(f""" + {"#pragma once" if with_pragma_once else ""} + #ifndef TEST_H_INCLUDED + #define TEST_H_INCLUDED + #else + #error header_was_already_included + #endif + """) + return header_file, "error: #error header_was_already_included" + +def __test_relative_header_create_source(dir, include1, include2, is_include1_sys=False, is_include2_sys=False, inv=False): + if inv: + return __test_relative_header_create_source(dir, include1=include2, include2=include1, is_include1_sys=is_include2_sys, is_include2_sys=is_include1_sys) + ## otherwise + + src_file = os.path.join(dir, 'test.c') + with open(src_file, 'wt') as f: + f.write(f""" + #undef TEST_H_INCLUDED + #include {format_include(include1, is_include1_sys)} + #include {format_include(include2, is_include2_sys)} + """) + return src_file + +@pytest.mark.parametrize("with_pragma_once", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +def test_relative_header_1(tmpdir, with_pragma_once, is_sys): + _, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) + + test_file = __test_relative_header_create_source(tmpdir, "test.h", "test.h", is_include1_sys=is_sys, is_include2_sys=is_sys) + + args = ([format_include_path_arg(tmpdir)] if is_sys else []) + [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + + if with_pragma_once: + assert stderr == '' + else: + assert double_include_error in stderr + +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_2(tmpdir, inv): + header_file, _ = __test_relative_header_create_header(tmpdir) + + test_file = __test_relative_header_create_source(tmpdir, "test.h", header_file, inv=inv) + + args = [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + assert stderr == '' + +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_3(tmpdir, is_sys, inv): + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + header_file, _ = __test_relative_header_create_header(test_subdir) + + test_file = __test_relative_header_create_source(tmpdir, "test_subdir/test.h", header_file, is_include1_sys=is_sys, inv=inv) + + args = [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + + if is_sys: + assert "missing header: Header not found" in stderr + else: + assert stderr == '' + +@pytest.mark.parametrize("use_short_path", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_4(tmpdir, use_short_path, is_sys, inv): + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + header_file, _ = __test_relative_header_create_header(test_subdir) + if use_short_path: + header_file = "test_subdir/test.h" + + test_file = __test_relative_header_create_source(tmpdir, header_file, "test.h", is_include2_sys=is_sys, inv=inv) + + args = [format_include_path_arg(test_subdir), test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + assert stderr == '' diff --git a/simplecpp.cpp b/simplecpp.cpp index 3e9dda6c..20ae2528 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -43,6 +43,8 @@ #ifdef SIMPLECPP_WINDOWS #include #undef ERROR +#else +#include #endif #if __cplusplus >= 201103L @@ -147,6 +149,11 @@ static unsigned long long stringToULL(const std::string &s) return ret; } +static bool startsWith(const std::string &s, const std::string &p) +{ + return (s.size() >= p.size()) && std::equal(p.begin(), p.end(), s.begin()); +} + static bool endsWith(const std::string &s, const std::string &e) { return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); @@ -2680,6 +2687,46 @@ static bool isCpp17OrLater(const simplecpp::DUI &dui) return !std_ver.empty() && (std_ver >= "201703L"); } + +static std::string currentDirectoryOSCalc() { +#ifdef SIMPLECPP_WINDOWS + TCHAR NPath[MAX_PATH]; + GetCurrentDirectory(MAX_PATH, NPath); + return NPath; +#else + const std::size_t size = 1024; + char the_path[size]; + getcwd(the_path, size); + return the_path; +#endif +} + +static const std::string& currentDirectory() { + static const std::string curdir = simplecpp::simplifyPath(currentDirectoryOSCalc()); + return curdir; +} + +static std::string toAbsolutePath(const std::string& path) { + if (path.empty()) { + return path;// preserve error file path that is indicated by an empty string + } + if (!isAbsolutePath(path)) { + return currentDirectory() + "/" + path; + } + // otherwise + return path; +} + +static std::pair extractRelativePathFromAbsolute(const std::string& absolutepath) { + static const std::string prefix = currentDirectory() + "/"; + if (startsWith(absolutepath, prefix)) { + const std::size_t size = prefix.size(); + return std::make_pair(absolutepath.substr(size, absolutepath.size() - size), true); + } + // otherwise + return std::make_pair("", false); +} + static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { @@ -3098,9 +3145,12 @@ static std::string openHeader(std::ifstream &f, const std::string &path) static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) { + std::string path; if (sourcefile.find_first_of("\\/") != std::string::npos) - return simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header); - return simplecpp::simplifyPath(header); + path = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + else + path = header; + return simplecpp::simplifyPath(path); } static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) @@ -3110,7 +3160,7 @@ static std::string openHeaderRelative(std::ifstream &f, const std::string &sourc static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) { - std::string path = includePath; + std::string path = toAbsolutePath(includePath); if (!path.empty() && path[path.size()-1U]!='/' && path[path.size()-1U]!='\\') path += '/'; return path + header; @@ -3119,9 +3169,9 @@ static std::string getIncludePathFileName(const std::string &includePath, const static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) { for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string simplePath = openHeader(f, getIncludePathFileName(*it, header)); - if (!simplePath.empty()) - return simplePath; + std::string path = openHeader(f, getIncludePathFileName(*it, header)); + if (!path.empty()) + return path; } return ""; } @@ -3131,49 +3181,76 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (isAbsolutePath(header)) return openHeader(f, header); - std::string ret; - if (systemheader) { - ret = openHeaderIncludePath(f, dui, header); - return ret; + // always return absolute path for systemheaders + return toAbsolutePath(openHeaderIncludePath(f, dui, header)); } + std::string ret; + ret = openHeaderRelative(f, sourcefile, header); if (ret.empty()) - return openHeaderIncludePath(f, dui, header); + return toAbsolutePath(openHeaderIncludePath(f, dui, header));// in a similar way to system headers return ret; } -static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +static std::string findPathInMapBothRelativeAndAbsolute(const std::map &filedata, const std::string& path) { + // here there are two possibilities - either we match this from absolute path or from a relative one + if (filedata.find(path) != filedata.end()) {// try first to respect the exact match + return path; + } + // otherwise - try to use the normalize to the correct representation + if (isAbsolutePath(path)) { + const std::pair relativeExtractedResult = extractRelativePathFromAbsolute(path); + if (relativeExtractedResult.second) { + const std::string relativePath = relativeExtractedResult.first; + if (filedata.find(relativePath) != filedata.end()) { + return relativePath; + } + } + } else { + const std::string absolutePath = toAbsolutePath(path); + if (filedata.find(absolutePath) != filedata.end()) + return absolutePath; + } + // otherwise + return ""; +} + +static std::string getFileIdPath(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { if (filedata.empty()) { return ""; } if (isAbsolutePath(header)) { - return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; + const std::string simplifiedHeaderPath = simplecpp::simplifyPath(header); + return (filedata.find(simplifiedHeaderPath) != filedata.end()) ? simplifiedHeaderPath : ""; } if (!systemheader) { - const std::string relativeFilename = getRelativeFileName(sourcefile, header); - if (filedata.find(relativeFilename) != filedata.end()) - return relativeFilename; + const std::string relativeOrAbsoluteFilename = getRelativeFileName(sourcefile, header);// unknown if absolute or relative, but always simplified + const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, relativeOrAbsoluteFilename); + if (!match.empty()) { + return match; + } } for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = simplecpp::simplifyPath(getIncludePathFileName(*it, header)); - if (filedata.find(s) != filedata.end()) - return s; + const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, simplecpp::simplifyPath(getIncludePathFileName(*it, header))); + if (!match.empty()) { + return match; + } } if (systemheader && filedata.find(header) != filedata.end()) - return header; + return header;// system header that its file wasn't found in the included paths but alreasy in the filedata - return this as is return ""; } static bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { - return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); + return !getFileIdPath(filedata, sourcefile, header, dui, systemheader).empty(); } std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) @@ -3529,7 +3606,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool systemheader = (inctok->str()[0] == '<'); const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); - std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui, systemheader); + std::string header2 = getFileIdPath(filedata, rawtok->location.file(), header, dui, systemheader); if (header2.empty()) { // try to load file.. std::ifstream f; diff --git a/testutils.py b/testutils.py new file mode 100644 index 00000000..55a2686d --- /dev/null +++ b/testutils.py @@ -0,0 +1,57 @@ +import os +import subprocess +import json + +def __run_subprocess(args, env=None, cwd=None, timeout=None): + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd) + + try: + stdout, stderr = p.communicate(timeout=timeout) + return_code = p.returncode + p = None + except subprocess.TimeoutExpired: + import psutil + # terminate all the child processes + child_procs = psutil.Process(p.pid).children(recursive=True) + if len(child_procs) > 0: + for child in child_procs: + child.terminate() + try: + # call with timeout since it might be stuck + p.communicate(timeout=5) + p = None + except subprocess.TimeoutExpired: + pass + raise + finally: + if p: + # sending the signal to the process groups causes the parent Python process to terminate as well + #os.killpg(os.getpgid(p.pid), signal.SIGTERM) # Send the signal to all the process groups + p.terminate() + stdout, stderr = p.communicate() + p = None + + stdout = stdout.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') + stderr = stderr.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') + + return return_code, stdout, stderr + +def simplecpp(args = [], cwd = None): + dir_path = os.path.dirname(os.path.realpath(__file__)) + if 'SIMPLECPP_EXE_PATH' in os.environ: + simplecpp_path = os.environ['SIMPLECPP_EXE_PATH'] + else: + simplecpp_path = os.path.join(dir_path, "simplecpp") + return __run_subprocess([simplecpp_path] + args, cwd = cwd) + +def quoted_string(s): + return json.dumps(str(s)) + +def format_include_path_arg(include_path): + return f"-I{str(include_path)}" + +def format_include(include, is_sys_header=False): + if is_sys_header: + return f"<{quoted_string(include)[1:-1]}>" + else: + return quoted_string(include) From aa3c4b6af92cbac5abf3c997f2e044db835e6cc6 Mon Sep 17 00:00:00 2001 From: olabetskyi <153490942+olabetskyi@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:29:03 +0200 Subject: [PATCH 250/381] Fix #404: simplecpp::TokenList::constFold does not fold '( 0 ) && 10 < X' properly (#405) --- simplecpp.cpp | 98 +++++++++++++++++++++++++++++++++------------------ simplecpp.h | 2 ++ test.cpp | 26 ++++++++++++++ 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 20ae2528..576b0da7 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -954,7 +954,7 @@ void simplecpp::TokenList::constFold() constFoldQuestionOp(&tok); // If there is no '(' we are done with the constant folding - if (tok->op != '(') + if (!tok || tok->op != '(') break; if (!tok->next || !tok->next->next || tok->next->next->op != ')') @@ -1164,10 +1164,7 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) } else continue; - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + simpleSquash(tok, toString(result)); } } @@ -1187,10 +1184,7 @@ void simplecpp::TokenList::constFoldAddSub(Token *tok) else continue; - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + simpleSquash(tok, toString(result)); } } @@ -1210,10 +1204,7 @@ void simplecpp::TokenList::constFoldShift(Token *tok) else continue; - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + simpleSquash(tok, toString(result)); } } @@ -1247,10 +1238,7 @@ void simplecpp::TokenList::constFoldComparison(Token *tok) else continue; - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + simpleSquash(tok, toString(result)); } } @@ -1282,12 +1270,51 @@ void simplecpp::TokenList::constFoldBitwise(Token *tok) result = (stringToLL(tok->previous->str()) ^ stringToLL(tok->next->str())); else /*if (*op == '|')*/ result = (stringToLL(tok->previous->str()) | stringToLL(tok->next->str())); - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + simpleSquash(tok, toString(result)); + } + } +} + +void simplecpp::TokenList::simpleSquash(Token *&tok, const std::string & result) +{ + tok = tok->previous; + tok->setstr(result); + deleteToken(tok->next); + deleteToken(tok->next); +} + +void simplecpp::TokenList::squashTokens(Token *&tok, const std::set & breakPoints, bool forwardDirection, const std::string & result) +{ + const char * const brackets = forwardDirection ? "()" : ")("; + Token* Token::* const step = forwardDirection ? &Token::next : &Token::previous; + int skip = 0; + const Token * const tok1 = tok->*step; + while (tok1 && tok1->*step) { + if ((tok1->*step)->op == brackets[1]){ + if (skip) { + --skip; + deleteToken(tok1->*step); + } else + break; + } else if ((tok1->*step)->op == brackets[0]) { + ++skip; + deleteToken(tok1->*step); + } else if (skip) { + deleteToken(tok1->*step); + } else if (breakPoints.count((tok1->*step)->str()) != 0) { + break; + } else { + deleteToken(tok1->*step); } } + simpleSquash(tok, result); +} + +static simplecpp::Token * constFoldGetOperand(simplecpp::Token * tok, bool forwardDirection) +{ + simplecpp::Token* simplecpp::Token::* const step = forwardDirection ? &simplecpp::Token::next : &simplecpp::Token::previous; + const char bracket = forwardDirection ? ')' : '('; + return tok->*step && (tok->*step)->number && (!((tok->*step)->*step) || (((tok->*step)->*step)->op == bracket)) ? tok->*step : nullptr; } static const std::string AND("and"); @@ -1303,21 +1330,24 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) } if (tok->str() != "&&" && tok->str() != "||") continue; - if (!tok->previous || !tok->previous->number) - continue; - if (!tok->next || !tok->next->number) + const Token* const lhs = constFoldGetOperand(tok, false); + const Token* const rhs = constFoldGetOperand(tok, true); + if (!lhs) // if lhs is not a single number we don't need to fold continue; - int result; - if (tok->str() == "||") - result = (stringToLL(tok->previous->str()) || stringToLL(tok->next->str())); - else /*if (tok->str() == "&&")*/ - result = (stringToLL(tok->previous->str()) && stringToLL(tok->next->str())); - - tok = tok->previous; - tok->setstr(toString(result)); - deleteToken(tok->next); - deleteToken(tok->next); + std::set breakPoints; + breakPoints.insert(":"); + breakPoints.insert("?"); + if (tok->str() == "||"){ + if (stringToLL(lhs->str()) != 0LL || (rhs && stringToLL(rhs->str()) != 0LL)) + squashTokens(tok, breakPoints, stringToLL(lhs->str()) != 0LL, toString(1)); + } else /*if (tok->str() == "&&")*/ { + breakPoints.insert("||"); + if (stringToLL(lhs->str()) == 0LL || (rhs && stringToLL(rhs->str()) == 0LL)) + squashTokens(tok, breakPoints, stringToLL(lhs->str()) == 0LL, toString(0)); + else if (rhs && stringToLL(lhs->str()) && stringToLL(rhs->str())) + simpleSquash(tok, "1"); + } } } diff --git a/simplecpp.h b/simplecpp.h index f5c69593..0be48306 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -301,6 +301,8 @@ namespace simplecpp { void constFoldLogicalOp(Token *tok); void constFoldQuestionOp(Token **tok1); + void simpleSquash(Token *&tok, const std::string & result); + void squashTokens(Token *&tok, const std::set & breakPoints, bool forwardDirection, const std::string & result); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); diff --git a/test.cpp b/test.cpp index 3ff00b33..622c9b90 100644 --- a/test.cpp +++ b/test.cpp @@ -452,6 +452,15 @@ static void constFold() ASSERT_EQUALS("1", testConstFold("010==8")); ASSERT_EQUALS("exception", testConstFold("!1 ? 2 :")); ASSERT_EQUALS("exception", testConstFold("?2:3")); + ASSERT_EQUALS("0", testConstFold("( 0 ) && 10 < X")); + ASSERT_EQUALS("0", testConstFold("1+2*(3+4) && 7 - 7")); + ASSERT_EQUALS("1", testConstFold("( 1 ) || 10 < X")); + ASSERT_EQUALS("1", testConstFold("1+2*(3+4) || 8 - 7")); + ASSERT_EQUALS("X && 0", testConstFold("X && 0")); + ASSERT_EQUALS("X >= 0 || 0 < Y", testConstFold("X >= 0 || 0 < Y")); + ASSERT_EQUALS("X && 1 && Z", testConstFold("X && (1 || Y) && Z")); + ASSERT_EQUALS("0 || Y", testConstFold("0 && X || Y")); + ASSERT_EQUALS("X > 0 && Y", testConstFold("X > 0 && Y")); } #ifdef __CYGWIN__ @@ -1598,6 +1607,22 @@ static void ifA() ASSERT_EQUALS("\nX", preprocess(code, dui)); } +static void ifXorY() +{ + const char code[] = "#if Z > 0 || 0 < Y\n" + "X\n" + "#endif"; + ASSERT_EQUALS("", preprocess(code)); + + simplecpp::DUI dui; + dui.defines.push_back("Z=1"); + ASSERT_EQUALS("\nX", preprocess(code, dui)); + + dui.defines.clear(); + dui.defines.push_back("Y=15"); + ASSERT_EQUALS("\nX", preprocess(code, dui)); +} + static void ifCharLiteral() { const char code[] = "#if ('A'==0x41)\n" @@ -3104,6 +3129,7 @@ int main(int argc, char **argv) TEST_CASE(ifdef2); TEST_CASE(ifndef); TEST_CASE(ifA); + TEST_CASE(ifXorY); TEST_CASE(ifCharLiteral); TEST_CASE(ifDefined); TEST_CASE(ifDefinedNoPar); From cc5738cbbb7ac0a8d121ef5a6c67af08bbb9b936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 10 Feb 2025 12:45:11 +0100 Subject: [PATCH 251/381] bumped minimum CMake version to 3.10 (#408) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 88c46b9e..c3fcf4ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.5) +cmake_minimum_required (VERSION 3.10) project (simplecpp LANGUAGES CXX) option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF) From 48a958fe25f0f0f39520b96858b5d6fa6b2c705c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 12 Feb 2025 09:59:54 +0000 Subject: [PATCH 252/381] Fix #403 (Stack overflow in Macro::expand()) (#411) --- simplecpp.cpp | 3 ++- test.cpp | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 576b0da7..d8880355 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2169,7 +2169,8 @@ namespace simplecpp { for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U];) { const MacroMap::const_iterator it = macros.find(partok->str()); if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { - const std::set expandedmacros2; // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation + std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation + expandedmacros2.erase(name()); partok = it->second.expand(output, loc, partok, macros, expandedmacros2); } else { output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); diff --git a/test.cpp b/test.cpp index 622c9b90..3e245516 100644 --- a/test.cpp +++ b/test.cpp @@ -859,6 +859,16 @@ static void define_define_22() // #400 inner macro not expanded after hash hash ASSERT_EQUALS("\n\n\n34", preprocess(code)); } +static void define_define_23() // #403 crash (infinite recursion) +{ + const char code[] = "#define C_(x, y) x ## y\n" + "#define C(x, y) C_(x, y)\n" + "#define X(func) C(Y, C(func, Z))\n" + "#define die X(die)\n" + "die(void);\n"; + ASSERT_EQUALS("\n\n\n\nYdieZ ( void ) ;", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -3055,6 +3065,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_20); // 384 arg contains comma TEST_CASE(define_define_21); TEST_CASE(define_define_22); // #400 + TEST_CASE(define_define_23); // #403 - crash, infinite recursion TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From 9b0c842f683edef45f758241011f50da93fe02ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 12 Feb 2025 12:49:07 +0000 Subject: [PATCH 253/381] Fix #409 (fuzzing crash in simplecpp::Macro::expandToken()) (#412) --- simplecpp.cpp | 2 +- test.cpp | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d8880355..b8dd063d 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2125,7 +2125,7 @@ namespace simplecpp { if (expandArg(&temp, defToken, parametertokens)) macroName = temp.cback()->str(); if (expandArg(&temp, defToken->next->next->next, parametertokens)) - macroName += temp.cback()->str(); + macroName += temp.cback() ? temp.cback()->str() : ""; else macroName += defToken->next->next->next->str(); lastToken = defToken->next->next->next; diff --git a/test.cpp b/test.cpp index 3e245516..82b9e1c7 100644 --- a/test.cpp +++ b/test.cpp @@ -1717,6 +1717,17 @@ static void ifDefinedHashHash() ASSERT_EQUALS("file0,4,#error,#error FOO is enabled\n", toString(outputList)); } +static void ifDefinedHashHash2() +{ + // #409 + // do not crash when expanding P() (as ## rhs is "null") + // note: gcc outputs "defined E" + const char code[] = "#define P(p)defined E##p\n" + "P()\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("\n0", preprocess(code, &outputList)); +} + static void ifLogical() { const char code[] = "#if defined(A) || defined(B)\n" @@ -3149,6 +3160,7 @@ int main(int argc, char **argv) TEST_CASE(ifDefinedInvalid1); TEST_CASE(ifDefinedInvalid2); TEST_CASE(ifDefinedHashHash); + TEST_CASE(ifDefinedHashHash2); TEST_CASE(ifLogical); TEST_CASE(ifSizeof); TEST_CASE(elif); From 9ce981ccb54928f148a48ad150da9e1b7520b748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 12 Feb 2025 14:37:25 +0000 Subject: [PATCH 254/381] Revert "fix: use both absolute and relative header paths in header matching (#362)" (#415) --- .github/workflows/CI-unixish.yml | 17 +---- .github/workflows/CI-windows.yml | 18 +---- .gitignore | 3 - integration_test.py | 94 ------------------------ simplecpp.cpp | 121 ++++++------------------------- testutils.py | 57 --------------- 6 files changed, 24 insertions(+), 286 deletions(-) delete mode 100644 integration_test.py delete mode 100644 testutils.py diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 9a0e8d8e..050e1d74 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -30,18 +30,7 @@ jobs: run: | sudo apt-get update sudo apt-get install libc++-18-dev - - - name: Install missing software on macos - if: contains(matrix.os, 'macos') - run: | - brew install python3 - - - name: Install missing Python packages - run: | - python3 -m pip config set global.break-system-packages true - python3 -m pip install pip --upgrade - python3 -m pip install pytest - + - name: make simplecpp run: make -j$(nproc) @@ -52,10 +41,6 @@ jobs: run: | make -j$(nproc) selfcheck - - name: integration test - run: | - python3 -m pytest integration_test.py - - name: Run CMake run: | cmake -S . -B cmake.output diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 50d5a84e..1f78876d 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -26,18 +26,7 @@ jobs: - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 - - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: '3.13' - check-latest: true - - - name: Install missing Python packages - run: | - python -m pip install pip --upgrade || exit /b !errorlevel! - python -m pip install pytest || exit /b !errorlevel! - + - name: Run cmake if: matrix.os == 'windows-2019' run: | @@ -59,9 +48,4 @@ jobs: - name: Selfcheck run: | .\${{ matrix.config }}\simplecpp.exe simplecpp.cpp -e || exit /b !errorlevel! - - - name: integration test - run: | - set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe - python -m pytest integration_test.py || exit /b !errorlevel! diff --git a/.gitignore b/.gitignore index 113cf360..183545f1 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,3 @@ testrunner # CLion /.idea /cmake-build-* - -# python -__pycache__/ diff --git a/integration_test.py b/integration_test.py deleted file mode 100644 index 0b2b0b38..00000000 --- a/integration_test.py +++ /dev/null @@ -1,94 +0,0 @@ -## test with python -m pytest integration_test.py - -import os -import pytest -from testutils import simplecpp, format_include_path_arg, format_include - -def __test_relative_header_create_header(dir, with_pragma_once=True): - header_file = os.path.join(dir, 'test.h') - with open(header_file, 'wt') as f: - f.write(f""" - {"#pragma once" if with_pragma_once else ""} - #ifndef TEST_H_INCLUDED - #define TEST_H_INCLUDED - #else - #error header_was_already_included - #endif - """) - return header_file, "error: #error header_was_already_included" - -def __test_relative_header_create_source(dir, include1, include2, is_include1_sys=False, is_include2_sys=False, inv=False): - if inv: - return __test_relative_header_create_source(dir, include1=include2, include2=include1, is_include1_sys=is_include2_sys, is_include2_sys=is_include1_sys) - ## otherwise - - src_file = os.path.join(dir, 'test.c') - with open(src_file, 'wt') as f: - f.write(f""" - #undef TEST_H_INCLUDED - #include {format_include(include1, is_include1_sys)} - #include {format_include(include2, is_include2_sys)} - """) - return src_file - -@pytest.mark.parametrize("with_pragma_once", (False, True)) -@pytest.mark.parametrize("is_sys", (False, True)) -def test_relative_header_1(tmpdir, with_pragma_once, is_sys): - _, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) - - test_file = __test_relative_header_create_source(tmpdir, "test.h", "test.h", is_include1_sys=is_sys, is_include2_sys=is_sys) - - args = ([format_include_path_arg(tmpdir)] if is_sys else []) + [test_file] - - _, _, stderr = simplecpp(args, cwd=tmpdir) - - if with_pragma_once: - assert stderr == '' - else: - assert double_include_error in stderr - -@pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_2(tmpdir, inv): - header_file, _ = __test_relative_header_create_header(tmpdir) - - test_file = __test_relative_header_create_source(tmpdir, "test.h", header_file, inv=inv) - - args = [test_file] - - _, _, stderr = simplecpp(args, cwd=tmpdir) - assert stderr == '' - -@pytest.mark.parametrize("is_sys", (False, True)) -@pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_3(tmpdir, is_sys, inv): - test_subdir = os.path.join(tmpdir, "test_subdir") - os.mkdir(test_subdir) - header_file, _ = __test_relative_header_create_header(test_subdir) - - test_file = __test_relative_header_create_source(tmpdir, "test_subdir/test.h", header_file, is_include1_sys=is_sys, inv=inv) - - args = [test_file] - - _, _, stderr = simplecpp(args, cwd=tmpdir) - - if is_sys: - assert "missing header: Header not found" in stderr - else: - assert stderr == '' - -@pytest.mark.parametrize("use_short_path", (False, True)) -@pytest.mark.parametrize("is_sys", (False, True)) -@pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_4(tmpdir, use_short_path, is_sys, inv): - test_subdir = os.path.join(tmpdir, "test_subdir") - os.mkdir(test_subdir) - header_file, _ = __test_relative_header_create_header(test_subdir) - if use_short_path: - header_file = "test_subdir/test.h" - - test_file = __test_relative_header_create_source(tmpdir, header_file, "test.h", is_include2_sys=is_sys, inv=inv) - - args = [format_include_path_arg(test_subdir), test_file] - - _, _, stderr = simplecpp(args, cwd=tmpdir) - assert stderr == '' diff --git a/simplecpp.cpp b/simplecpp.cpp index b8dd063d..1ae47f5e 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -43,8 +43,6 @@ #ifdef SIMPLECPP_WINDOWS #include #undef ERROR -#else -#include #endif #if __cplusplus >= 201103L @@ -149,11 +147,6 @@ static unsigned long long stringToULL(const std::string &s) return ret; } -static bool startsWith(const std::string &s, const std::string &p) -{ - return (s.size() >= p.size()) && std::equal(p.begin(), p.end(), s.begin()); -} - static bool endsWith(const std::string &s, const std::string &e) { return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); @@ -2718,46 +2711,6 @@ static bool isCpp17OrLater(const simplecpp::DUI &dui) return !std_ver.empty() && (std_ver >= "201703L"); } - -static std::string currentDirectoryOSCalc() { -#ifdef SIMPLECPP_WINDOWS - TCHAR NPath[MAX_PATH]; - GetCurrentDirectory(MAX_PATH, NPath); - return NPath; -#else - const std::size_t size = 1024; - char the_path[size]; - getcwd(the_path, size); - return the_path; -#endif -} - -static const std::string& currentDirectory() { - static const std::string curdir = simplecpp::simplifyPath(currentDirectoryOSCalc()); - return curdir; -} - -static std::string toAbsolutePath(const std::string& path) { - if (path.empty()) { - return path;// preserve error file path that is indicated by an empty string - } - if (!isAbsolutePath(path)) { - return currentDirectory() + "/" + path; - } - // otherwise - return path; -} - -static std::pair extractRelativePathFromAbsolute(const std::string& absolutepath) { - static const std::string prefix = currentDirectory() + "/"; - if (startsWith(absolutepath, prefix)) { - const std::size_t size = prefix.size(); - return std::make_pair(absolutepath.substr(size, absolutepath.size() - size), true); - } - // otherwise - return std::make_pair("", false); -} - static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { @@ -3176,12 +3129,9 @@ static std::string openHeader(std::ifstream &f, const std::string &path) static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) { - std::string path; if (sourcefile.find_first_of("\\/") != std::string::npos) - path = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; - else - path = header; - return simplecpp::simplifyPath(path); + return simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header); + return simplecpp::simplifyPath(header); } static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) @@ -3191,7 +3141,7 @@ static std::string openHeaderRelative(std::ifstream &f, const std::string &sourc static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) { - std::string path = toAbsolutePath(includePath); + std::string path = includePath; if (!path.empty() && path[path.size()-1U]!='/' && path[path.size()-1U]!='\\') path += '/'; return path + header; @@ -3200,9 +3150,9 @@ static std::string getIncludePathFileName(const std::string &includePath, const static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) { for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string path = openHeader(f, getIncludePathFileName(*it, header)); - if (!path.empty()) - return path; + std::string simplePath = openHeader(f, getIncludePathFileName(*it, header)); + if (!simplePath.empty()) + return simplePath; } return ""; } @@ -3212,76 +3162,49 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (isAbsolutePath(header)) return openHeader(f, header); + std::string ret; + if (systemheader) { - // always return absolute path for systemheaders - return toAbsolutePath(openHeaderIncludePath(f, dui, header)); + ret = openHeaderIncludePath(f, dui, header); + return ret; } - std::string ret; - ret = openHeaderRelative(f, sourcefile, header); if (ret.empty()) - return toAbsolutePath(openHeaderIncludePath(f, dui, header));// in a similar way to system headers + return openHeaderIncludePath(f, dui, header); return ret; } -static std::string findPathInMapBothRelativeAndAbsolute(const std::map &filedata, const std::string& path) { - // here there are two possibilities - either we match this from absolute path or from a relative one - if (filedata.find(path) != filedata.end()) {// try first to respect the exact match - return path; - } - // otherwise - try to use the normalize to the correct representation - if (isAbsolutePath(path)) { - const std::pair relativeExtractedResult = extractRelativePathFromAbsolute(path); - if (relativeExtractedResult.second) { - const std::string relativePath = relativeExtractedResult.first; - if (filedata.find(relativePath) != filedata.end()) { - return relativePath; - } - } - } else { - const std::string absolutePath = toAbsolutePath(path); - if (filedata.find(absolutePath) != filedata.end()) - return absolutePath; - } - // otherwise - return ""; -} - -static std::string getFileIdPath(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { if (filedata.empty()) { return ""; } if (isAbsolutePath(header)) { - const std::string simplifiedHeaderPath = simplecpp::simplifyPath(header); - return (filedata.find(simplifiedHeaderPath) != filedata.end()) ? simplifiedHeaderPath : ""; + return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; } if (!systemheader) { - const std::string relativeOrAbsoluteFilename = getRelativeFileName(sourcefile, header);// unknown if absolute or relative, but always simplified - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, relativeOrAbsoluteFilename); - if (!match.empty()) { - return match; - } + const std::string relativeFilename = getRelativeFileName(sourcefile, header); + if (filedata.find(relativeFilename) != filedata.end()) + return relativeFilename; } for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, simplecpp::simplifyPath(getIncludePathFileName(*it, header))); - if (!match.empty()) { - return match; - } + std::string s = simplecpp::simplifyPath(getIncludePathFileName(*it, header)); + if (filedata.find(s) != filedata.end()) + return s; } if (systemheader && filedata.find(header) != filedata.end()) - return header;// system header that its file wasn't found in the included paths but alreasy in the filedata - return this as is + return header; return ""; } static bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { - return !getFileIdPath(filedata, sourcefile, header, dui, systemheader).empty(); + return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); } std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) @@ -3637,7 +3560,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool systemheader = (inctok->str()[0] == '<'); const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); - std::string header2 = getFileIdPath(filedata, rawtok->location.file(), header, dui, systemheader); + std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui, systemheader); if (header2.empty()) { // try to load file.. std::ifstream f; diff --git a/testutils.py b/testutils.py deleted file mode 100644 index 55a2686d..00000000 --- a/testutils.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import subprocess -import json - -def __run_subprocess(args, env=None, cwd=None, timeout=None): - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd) - - try: - stdout, stderr = p.communicate(timeout=timeout) - return_code = p.returncode - p = None - except subprocess.TimeoutExpired: - import psutil - # terminate all the child processes - child_procs = psutil.Process(p.pid).children(recursive=True) - if len(child_procs) > 0: - for child in child_procs: - child.terminate() - try: - # call with timeout since it might be stuck - p.communicate(timeout=5) - p = None - except subprocess.TimeoutExpired: - pass - raise - finally: - if p: - # sending the signal to the process groups causes the parent Python process to terminate as well - #os.killpg(os.getpgid(p.pid), signal.SIGTERM) # Send the signal to all the process groups - p.terminate() - stdout, stderr = p.communicate() - p = None - - stdout = stdout.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') - stderr = stderr.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') - - return return_code, stdout, stderr - -def simplecpp(args = [], cwd = None): - dir_path = os.path.dirname(os.path.realpath(__file__)) - if 'SIMPLECPP_EXE_PATH' in os.environ: - simplecpp_path = os.environ['SIMPLECPP_EXE_PATH'] - else: - simplecpp_path = os.path.join(dir_path, "simplecpp") - return __run_subprocess([simplecpp_path] + args, cwd = cwd) - -def quoted_string(s): - return json.dumps(str(s)) - -def format_include_path_arg(include_path): - return f"-I{str(include_path)}" - -def format_include(include, is_sys_header=False): - if is_sys_header: - return f"<{quoted_string(include)[1:-1]}>" - else: - return quoted_string(include) From 09a816319ca6ccc89f45b4e883db07276d19c295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 12 Feb 2025 19:54:55 +0100 Subject: [PATCH 255/381] Revert "Fix #404: simplecpp::TokenList::constFold does not fold '( 0 ) && 10 < X' properly (#405)" (#416) --- simplecpp.cpp | 98 ++++++++++++++++++--------------------------------- simplecpp.h | 2 -- test.cpp | 26 -------------- 3 files changed, 34 insertions(+), 92 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1ae47f5e..2316c42b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -947,7 +947,7 @@ void simplecpp::TokenList::constFold() constFoldQuestionOp(&tok); // If there is no '(' we are done with the constant folding - if (!tok || tok->op != '(') + if (tok->op != '(') break; if (!tok->next || !tok->next->next || tok->next->next->op != ')') @@ -1157,7 +1157,10 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) } else continue; - simpleSquash(tok, toString(result)); + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } @@ -1177,7 +1180,10 @@ void simplecpp::TokenList::constFoldAddSub(Token *tok) else continue; - simpleSquash(tok, toString(result)); + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } @@ -1197,7 +1203,10 @@ void simplecpp::TokenList::constFoldShift(Token *tok) else continue; - simpleSquash(tok, toString(result)); + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } @@ -1231,7 +1240,10 @@ void simplecpp::TokenList::constFoldComparison(Token *tok) else continue; - simpleSquash(tok, toString(result)); + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } @@ -1263,51 +1275,12 @@ void simplecpp::TokenList::constFoldBitwise(Token *tok) result = (stringToLL(tok->previous->str()) ^ stringToLL(tok->next->str())); else /*if (*op == '|')*/ result = (stringToLL(tok->previous->str()) | stringToLL(tok->next->str())); - simpleSquash(tok, toString(result)); - } - } -} - -void simplecpp::TokenList::simpleSquash(Token *&tok, const std::string & result) -{ - tok = tok->previous; - tok->setstr(result); - deleteToken(tok->next); - deleteToken(tok->next); -} - -void simplecpp::TokenList::squashTokens(Token *&tok, const std::set & breakPoints, bool forwardDirection, const std::string & result) -{ - const char * const brackets = forwardDirection ? "()" : ")("; - Token* Token::* const step = forwardDirection ? &Token::next : &Token::previous; - int skip = 0; - const Token * const tok1 = tok->*step; - while (tok1 && tok1->*step) { - if ((tok1->*step)->op == brackets[1]){ - if (skip) { - --skip; - deleteToken(tok1->*step); - } else - break; - } else if ((tok1->*step)->op == brackets[0]) { - ++skip; - deleteToken(tok1->*step); - } else if (skip) { - deleteToken(tok1->*step); - } else if (breakPoints.count((tok1->*step)->str()) != 0) { - break; - } else { - deleteToken(tok1->*step); + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } - simpleSquash(tok, result); -} - -static simplecpp::Token * constFoldGetOperand(simplecpp::Token * tok, bool forwardDirection) -{ - simplecpp::Token* simplecpp::Token::* const step = forwardDirection ? &simplecpp::Token::next : &simplecpp::Token::previous; - const char bracket = forwardDirection ? ')' : '('; - return tok->*step && (tok->*step)->number && (!((tok->*step)->*step) || (((tok->*step)->*step)->op == bracket)) ? tok->*step : nullptr; } static const std::string AND("and"); @@ -1323,24 +1296,21 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) } if (tok->str() != "&&" && tok->str() != "||") continue; - const Token* const lhs = constFoldGetOperand(tok, false); - const Token* const rhs = constFoldGetOperand(tok, true); - if (!lhs) // if lhs is not a single number we don't need to fold + if (!tok->previous || !tok->previous->number) + continue; + if (!tok->next || !tok->next->number) continue; - std::set breakPoints; - breakPoints.insert(":"); - breakPoints.insert("?"); - if (tok->str() == "||"){ - if (stringToLL(lhs->str()) != 0LL || (rhs && stringToLL(rhs->str()) != 0LL)) - squashTokens(tok, breakPoints, stringToLL(lhs->str()) != 0LL, toString(1)); - } else /*if (tok->str() == "&&")*/ { - breakPoints.insert("||"); - if (stringToLL(lhs->str()) == 0LL || (rhs && stringToLL(rhs->str()) == 0LL)) - squashTokens(tok, breakPoints, stringToLL(lhs->str()) == 0LL, toString(0)); - else if (rhs && stringToLL(lhs->str()) && stringToLL(rhs->str())) - simpleSquash(tok, "1"); - } + int result; + if (tok->str() == "||") + result = (stringToLL(tok->previous->str()) || stringToLL(tok->next->str())); + else /*if (tok->str() == "&&")*/ + result = (stringToLL(tok->previous->str()) && stringToLL(tok->next->str())); + + tok = tok->previous; + tok->setstr(toString(result)); + deleteToken(tok->next); + deleteToken(tok->next); } } diff --git a/simplecpp.h b/simplecpp.h index 0be48306..f5c69593 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -301,8 +301,6 @@ namespace simplecpp { void constFoldLogicalOp(Token *tok); void constFoldQuestionOp(Token **tok1); - void simpleSquash(Token *&tok, const std::string & result); - void squashTokens(Token *&tok, const std::set & breakPoints, bool forwardDirection, const std::string & result); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); diff --git a/test.cpp b/test.cpp index 82b9e1c7..187b7ec6 100644 --- a/test.cpp +++ b/test.cpp @@ -452,15 +452,6 @@ static void constFold() ASSERT_EQUALS("1", testConstFold("010==8")); ASSERT_EQUALS("exception", testConstFold("!1 ? 2 :")); ASSERT_EQUALS("exception", testConstFold("?2:3")); - ASSERT_EQUALS("0", testConstFold("( 0 ) && 10 < X")); - ASSERT_EQUALS("0", testConstFold("1+2*(3+4) && 7 - 7")); - ASSERT_EQUALS("1", testConstFold("( 1 ) || 10 < X")); - ASSERT_EQUALS("1", testConstFold("1+2*(3+4) || 8 - 7")); - ASSERT_EQUALS("X && 0", testConstFold("X && 0")); - ASSERT_EQUALS("X >= 0 || 0 < Y", testConstFold("X >= 0 || 0 < Y")); - ASSERT_EQUALS("X && 1 && Z", testConstFold("X && (1 || Y) && Z")); - ASSERT_EQUALS("0 || Y", testConstFold("0 && X || Y")); - ASSERT_EQUALS("X > 0 && Y", testConstFold("X > 0 && Y")); } #ifdef __CYGWIN__ @@ -1617,22 +1608,6 @@ static void ifA() ASSERT_EQUALS("\nX", preprocess(code, dui)); } -static void ifXorY() -{ - const char code[] = "#if Z > 0 || 0 < Y\n" - "X\n" - "#endif"; - ASSERT_EQUALS("", preprocess(code)); - - simplecpp::DUI dui; - dui.defines.push_back("Z=1"); - ASSERT_EQUALS("\nX", preprocess(code, dui)); - - dui.defines.clear(); - dui.defines.push_back("Y=15"); - ASSERT_EQUALS("\nX", preprocess(code, dui)); -} - static void ifCharLiteral() { const char code[] = "#if ('A'==0x41)\n" @@ -3151,7 +3126,6 @@ int main(int argc, char **argv) TEST_CASE(ifdef2); TEST_CASE(ifndef); TEST_CASE(ifA); - TEST_CASE(ifXorY); TEST_CASE(ifCharLiteral); TEST_CASE(ifDefined); TEST_CASE(ifDefinedNoPar); From 78f0f7cef4d84ffea31af2b810f6778b24f4ea91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 18 Mar 2025 14:51:24 +0100 Subject: [PATCH 256/381] clang-tidy.yml: updated to Clang 20 (#388) --- .github/workflows/clang-tidy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 22602075..9a4f43c9 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -21,19 +21,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 19 - sudo apt-get install clang-tidy-19 + sudo ./llvm.sh 20 + sudo apt-get install clang-tidy-20 - name: Verify clang-tidy configuration run: | - clang-tidy-19 --verify-config + clang-tidy-20 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON env: - CXX: clang-19 + CXX: clang-20 - name: Clang-Tidy run: | - run-clang-tidy-19 -q -j $(nproc) -p=cmake.output + run-clang-tidy-20 -q -j $(nproc) -p=cmake.output From 0c8867436fca46820b00cd36cfceebf6f48b56ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 13 Apr 2025 14:50:09 +0200 Subject: [PATCH 257/381] CI-unixish.yml: removed `ubuntu-20.04` (#425) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 050e1d74..46f3dc02 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, macos-13, macos-14] + os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14] fail-fast: false runs-on: ${{ matrix.os }} From 6adb70f125439a97af6d0136e78de482c3799f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 6 May 2025 10:49:50 +0200 Subject: [PATCH 258/381] CI-windows.yml: removed `windows-2019` and added `windows-2025` (#426) --- .github/workflows/CI-windows.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 1f78876d..37ebf4fc 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - os: [windows-2019, windows-2022] + os: [windows-2022, windows-2025] config: [Release, Debug] fail-fast: false @@ -27,13 +27,7 @@ jobs: - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 - - name: Run cmake - if: matrix.os == 'windows-2019' - run: | - cmake -G "Visual Studio 16 2019" -A x64 . || exit /b !errorlevel! - - - name: Run cmake - if: matrix.os == 'windows-2022' + - name: Run CMake run: | cmake -G "Visual Studio 17 2022" -A x64 . || exit /b !errorlevel! From 62afdd0b754dbbeba1d2c34cae172786da1e201b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 6 May 2025 17:17:14 +0200 Subject: [PATCH 259/381] CI-unixish.yml: added `macos-15` (#407) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 46f3dc02..7e962a47 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: compiler: [clang++, g++] - os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14] + os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14, macos-15] fail-fast: false runs-on: ${{ matrix.os }} From 07757314899f0a59d4bdb480015f21789a1db9be Mon Sep 17 00:00:00 2001 From: Tal500 Date: Fri, 9 May 2025 16:40:01 +0300 Subject: [PATCH 260/381] Fix relative paths, again (#418) Co-authored-by: Tal Hadad --- .github/workflows/CI-unixish.yml | 16 +++- .github/workflows/CI-windows.yml | 18 ++++- .gitignore | 3 + integration_test.py | 94 ++++++++++++++++++++++ simplecpp.cpp | 134 ++++++++++++++++++++++++------- testutils.py | 57 +++++++++++++ 6 files changed, 292 insertions(+), 30 deletions(-) create mode 100644 integration_test.py create mode 100644 testutils.py diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 7e962a47..c84fc052 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -30,7 +30,17 @@ jobs: run: | sudo apt-get update sudo apt-get install libc++-18-dev - + + - name: Install missing software on macos + if: contains(matrix.os, 'macos') + run: | + brew install python3 + + - name: Install missing Python packages + run: | + python3 -m pip config set global.break-system-packages true + python3 -m pip install pytest + - name: make simplecpp run: make -j$(nproc) @@ -41,6 +51,10 @@ jobs: run: | make -j$(nproc) selfcheck + - name: integration test + run: | + python3 -m pytest integration_test.py + - name: Run CMake run: | cmake -S . -B cmake.output diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 37ebf4fc..3e017182 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -26,7 +26,18 @@ jobs: - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 - + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + check-latest: true + + - name: Install missing Python packages + run: | + python -m pip install pip --upgrade || exit /b !errorlevel! + python -m pip install pytest || exit /b !errorlevel! + - name: Run CMake run: | cmake -G "Visual Studio 17 2022" -A x64 . || exit /b !errorlevel! @@ -42,4 +53,9 @@ jobs: - name: Selfcheck run: | .\${{ matrix.config }}\simplecpp.exe simplecpp.cpp -e || exit /b !errorlevel! + + - name: integration test + run: | + set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe + python -m pytest integration_test.py || exit /b !errorlevel! diff --git a/.gitignore b/.gitignore index 183545f1..113cf360 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ testrunner # CLion /.idea /cmake-build-* + +# python +__pycache__/ diff --git a/integration_test.py b/integration_test.py new file mode 100644 index 00000000..0b2b0b38 --- /dev/null +++ b/integration_test.py @@ -0,0 +1,94 @@ +## test with python -m pytest integration_test.py + +import os +import pytest +from testutils import simplecpp, format_include_path_arg, format_include + +def __test_relative_header_create_header(dir, with_pragma_once=True): + header_file = os.path.join(dir, 'test.h') + with open(header_file, 'wt') as f: + f.write(f""" + {"#pragma once" if with_pragma_once else ""} + #ifndef TEST_H_INCLUDED + #define TEST_H_INCLUDED + #else + #error header_was_already_included + #endif + """) + return header_file, "error: #error header_was_already_included" + +def __test_relative_header_create_source(dir, include1, include2, is_include1_sys=False, is_include2_sys=False, inv=False): + if inv: + return __test_relative_header_create_source(dir, include1=include2, include2=include1, is_include1_sys=is_include2_sys, is_include2_sys=is_include1_sys) + ## otherwise + + src_file = os.path.join(dir, 'test.c') + with open(src_file, 'wt') as f: + f.write(f""" + #undef TEST_H_INCLUDED + #include {format_include(include1, is_include1_sys)} + #include {format_include(include2, is_include2_sys)} + """) + return src_file + +@pytest.mark.parametrize("with_pragma_once", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +def test_relative_header_1(tmpdir, with_pragma_once, is_sys): + _, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) + + test_file = __test_relative_header_create_source(tmpdir, "test.h", "test.h", is_include1_sys=is_sys, is_include2_sys=is_sys) + + args = ([format_include_path_arg(tmpdir)] if is_sys else []) + [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + + if with_pragma_once: + assert stderr == '' + else: + assert double_include_error in stderr + +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_2(tmpdir, inv): + header_file, _ = __test_relative_header_create_header(tmpdir) + + test_file = __test_relative_header_create_source(tmpdir, "test.h", header_file, inv=inv) + + args = [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + assert stderr == '' + +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_3(tmpdir, is_sys, inv): + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + header_file, _ = __test_relative_header_create_header(test_subdir) + + test_file = __test_relative_header_create_source(tmpdir, "test_subdir/test.h", header_file, is_include1_sys=is_sys, inv=inv) + + args = [test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + + if is_sys: + assert "missing header: Header not found" in stderr + else: + assert stderr == '' + +@pytest.mark.parametrize("use_short_path", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_4(tmpdir, use_short_path, is_sys, inv): + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + header_file, _ = __test_relative_header_create_header(test_subdir) + if use_short_path: + header_file = "test_subdir/test.h" + + test_file = __test_relative_header_create_source(tmpdir, header_file, "test.h", is_include2_sys=is_sys, inv=inv) + + args = [format_include_path_arg(test_subdir), test_file] + + _, _, stderr = simplecpp(args, cwd=tmpdir) + assert stderr == '' diff --git a/simplecpp.cpp b/simplecpp.cpp index 2316c42b..25c4124a 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -40,6 +40,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #ifdef SIMPLECPP_WINDOWS #include #undef ERROR @@ -147,6 +153,12 @@ static unsigned long long stringToULL(const std::string &s) return ret; } +// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild +static bool startsWith_(const std::string &s, const std::string &p) +{ + return (s.size() >= p.size()) && std::equal(p.begin(), p.end(), s.begin()); +} + static bool endsWith(const std::string &s, const std::string &e) { return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); @@ -2334,17 +2346,12 @@ namespace simplecpp { namespace simplecpp { #ifdef __CYGWIN__ - bool startsWith(const std::string &str, const std::string &s) - { - return (str.size() >= s.size() && str.compare(0, s.size(), s) == 0); - } - std::string convertCygwinToWindowsPath(const std::string &cygwinPath) { std::string windowsPath; std::string::size_type pos = 0; - if (cygwinPath.size() >= 11 && startsWith(cygwinPath, "/cygdrive/")) { + if (cygwinPath.size() >= 11 && startsWith_(cygwinPath, "/cygdrive/")) { const unsigned char driveLetter = cygwinPath[10]; if (std::isalpha(driveLetter)) { if (cygwinPath.size() == 11) { @@ -2681,6 +2688,47 @@ static bool isCpp17OrLater(const simplecpp::DUI &dui) return !std_ver.empty() && (std_ver >= "201703L"); } + +static std::string currentDirectoryOSCalc() { + const std::size_t size = 4096; + char currentPath[size]; + +#ifndef _WIN32 + if (getcwd(currentPath, size) != nullptr) +#else + if (_getcwd(currentPath, size) != nullptr) +#endif + return std::string(currentPath); + + return ""; +} + +static const std::string& currentDirectory() { + static const std::string curdir = simplecpp::simplifyPath(currentDirectoryOSCalc()); + return curdir; +} + +static std::string toAbsolutePath(const std::string& path) { + if (path.empty()) { + return path;// preserve error file path that is indicated by an empty string + } + if (!isAbsolutePath(path)) { + return simplecpp::simplifyPath(currentDirectory() + "/" + path); + } + // otherwise + return simplecpp::simplifyPath(path); +} + +static std::pair extractRelativePathFromAbsolute(const std::string& absolutepath) { + static const std::string prefix = currentDirectory() + "/"; + if (startsWith_(absolutepath, prefix)) { + const std::size_t size = prefix.size(); + return std::make_pair(absolutepath.substr(size, absolutepath.size() - size), true); + } + // otherwise + return std::make_pair("", false); +} + static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { @@ -3099,9 +3147,12 @@ static std::string openHeader(std::ifstream &f, const std::string &path) static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) { + std::string path; if (sourcefile.find_first_of("\\/") != std::string::npos) - return simplecpp::simplifyPath(sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header); - return simplecpp::simplifyPath(header); + path = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + else + path = header; + return simplecpp::simplifyPath(path); } static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) @@ -3111,7 +3162,7 @@ static std::string openHeaderRelative(std::ifstream &f, const std::string &sourc static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) { - std::string path = includePath; + std::string path = toAbsolutePath(includePath); if (!path.empty() && path[path.size()-1U]!='/' && path[path.size()-1U]!='\\') path += '/'; return path + header; @@ -3120,9 +3171,9 @@ static std::string getIncludePathFileName(const std::string &includePath, const static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) { for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string simplePath = openHeader(f, getIncludePathFileName(*it, header)); - if (!simplePath.empty()) - return simplePath; + std::string path = openHeader(f, getIncludePathFileName(*it, header)); + if (!path.empty()) + return path; } return ""; } @@ -3132,49 +3183,76 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (isAbsolutePath(header)) return openHeader(f, header); - std::string ret; - if (systemheader) { - ret = openHeaderIncludePath(f, dui, header); - return ret; + // always return absolute path for systemheaders + return toAbsolutePath(openHeaderIncludePath(f, dui, header)); } + std::string ret; + ret = openHeaderRelative(f, sourcefile, header); if (ret.empty()) - return openHeaderIncludePath(f, dui, header); + return toAbsolutePath(openHeaderIncludePath(f, dui, header));// in a similar way to system headers return ret; } -static std::string getFileName(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +static std::string findPathInMapBothRelativeAndAbsolute(const std::map &filedata, const std::string& path) { + // here there are two possibilities - either we match this from absolute path or from a relative one + if (filedata.find(path) != filedata.end()) {// try first to respect the exact match + return path; + } + // otherwise - try to use the normalize to the correct representation + if (isAbsolutePath(path)) { + const std::pair relativeExtractedResult = extractRelativePathFromAbsolute(path); + if (relativeExtractedResult.second) { + const std::string relativePath = relativeExtractedResult.first; + if (filedata.find(relativePath) != filedata.end()) { + return relativePath; + } + } + } else { + const std::string absolutePath = toAbsolutePath(path); + if (filedata.find(absolutePath) != filedata.end()) + return absolutePath; + } + // otherwise + return ""; +} + +static std::string getFileIdPath(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { if (filedata.empty()) { return ""; } if (isAbsolutePath(header)) { - return (filedata.find(header) != filedata.end()) ? simplecpp::simplifyPath(header) : ""; + const std::string simplifiedHeaderPath = simplecpp::simplifyPath(header); + return (filedata.find(simplifiedHeaderPath) != filedata.end()) ? simplifiedHeaderPath : ""; } if (!systemheader) { - const std::string relativeFilename = getRelativeFileName(sourcefile, header); - if (filedata.find(relativeFilename) != filedata.end()) - return relativeFilename; + const std::string relativeOrAbsoluteFilename = getRelativeFileName(sourcefile, header);// unknown if absolute or relative, but always simplified + const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, relativeOrAbsoluteFilename); + if (!match.empty()) { + return match; + } } for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string s = simplecpp::simplifyPath(getIncludePathFileName(*it, header)); - if (filedata.find(s) != filedata.end()) - return s; + const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, simplecpp::simplifyPath(getIncludePathFileName(*it, header))); + if (!match.empty()) { + return match; + } } if (systemheader && filedata.find(header) != filedata.end()) - return header; + return header;// system header that its file wasn't found in the included paths but alreasy in the filedata - return this as is return ""; } static bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) { - return !getFileName(filedata, sourcefile, header, dui, systemheader).empty(); + return !getFileIdPath(filedata, sourcefile, header, dui, systemheader).empty(); } std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) @@ -3530,7 +3608,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool systemheader = (inctok->str()[0] == '<'); const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); - std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui, systemheader); + std::string header2 = getFileIdPath(filedata, rawtok->location.file(), header, dui, systemheader); if (header2.empty()) { // try to load file.. std::ifstream f; diff --git a/testutils.py b/testutils.py new file mode 100644 index 00000000..55a2686d --- /dev/null +++ b/testutils.py @@ -0,0 +1,57 @@ +import os +import subprocess +import json + +def __run_subprocess(args, env=None, cwd=None, timeout=None): + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd) + + try: + stdout, stderr = p.communicate(timeout=timeout) + return_code = p.returncode + p = None + except subprocess.TimeoutExpired: + import psutil + # terminate all the child processes + child_procs = psutil.Process(p.pid).children(recursive=True) + if len(child_procs) > 0: + for child in child_procs: + child.terminate() + try: + # call with timeout since it might be stuck + p.communicate(timeout=5) + p = None + except subprocess.TimeoutExpired: + pass + raise + finally: + if p: + # sending the signal to the process groups causes the parent Python process to terminate as well + #os.killpg(os.getpgid(p.pid), signal.SIGTERM) # Send the signal to all the process groups + p.terminate() + stdout, stderr = p.communicate() + p = None + + stdout = stdout.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') + stderr = stderr.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n') + + return return_code, stdout, stderr + +def simplecpp(args = [], cwd = None): + dir_path = os.path.dirname(os.path.realpath(__file__)) + if 'SIMPLECPP_EXE_PATH' in os.environ: + simplecpp_path = os.environ['SIMPLECPP_EXE_PATH'] + else: + simplecpp_path = os.path.join(dir_path, "simplecpp") + return __run_subprocess([simplecpp_path] + args, cwd = cwd) + +def quoted_string(s): + return json.dumps(str(s)) + +def format_include_path_arg(include_path): + return f"-I{str(include_path)}" + +def format_include(include, is_sys_header=False): + if is_sys_header: + return f"<{quoted_string(include)[1:-1]}>" + else: + return quoted_string(include) From 2bbb496fe2bdc558c05fb51ce3de0b54023f9007 Mon Sep 17 00:00:00 2001 From: Tal500 Date: Fri, 16 May 2025 14:18:04 +0300 Subject: [PATCH 261/381] Preserve relativeness of included paths (w.r.t. current directory) (#428) Co-authored-by: Tal Hadad --- integration_test.py | 24 ++++++++++++++----- simplecpp.cpp | 56 +++++++++++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/integration_test.py b/integration_test.py index 0b2b0b38..4fe0129b 100644 --- a/integration_test.py +++ b/integration_test.py @@ -1,6 +1,7 @@ ## test with python -m pytest integration_test.py import os +import pathlib import pytest from testutils import simplecpp, format_include_path_arg, format_include @@ -14,6 +15,7 @@ def __test_relative_header_create_header(dir, with_pragma_once=True): #else #error header_was_already_included #endif + const int dummy = 1; """) return header_file, "error: #error header_was_already_included" @@ -48,33 +50,43 @@ def test_relative_header_1(tmpdir, with_pragma_once, is_sys): assert double_include_error in stderr @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_2(tmpdir, inv): +@pytest.mark.parametrize("source_relative", (False, True)) +def test_relative_header_2(tmpdir, inv, source_relative): header_file, _ = __test_relative_header_create_header(tmpdir) test_file = __test_relative_header_create_source(tmpdir, "test.h", header_file, inv=inv) - args = [test_file] + args = ["test.c" if source_relative else test_file] - _, _, stderr = simplecpp(args, cwd=tmpdir) + _, stdout, stderr = simplecpp(args, cwd=tmpdir) assert stderr == '' + if source_relative and not inv: + assert '#line 8 "test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_3(tmpdir, is_sys, inv): +@pytest.mark.parametrize("source_relative", (False, True)) +def test_relative_header_3(tmpdir, is_sys, inv, source_relative): test_subdir = os.path.join(tmpdir, "test_subdir") os.mkdir(test_subdir) header_file, _ = __test_relative_header_create_header(test_subdir) test_file = __test_relative_header_create_source(tmpdir, "test_subdir/test.h", header_file, is_include1_sys=is_sys, inv=inv) - args = [test_file] + args = ["test.c" if source_relative else test_file] - _, _, stderr = simplecpp(args, cwd=tmpdir) + _, stdout, stderr = simplecpp(args, cwd=tmpdir) if is_sys: assert "missing header: Header not found" in stderr else: assert stderr == '' + if source_relative and not inv: + assert '#line 8 "test_subdir/test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout @pytest.mark.parametrize("use_short_path", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) diff --git a/simplecpp.cpp b/simplecpp.cpp index 25c4124a..d1fa91bf 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3145,11 +3145,11 @@ static std::string openHeader(std::ifstream &f, const std::string &path) return ""; } -static std::string getRelativeFileName(const std::string &sourcefile, const std::string &header) +static std::string getRelativeFileName(const std::string &baseFile, const std::string &header) { std::string path; - if (sourcefile.find_first_of("\\/") != std::string::npos) - path = sourcefile.substr(0, sourcefile.find_last_of("\\/") + 1U) + header; + if (baseFile.find_first_of("\\/") != std::string::npos) + path = baseFile.substr(0, baseFile.find_last_of("\\/") + 1U) + header; else path = header; return simplecpp::simplifyPath(path); @@ -3160,12 +3160,22 @@ static std::string openHeaderRelative(std::ifstream &f, const std::string &sourc return openHeader(f, getRelativeFileName(sourcefile, header)); } +// returns the simplified header path: +// * If the header path is absolute, returns it in absolute path +// * Otherwise, returns it in relative path with respect to the current directory static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) { - std::string path = toAbsolutePath(includePath); - if (!path.empty() && path[path.size()-1U]!='/' && path[path.size()-1U]!='\\') - path += '/'; - return path + header; + std::string simplifiedHeader = simplecpp::simplifyPath(header); + + if (isAbsolutePath(simplifiedHeader)) { + return simplifiedHeader; + } + + std::string basePath = toAbsolutePath(includePath); + if (!basePath.empty() && basePath[basePath.size()-1U]!='/' && basePath[basePath.size()-1U]!='\\') + basePath += '/'; + const std::string absolutesimplifiedHeaderPath = basePath + simplifiedHeader; + return extractRelativePathFromAbsolute(absolutesimplifiedHeaderPath).first; } static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) @@ -3183,17 +3193,16 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const if (isAbsolutePath(header)) return openHeader(f, header); - if (systemheader) { - // always return absolute path for systemheaders - return toAbsolutePath(openHeaderIncludePath(f, dui, header)); + // prefer first to search the header relatively to source file if found, when not a system header + if (!systemheader) { + std::string relativeHeader = openHeaderRelative(f, sourcefile, header); + if (!relativeHeader.empty()) { + return relativeHeader; + } } - std::string ret; - - ret = openHeaderRelative(f, sourcefile, header); - if (ret.empty()) - return toAbsolutePath(openHeaderIncludePath(f, dui, header));// in a similar way to system headers - return ret; + // search the header on the include paths (provided by the flags "-I...") + return openHeaderIncludePath(f, dui, header); } static std::string findPathInMapBothRelativeAndAbsolute(const std::map &filedata, const std::string& path) { @@ -3212,8 +3221,9 @@ static std::string findPathInMapBothRelativeAndAbsolute(const std::map::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, simplecpp::simplifyPath(getIncludePathFileName(*it, header))); + const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, getIncludePathFileName(*it, header)); if (!match.empty()) { return match; } } - if (systemheader && filedata.find(header) != filedata.end()) - return header;// system header that its file wasn't found in the included paths but alreasy in the filedata - return this as is - return ""; } From 76df97f64b2906d01587b379511c2d9836568c87 Mon Sep 17 00:00:00 2001 From: Tal500 Date: Fri, 30 May 2025 17:44:43 +0300 Subject: [PATCH 262/381] fix: parent relative paths, and rework on the whole path extraction mechanics (#429) --- .github/workflows/CI-unixish.yml | 2 +- .github/workflows/CI-windows.yml | 2 +- integration_test.py | 103 ++++++++++++++++++++++++++----- simplecpp.cpp | 98 ++++++++++++++++++++--------- 4 files changed, 159 insertions(+), 46 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index c84fc052..f5a78ea5 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -53,7 +53,7 @@ jobs: - name: integration test run: | - python3 -m pytest integration_test.py + python3 -m pytest integration_test.py -vv - name: Run CMake run: | diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 3e017182..971f3827 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -57,5 +57,5 @@ jobs: - name: integration test run: | set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe - python -m pytest integration_test.py || exit /b !errorlevel! + python -m pytest integration_test.py -vv || exit /b !errorlevel! diff --git a/integration_test.py b/integration_test.py index 4fe0129b..122ce9aa 100644 --- a/integration_test.py +++ b/integration_test.py @@ -35,40 +35,48 @@ def __test_relative_header_create_source(dir, include1, include2, is_include1_sy @pytest.mark.parametrize("with_pragma_once", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) -def test_relative_header_1(tmpdir, with_pragma_once, is_sys): +def test_relative_header_1(record_property, tmpdir, with_pragma_once, is_sys): _, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) test_file = __test_relative_header_create_source(tmpdir, "test.h", "test.h", is_include1_sys=is_sys, is_include2_sys=is_sys) args = ([format_include_path_arg(tmpdir)] if is_sys else []) + [test_file] - _, _, stderr = simplecpp(args, cwd=tmpdir) + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) if with_pragma_once: assert stderr == '' else: assert double_include_error in stderr +@pytest.mark.parametrize("with_pragma_once", (False, True)) @pytest.mark.parametrize("inv", (False, True)) @pytest.mark.parametrize("source_relative", (False, True)) -def test_relative_header_2(tmpdir, inv, source_relative): - header_file, _ = __test_relative_header_create_header(tmpdir) +def test_relative_header_2(record_property, tmpdir, with_pragma_once, inv, source_relative): + header_file, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) test_file = __test_relative_header_create_source(tmpdir, "test.h", header_file, inv=inv) args = ["test.c" if source_relative else test_file] _, stdout, stderr = simplecpp(args, cwd=tmpdir) - assert stderr == '' - if source_relative and not inv: - assert '#line 8 "test.h"' in stdout + record_property("stdout", stdout) + record_property("stderr", stderr) + if with_pragma_once: + assert stderr == '' + if inv: + assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout + else: + assert '#line 8 "test.h"' in stdout else: - assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout + assert double_include_error in stderr @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) @pytest.mark.parametrize("source_relative", (False, True)) -def test_relative_header_3(tmpdir, is_sys, inv, source_relative): +def test_relative_header_3(record_property, tmpdir, is_sys, inv, source_relative): test_subdir = os.path.join(tmpdir, "test_subdir") os.mkdir(test_subdir) header_file, _ = __test_relative_header_create_header(test_subdir) @@ -78,20 +86,23 @@ def test_relative_header_3(tmpdir, is_sys, inv, source_relative): args = ["test.c" if source_relative else test_file] _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) if is_sys: assert "missing header: Header not found" in stderr else: assert stderr == '' - if source_relative and not inv: - assert '#line 8 "test_subdir/test.h"' in stdout - else: + if inv: assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout + else: + assert '#line 8 "test_subdir/test.h"' in stdout @pytest.mark.parametrize("use_short_path", (False, True)) +@pytest.mark.parametrize("relative_include_dir", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_4(tmpdir, use_short_path, is_sys, inv): +def test_relative_header_4(record_property, tmpdir, use_short_path, relative_include_dir, is_sys, inv): test_subdir = os.path.join(tmpdir, "test_subdir") os.mkdir(test_subdir) header_file, _ = __test_relative_header_create_header(test_subdir) @@ -100,7 +111,69 @@ def test_relative_header_4(tmpdir, use_short_path, is_sys, inv): test_file = __test_relative_header_create_source(tmpdir, header_file, "test.h", is_include2_sys=is_sys, inv=inv) - args = [format_include_path_arg(test_subdir), test_file] + args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), test_file] - _, _, stderr = simplecpp(args, cwd=tmpdir) + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) assert stderr == '' + if (use_short_path and not inv) or (relative_include_dir and inv): + assert '#line 8 "test_subdir/test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout + +@pytest.mark.parametrize("with_pragma_once", (False, True)) +@pytest.mark.parametrize("relative_include_dir", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_5(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv): # test relative paths with .. + ## in this test, the subdir role is the opposite then the previous - it contains the test.c file, while the parent tmpdir contains the header file + header_file, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) + if is_sys: + header_file_second_path = "test.h" + else: + header_file_second_path = "../test.h" + + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + test_file = __test_relative_header_create_source(test_subdir, header_file, header_file_second_path, is_include2_sys=is_sys, inv=inv) + + args = ([format_include_path_arg(".." if relative_include_dir else tmpdir)] if is_sys else []) + ["test.c"] + + _, stdout, stderr = simplecpp(args, cwd=test_subdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + if with_pragma_once: + assert stderr == '' + if (relative_include_dir or not is_sys) and inv: + assert '#line 8 "../test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout + else: + assert double_include_error in stderr + +@pytest.mark.parametrize("with_pragma_once", (False, True)) +@pytest.mark.parametrize("relative_include_dir", (False, True)) +@pytest.mark.parametrize("is_sys", (False, True)) +@pytest.mark.parametrize("inv", (False, True)) +def test_relative_header_6(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv): # test relative paths with .. that is resolved only by an include dir + ## in this test, both the header and the source file are at the same dir, but there is a dummy inclusion dir as a subdir + header_file, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) + + test_subdir = os.path.join(tmpdir, "test_subdir") + os.mkdir(test_subdir) + test_file = __test_relative_header_create_source(tmpdir, header_file, "../test.h", is_include2_sys=is_sys, inv=inv) + + args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), "test.c"] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + if with_pragma_once: + assert stderr == '' + if relative_include_dir and inv: + assert '#line 8 "test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout + else: + assert double_include_error in stderr diff --git a/simplecpp.cpp b/simplecpp.cpp index d1fa91bf..9ff66d8b 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2719,14 +2719,42 @@ static std::string toAbsolutePath(const std::string& path) { return simplecpp::simplifyPath(path); } -static std::pair extractRelativePathFromAbsolute(const std::string& absolutepath) { - static const std::string prefix = currentDirectory() + "/"; - if (startsWith_(absolutepath, prefix)) { - const std::size_t size = prefix.size(); - return std::make_pair(absolutepath.substr(size, absolutepath.size() - size), true); +static std::string dirPath(const std::string& path, bool withTrailingSlash=true) { + const std::size_t lastSlash = path.find_last_of("\\/"); + if (lastSlash == std::string::npos) { + return ""; } - // otherwise - return std::make_pair("", false); + return path.substr(0, lastSlash + (withTrailingSlash ? 1U : 0U)); +} + +static std::string omitPathTrailingSlash(const std::string& path) { + if (endsWith(path, "/")) { + return path.substr(0, path.size() - 1U); + } + return path; +} + +static std::string extractRelativePathFromAbsolute(const std::string& absoluteSimplifiedPath, const std::string& prefixSimplifiedAbsoluteDir = currentDirectory()) { + const std::string normalizedAbsolutePath = omitPathTrailingSlash(absoluteSimplifiedPath); + std::string currentPrefix = omitPathTrailingSlash(prefixSimplifiedAbsoluteDir); + std::string leadingParenting; + while (!startsWith_(normalizedAbsolutePath, currentPrefix)) { + leadingParenting = "../" + leadingParenting; + currentPrefix = dirPath(currentPrefix, false); + } + const std::size_t size = currentPrefix.size(); + std::string relativeFromMeetingPath = normalizedAbsolutePath.substr(size, normalizedAbsolutePath.size() - size); + if (currentPrefix.empty() && !(startsWith_(absoluteSimplifiedPath, "/") && startsWith_(prefixSimplifiedAbsoluteDir, "/"))) { + // In the case that there is no common prefix path, + // and at not both of the paths start with `/` (can happen only in Windows paths on distinct partitions), + // return the absolute simplified path as is because no relative path can match. + return absoluteSimplifiedPath; + } + if (startsWith_(relativeFromMeetingPath, "/")) { + // omit the leading slash + relativeFromMeetingPath = relativeFromMeetingPath.substr(1, relativeFromMeetingPath.size()); + } + return leadingParenting + relativeFromMeetingPath; } static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); @@ -3147,12 +3175,17 @@ static std::string openHeader(std::ifstream &f, const std::string &path) static std::string getRelativeFileName(const std::string &baseFile, const std::string &header) { - std::string path; - if (baseFile.find_first_of("\\/") != std::string::npos) - path = baseFile.substr(0, baseFile.find_last_of("\\/") + 1U) + header; - else - path = header; - return simplecpp::simplifyPath(path); + const std::string baseFileSimplified = simplecpp::simplifyPath(baseFile); + const std::string baseFileAbsolute = isAbsolutePath(baseFileSimplified) ? + baseFileSimplified : + simplecpp::simplifyPath(currentDirectory() + "/" + baseFileSimplified); + + const std::string headerSimplified = simplecpp::simplifyPath(header); + const std::string path = isAbsolutePath(headerSimplified) ? + headerSimplified : + simplecpp::simplifyPath(dirPath(baseFileAbsolute) + headerSimplified); + + return extractRelativePathFromAbsolute(path); } static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) @@ -3174,8 +3207,9 @@ static std::string getIncludePathFileName(const std::string &includePath, const std::string basePath = toAbsolutePath(includePath); if (!basePath.empty() && basePath[basePath.size()-1U]!='/' && basePath[basePath.size()-1U]!='\\') basePath += '/'; - const std::string absolutesimplifiedHeaderPath = basePath + simplifiedHeader; - return extractRelativePathFromAbsolute(absolutesimplifiedHeaderPath).first; + const std::string absoluteSimplifiedHeaderPath = simplecpp::simplifyPath(basePath + simplifiedHeader); + // preserve absoluteness/relativieness of the including dir + return isAbsolutePath(includePath) ? absoluteSimplifiedHeaderPath : extractRelativePathFromAbsolute(absoluteSimplifiedHeaderPath); } static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) @@ -3210,22 +3244,18 @@ static std::string findPathInMapBothRelativeAndAbsolute(const std::map relativeExtractedResult = extractRelativePathFromAbsolute(path); - if (relativeExtractedResult.second) { - const std::string relativePath = relativeExtractedResult.first; - if (filedata.find(relativePath) != filedata.end()) { - return relativePath; - } - } + alternativePath = extractRelativePathFromAbsolute(simplecpp::simplifyPath(path)); } else { - const std::string absolutePath = toAbsolutePath(path); - if (filedata.find(absolutePath) != filedata.end()) { - return absolutePath; - } + alternativePath = toAbsolutePath(path); + } + + if (filedata.find(alternativePath) != filedata.end()) { + return alternativePath; } - // otherwise return ""; } @@ -3267,6 +3297,16 @@ static bool hasFile(const std::map &filedat return !getFileIdPath(filedata, sourcefile, header, dui, systemheader).empty(); } +static void safeInsertTokenListToMap(std::map &filedata, const std::string &header2, simplecpp::TokenList *tokens, const std::string &header, const std::string &sourcefile, bool systemheader, const char* contextDesc) +{ + const bool inserted = filedata.insert(std::make_pair(header2, tokens)).second; + if (!inserted) { + std::cerr << "error in " << contextDesc << " - attempt to add a tokenized file to the file map, but this file is already in the map! Details:" << + "header: " << header << " header2: " << header2 << " source: " << sourcefile << " systemheader: " << systemheader << std::endl; + std::abort(); + } +} + std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { #ifdef SIMPLECPP_WINDOWS @@ -3343,7 +3383,7 @@ std::map simplecpp::load(const simplecpp::To TokenList *tokens = new TokenList(header2, filenames, outputList); if (dui.removeComments) tokens->removeComments(); - ret[header2] = tokens; + safeInsertTokenListToMap(ret, header2, tokens, header, rawtok->location.file(), systemheader, "simplecpp::load"); if (tokens->front()) filelist.push_back(tokens->front()); } @@ -3630,7 +3670,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL TokenList * const tokens = new TokenList(header2, files, outputList); if (dui.removeComments) tokens->removeComments(); - filedata[header2] = tokens; + safeInsertTokenListToMap(filedata, header2, tokens, header, rawtok->location.file(), systemheader, "simplecpp::preprocess"); } } if (header2.empty()) { From 9ba39709e34a8ccaee5c7afa0d87dd478eaba959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 12 Jun 2025 11:23:30 +0200 Subject: [PATCH 263/381] aligned GCC warnings with Cppcheck (#434) --- CMakeLists.txt | 19 ++++++++++++++++++- test.cpp | 6 +++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3fcf4ba..672e63bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,24 @@ function(add_compile_options_safe FLAG) endfunction() if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - add_compile_options(-Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wshadow -Wundef -Wold-style-cast -Wno-multichar) + add_compile_options(-pedantic) + add_compile_options(-Wall) + add_compile_options(-Wextra) + add_compile_options(-Wcast-qual) # Cast for removing type qualifiers + add_compile_options(-Wfloat-equal) # Floating values used in equality comparisons + add_compile_options(-Wmissing-declarations) # If a global function is defined without a previous declaration + add_compile_options(-Wmissing-format-attribute) # + add_compile_options(-Wno-long-long) + add_compile_options(-Wpacked) # + add_compile_options(-Wredundant-decls) # if anything is declared more than once in the same scope + add_compile_options(-Wundef) + add_compile_options(-Wno-missing-braces) + add_compile_options(-Wno-sign-compare) + add_compile_options(-Wno-multichar) + add_compile_options(-Woverloaded-virtual) # when a function declaration hides virtual functions from a base class + + add_compile_options(-Wsuggest-attribute=noreturn) + add_compile_options_safe(-Wuseless-cast) elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") add_compile_definitions(_CRT_SECURE_NO_WARNINGS) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/test.cpp b/test.cpp index 187b7ec6..86e6ddc8 100644 --- a/test.cpp +++ b/test.cpp @@ -279,9 +279,9 @@ static void characterLiteral() #ifdef __GNUC__ // BEGIN Implementation-specific results - ASSERT_EQUALS(static_cast('AB'), simplecpp::characterLiteralToLL("'AB'")); - ASSERT_EQUALS(static_cast('ABC'), simplecpp::characterLiteralToLL("'ABC'")); - ASSERT_EQUALS(static_cast('ABCD'), simplecpp::characterLiteralToLL("'ABCD'")); + ASSERT_EQUALS('AB', simplecpp::characterLiteralToLL("'AB'")); + ASSERT_EQUALS('ABC', simplecpp::characterLiteralToLL("'ABC'")); + ASSERT_EQUALS('ABCD', simplecpp::characterLiteralToLL("'ABCD'")); ASSERT_EQUALS('\134t', simplecpp::characterLiteralToLL("'\\134t'")); // cppcheck ticket #7452 // END Implementation-specific results #endif From 6cc4f53f4c7ff2adebec67ef21795260355478c7 Mon Sep 17 00:00:00 2001 From: glankk Date: Thu, 12 Jun 2025 17:23:14 +0200 Subject: [PATCH 264/381] Add encoding to open() in run-tests.py (#440) --- run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.py b/run-tests.py index 2f28bf0f..5017f4c1 100644 --- a/run-tests.py +++ b/run-tests.py @@ -16,7 +16,7 @@ def cleanup(out): commands = [] for f in sorted(glob.glob(os.path.expanduser('testsuite/clang-preprocessor-tests/*.c*'))): - for line in open(f, 'rt'): + for line in open(f, 'rt', encoding='utf-8'): if line.startswith('// RUN: %clang_cc1 '): cmd = '' for arg in line[19:].split(): From eb18d11f0fc2aa70330dc5592629dc06ed0213f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 13 Jun 2025 22:26:46 +0200 Subject: [PATCH 265/381] main.cpp: added option `-f` to fail on output error (#436) --- main.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.cpp b/main.cpp index f05cf793..424ef6fa 100644 --- a/main.cpp +++ b/main.cpp @@ -18,6 +18,7 @@ int main(int argc, char **argv) bool error = false; const char *filename = nullptr; bool use_istream = false; + bool fail_on_error = false; // Settings.. simplecpp::DUI dui; @@ -70,6 +71,10 @@ int main(int argc, char **argv) error_only = true; found = true; break; + case 'f': + fail_on_error = true; + found = true; + break; } if (!found) { std::cout << "error: option '" << arg << "' is unknown." << std::endl; @@ -172,5 +177,8 @@ int main(int argc, char **argv) } } + if (fail_on_error && !outputList.empty()) + return 1; + return 0; } From a46cb125243f4d6e743eceeba37a485ad131d420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 13 Jun 2025 22:33:42 +0200 Subject: [PATCH 266/381] selfcheck.sh: actually fail if we encountered unexpected errors (#437) --- selfcheck.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/selfcheck.sh b/selfcheck.sh index 3518c654..a43ef8c5 100755 --- a/selfcheck.sh +++ b/selfcheck.sh @@ -1,6 +1,11 @@ #!/bin/sh -output=$(./simplecpp simplecpp.cpp -e 2>&1) +output=$(./simplecpp simplecpp.cpp -e -f 2>&1) ec=$? -echo "$output" | grep -v 'Header not found: <' -exit $ec \ No newline at end of file +errors=$(echo "$output" | grep -v 'Header not found: <') +if [ $ec -ne 0 ]; then + # only fail if got errors which do not refer to missing system includes + if [ ! -z "$errors" ]; then + exit $ec + fi +fi \ No newline at end of file From 8e5c5f49ca66c053c9ab5bd699b7517b5b796536 Mon Sep 17 00:00:00 2001 From: glankk Date: Mon, 16 Jun 2025 13:59:17 +0200 Subject: [PATCH 267/381] Fix #381 (When there are header files with the same name, the correct file cannot be found) (#443) --- integration_test.py | 111 ++++++++++++++++++++++++++++++++++++++++++++ simplecpp.cpp | 7 +++ test.cpp | 2 + 3 files changed, 120 insertions(+) diff --git a/integration_test.py b/integration_test.py index 122ce9aa..ccef84eb 100644 --- a/integration_test.py +++ b/integration_test.py @@ -2,6 +2,7 @@ import os import pathlib +import platform import pytest from testutils import simplecpp, format_include_path_arg, format_include @@ -177,3 +178,113 @@ def test_relative_header_6(record_property, tmpdir, with_pragma_once, relative_i assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout else: assert double_include_error in stderr + +def test_same_name_header(record_property, tmpdir): + include_a = os.path.join(tmpdir, "include_a") + include_b = os.path.join(tmpdir, "include_b") + + test_file = os.path.join(tmpdir, "test.c") + header_a = os.path.join(include_a, "header_a.h") + header_b = os.path.join(include_b, "header_b.h") + same_name_a = os.path.join(include_a, "same_name.h") + same_name_b = os.path.join(include_b, "same_name.h") + + os.mkdir(include_a) + os.mkdir(include_b) + + with open(test_file, "wt") as f: + f.write(""" + #include + #include + TEST + """) + + with open(header_a, "wt") as f: + f.write(""" + #include "same_name.h" + """) + + with open(header_b, "wt") as f: + f.write(""" + #include "same_name.h" + """) + + with open(same_name_a, "wt") as f: + f.write(""" + #define TEST E + """) + + with open(same_name_b, "wt") as f: + f.write(""" + #define TEST OK + """) + + args = [ + format_include_path_arg(include_a), + format_include_path_arg(include_b), + test_file + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert "OK" in stdout + assert stderr == "" + +def test_pragma_once_matching(record_property, tmpdir): + if platform.system() == "win32": + names_to_test = [ + '"once.h"', + '"Once.h"', + '', + '', + '"../test_dir/once.h"', + '"../test_dir/Once.h"', + '"../Test_Dir/once.h"', + '"../Test_Dir/Once.h"', + '"test_subdir/../once.h"', + '"test_subdir/../Once.h"', + '"Test_Subdir/../once.h"', + '"Test_Subdir/../Once.h"', + ] + else: + names_to_test = [ + '"once.h"', + '', + '"../test_dir/once.h"', + '"test_subdir/../once.h"', + ] + + test_dir = os.path.join(tmpdir, "test_dir") + test_subdir = os.path.join(test_dir, "test_subdir") + + test_file = os.path.join(test_dir, "test.c") + once_header = os.path.join(test_dir, "once.h") + + os.mkdir(test_dir) + os.mkdir(test_subdir) + + with open(test_file, "wt") as f: + for n in names_to_test: + f.write(f""" + #include {n} + """); + + with open(once_header, "wt") as f: + f.write(f""" + #pragma once + ONCE + """); + + args = [ + format_include_path_arg(test_dir), + test_file + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert stdout.count("ONCE") == 1 + assert stderr == "" diff --git a/simplecpp.cpp b/simplecpp.cpp index 9ff66d8b..e4f98bb6 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3278,6 +3278,13 @@ static std::string getFileIdPath(const std::map Date: Fri, 20 Jun 2025 21:33:36 +0200 Subject: [PATCH 268/381] Fix #446 (change the rules of relativeness preserving to depend on the source file including it for relative path includes) (#445) --- integration_test.py | 28 ++++++++++++++++------------ simplecpp.cpp | 12 ++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/integration_test.py b/integration_test.py index ccef84eb..27528e16 100644 --- a/integration_test.py +++ b/integration_test.py @@ -67,7 +67,7 @@ def test_relative_header_2(record_property, tmpdir, with_pragma_once, inv, sourc record_property("stderr", stderr) if with_pragma_once: assert stderr == '' - if inv: + if inv or not source_relative: assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout else: assert '#line 8 "test.h"' in stdout @@ -94,16 +94,17 @@ def test_relative_header_3(record_property, tmpdir, is_sys, inv, source_relative assert "missing header: Header not found" in stderr else: assert stderr == '' - if inv: - assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout - else: + if source_relative and not inv: assert '#line 8 "test_subdir/test.h"' in stdout + else: + assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout @pytest.mark.parametrize("use_short_path", (False, True)) @pytest.mark.parametrize("relative_include_dir", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_4(record_property, tmpdir, use_short_path, relative_include_dir, is_sys, inv): +@pytest.mark.parametrize("source_relative", (False, True)) +def test_relative_header_4(record_property, tmpdir, use_short_path, relative_include_dir, is_sys, inv, source_relative): test_subdir = os.path.join(tmpdir, "test_subdir") os.mkdir(test_subdir) header_file, _ = __test_relative_header_create_header(test_subdir) @@ -112,13 +113,14 @@ def test_relative_header_4(record_property, tmpdir, use_short_path, relative_inc test_file = __test_relative_header_create_source(tmpdir, header_file, "test.h", is_include2_sys=is_sys, inv=inv) - args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), test_file] + args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), "test.c" if source_relative else test_file] _, stdout, stderr = simplecpp(args, cwd=tmpdir) record_property("stdout", stdout) record_property("stderr", stderr) + assert stderr == '' - if (use_short_path and not inv) or (relative_include_dir and inv): + if (source_relative and use_short_path and not inv) or (relative_include_dir and inv): assert '#line 8 "test_subdir/test.h"' in stdout else: assert f'#line 8 "{pathlib.PurePath(test_subdir).as_posix()}/test.h"' in stdout @@ -127,7 +129,8 @@ def test_relative_header_4(record_property, tmpdir, use_short_path, relative_inc @pytest.mark.parametrize("relative_include_dir", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_5(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv): # test relative paths with .. +@pytest.mark.parametrize("source_relative", (False, True)) +def test_relative_header_5(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv, source_relative): # test relative paths with .. ## in this test, the subdir role is the opposite then the previous - it contains the test.c file, while the parent tmpdir contains the header file header_file, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) if is_sys: @@ -139,14 +142,14 @@ def test_relative_header_5(record_property, tmpdir, with_pragma_once, relative_i os.mkdir(test_subdir) test_file = __test_relative_header_create_source(test_subdir, header_file, header_file_second_path, is_include2_sys=is_sys, inv=inv) - args = ([format_include_path_arg(".." if relative_include_dir else tmpdir)] if is_sys else []) + ["test.c"] + args = ([format_include_path_arg(".." if relative_include_dir else tmpdir)] if is_sys else []) + ["test.c" if source_relative else test_file] _, stdout, stderr = simplecpp(args, cwd=test_subdir) record_property("stdout", stdout) record_property("stderr", stderr) if with_pragma_once: assert stderr == '' - if (relative_include_dir or not is_sys) and inv: + if (relative_include_dir if is_sys else source_relative) and inv: assert '#line 8 "../test.h"' in stdout else: assert f'#line 8 "{pathlib.PurePath(tmpdir).as_posix()}/test.h"' in stdout @@ -157,7 +160,8 @@ def test_relative_header_5(record_property, tmpdir, with_pragma_once, relative_i @pytest.mark.parametrize("relative_include_dir", (False, True)) @pytest.mark.parametrize("is_sys", (False, True)) @pytest.mark.parametrize("inv", (False, True)) -def test_relative_header_6(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv): # test relative paths with .. that is resolved only by an include dir +@pytest.mark.parametrize("source_relative", (False, True)) +def test_relative_header_6(record_property, tmpdir, with_pragma_once, relative_include_dir, is_sys, inv, source_relative): # test relative paths with .. that is resolved only by an include dir ## in this test, both the header and the source file are at the same dir, but there is a dummy inclusion dir as a subdir header_file, double_include_error = __test_relative_header_create_header(tmpdir, with_pragma_once=with_pragma_once) @@ -165,7 +169,7 @@ def test_relative_header_6(record_property, tmpdir, with_pragma_once, relative_i os.mkdir(test_subdir) test_file = __test_relative_header_create_source(tmpdir, header_file, "../test.h", is_include2_sys=is_sys, inv=inv) - args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), "test.c"] + args = [format_include_path_arg("test_subdir" if relative_include_dir else test_subdir), "test.c" if source_relative else test_file] _, stdout, stderr = simplecpp(args, cwd=tmpdir) record_property("stdout", stdout) diff --git a/simplecpp.cpp b/simplecpp.cpp index e4f98bb6..599ffdfe 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3173,7 +3173,7 @@ static std::string openHeader(std::ifstream &f, const std::string &path) return ""; } -static std::string getRelativeFileName(const std::string &baseFile, const std::string &header) +static std::string getRelativeFileName(const std::string &baseFile, const std::string &header, bool returnAbsolutePath) { const std::string baseFileSimplified = simplecpp::simplifyPath(baseFile); const std::string baseFileAbsolute = isAbsolutePath(baseFileSimplified) ? @@ -3185,12 +3185,12 @@ static std::string getRelativeFileName(const std::string &baseFile, const std::s headerSimplified : simplecpp::simplifyPath(dirPath(baseFileAbsolute) + headerSimplified); - return extractRelativePathFromAbsolute(path); + return returnAbsolutePath ? toAbsolutePath(path) : extractRelativePathFromAbsolute(path); } static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) { - return openHeader(f, getRelativeFileName(sourcefile, header)); + return openHeader(f, getRelativeFileName(sourcefile, header, isAbsolutePath(sourcefile))); } // returns the simplified header path: @@ -3273,14 +3273,14 @@ static std::string getFileIdPath(const std::map Date: Tue, 1 Jul 2025 16:32:07 +0200 Subject: [PATCH 269/381] Fix #368 (__VA_OPT__ is not handled good enough) (#451) --- simplecpp.cpp | 111 ++++++++++++++---- test.cpp | 51 +++++++- .../macro_fn_va_opt.c | 13 ++ 3 files changed, 148 insertions(+), 27 deletions(-) create mode 100644 testsuite/clang-preprocessor-tests/macro_fn_va_opt.c diff --git a/simplecpp.cpp b/simplecpp.cpp index 599ffdfe..97657d60 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1485,7 +1485,7 @@ namespace simplecpp { class Macro { public: - explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), valueDefinedInCode_(false) {} + explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), optExpandValue(nullptr), optNoExpandValue(nullptr), valueDefinedInCode_(false) {} Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previousSkipComments(), tok)) @@ -1515,6 +1515,11 @@ namespace simplecpp { *this = other; } + ~Macro() { + delete optExpandValue; + delete optNoExpandValue; + } + Macro &operator=(const Macro &other) { if (this != &other) { files = other.files; @@ -1707,6 +1712,9 @@ namespace simplecpp { bool parseDefine(const Token *nametoken) { nameTokDef = nametoken; variadic = false; + variadicOpt = false; + optExpandValue = nullptr; + optNoExpandValue = nullptr; if (!nameTokDef) { valueToken = endToken = nullptr; args.clear(); @@ -1744,8 +1752,54 @@ namespace simplecpp { if (!sameline(valueToken, nameTokDef)) valueToken = nullptr; endToken = valueToken; - while (sameline(endToken, nameTokDef)) + while (sameline(endToken, nameTokDef)) { + if (variadic && endToken->str() == "__VA_OPT__") + variadicOpt = true; endToken = endToken->next; + } + + if (variadicOpt) { + TokenList expandValue(files); + TokenList noExpandValue(files); + for (const Token *tok = valueToken; tok && tok != endToken;) { + if (tok->str() == "__VA_OPT__") { + if (!sameline(tok, tok->next) || tok->next->op != '(') + throw Error(tok->location, "In definition of '" + nameTokDef->str() + "': Missing opening parenthesis for __VA_OPT__"); + tok = tok->next->next; + int par = 1; + while (tok && tok != endToken) { + if (tok->op == '(') + par++; + else if (tok->op == ')') + par--; + else if (tok->str() == "__VA_OPT__") + throw Error(tok->location, "In definition of '" + nameTokDef->str() + "': __VA_OPT__ cannot be nested"); + if (par == 0) { + tok = tok->next; + break; + } + expandValue.push_back(new Token(*tok)); + tok = tok->next; + } + if (par != 0) { + const Token *const lastTok = expandValue.back() ? expandValue.back() : valueToken->next; + throw Error(lastTok->location, "In definition of '" + nameTokDef->str() + "': Missing closing parenthesis for __VA_OPT__"); + } + } else { + expandValue.push_back(new Token(*tok)); + noExpandValue.push_back(new Token(*tok)); + tok = tok->next; + } + } +#if __cplusplus >= 201103L + optExpandValue = new TokenList(std::move(expandValue)); + optNoExpandValue = new TokenList(std::move(noExpandValue)); +#else + optExpandValue = new TokenList(expandValue); + optNoExpandValue = new TokenList(noExpandValue); +#endif + } + return true; } @@ -1900,8 +1954,22 @@ namespace simplecpp { Token * const output_end_1 = output->back(); + const Token *valueToken2; + const Token *endToken2; + + if (variadicOpt) { + if (parametertokens2.size() > args.size() && parametertokens2[args.size() - 1]->next->op != ')') + valueToken2 = optExpandValue->cfront(); + else + valueToken2 = optNoExpandValue->cfront(); + endToken2 = nullptr; + } else { + valueToken2 = valueToken; + endToken2 = endToken; + } + // expand - for (const Token *tok = valueToken; tok != endToken;) { + for (const Token *tok = valueToken2; tok != endToken2;) { if (tok->op != '#') { // A##B => AB if (sameline(tok, tok->next) && tok->next && tok->next->op == '#' && tok->next->next && tok->next->next->op == '#') { @@ -1950,7 +2018,7 @@ namespace simplecpp { } tok = tok->next; - if (tok == endToken) { + if (tok == endToken2) { output->push_back(new Token(*tok->previous)); break; } @@ -2020,24 +2088,6 @@ namespace simplecpp { // Macro parameter.. { TokenList temp(files); - if (tok->str() == "__VA_OPT__") { - if (sameline(tok, tok->next) && tok->next->str() == "(") { - tok = tok->next; - int paren = 1; - while (sameline(tok, tok->next)) { - if (tok->next->str() == "(") - ++paren; - else if (tok->next->str() == ")") - --paren; - if (paren == 0) - return tok->next->next; - tok = tok->next; - if (parametertokens.size() > args.size() && parametertokens.front()->next->str() != ")") - tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens)->previous; - } - } - throw Error(tok->location, "Missing parenthesis for __VA_OPT__(content)"); - } if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) { if (tok->str() == "__VA_ARGS__" && temp.empty() && output->cback() && output->cback()->str() == "," && tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") @@ -2338,6 +2388,13 @@ namespace simplecpp { /** is macro variadic? */ bool variadic; + /** does the macro expansion have __VA_OPT__? */ + bool variadicOpt; + + /** Expansion value for varadic macros with __VA_OPT__ expanded and discarded respectively */ + const TokenList *optExpandValue; + const TokenList *optNoExpandValue; + /** was the value of this macro actually defined in the code? */ bool valueDefinedInCode_; }; @@ -3621,6 +3678,16 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } output.clear(); return; + } catch (simplecpp::Macro::Error &err) { + if (outputList) { + simplecpp::Output out(files); + out.type = simplecpp::Output::SYNTAX_ERROR; + out.location = err.location; + out.msg = "Failed to parse #define, " + err.what; + outputList->push_back(out); + } + output.clear(); + return; } } else if (ifstates.top() == True && rawtok->str() == INCLUDE) { TokenList inc1(files); diff --git a/test.cpp b/test.cpp index cec253b8..caa6137e 100644 --- a/test.cpp +++ b/test.cpp @@ -923,7 +923,7 @@ static void define_va_opt_3() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code1, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing closing parenthesis for __VA_OPT__\n", toString(outputList)); outputList.clear(); @@ -934,7 +934,7 @@ static void define_va_opt_3() "err()"; ASSERT_EQUALS("", preprocess(code2, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing opening parenthesis for __VA_OPT__\n", toString(outputList)); } @@ -946,7 +946,7 @@ static void define_va_opt_4() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code1, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing opening parenthesis for __VA_OPT__\n", toString(outputList)); outputList.clear(); @@ -956,7 +956,7 @@ static void define_va_opt_4() "err()"; ASSERT_EQUALS("", preprocess(code2, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing opening parenthesis for __VA_OPT__\n", toString(outputList)); } @@ -968,7 +968,46 @@ static void define_va_opt_5() simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'err', Missing parenthesis for __VA_OPT__(content)\n", + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing opening parenthesis for __VA_OPT__\n", + toString(outputList)); +} + +static void define_va_opt_6() +{ + // nested __VA_OPT__ + const char code[] = "#define err(...) __VA_OPT__(__VA_OPT__(something))\n" + "err()"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': __VA_OPT__ cannot be nested\n", + toString(outputList)); +} + +static void define_va_opt_7() +{ + // eof in __VA_OPT__ + const char code1[] = "#define err(...) __VA_OPT__"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code1, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing opening parenthesis for __VA_OPT__\n", + toString(outputList)); + + outputList.clear(); + + const char code2[] = "#define err(...) __VA_OPT__("; + + ASSERT_EQUALS("", preprocess(code2, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing closing parenthesis for __VA_OPT__\n", + toString(outputList)); + + outputList.clear(); + + const char code3[] = "#define err(...) __VA_OPT__(x"; + + ASSERT_EQUALS("", preprocess(code3, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, In definition of 'err': Missing closing parenthesis for __VA_OPT__\n", toString(outputList)); } @@ -3063,6 +3102,8 @@ int main(int argc, char **argv) TEST_CASE(define_va_opt_3); TEST_CASE(define_va_opt_4); TEST_CASE(define_va_opt_5); + TEST_CASE(define_va_opt_6); + TEST_CASE(define_va_opt_7); TEST_CASE(pragma_backslash); // multiline pragma directive diff --git a/testsuite/clang-preprocessor-tests/macro_fn_va_opt.c b/testsuite/clang-preprocessor-tests/macro_fn_va_opt.c new file mode 100644 index 00000000..ccb09e95 --- /dev/null +++ b/testsuite/clang-preprocessor-tests/macro_fn_va_opt.c @@ -0,0 +1,13 @@ +// RUN: %clang_cc1 -E %s | grep '^ printf( "%%s" , "Hello" );$' + +#define P( x, ...) printf( x __VA_OPT__(,) __VA_ARGS__ ) +#define PF( x, ...) P( x __VA_OPT__(,) __VA_ARGS__ ) + +int main() +{ + PF( "%s", "Hello" ); + PF( "Hello", ); + PF( "Hello" ); + PF( , ); + PF( ); +} From 0ff0149510fcebc8ccdaf56402d400d5e290fca7 Mon Sep 17 00:00:00 2001 From: glankk Date: Thu, 3 Jul 2025 11:33:59 +0200 Subject: [PATCH 270/381] Fix #449 (Update c++ standard to c++11) (#450) --- .github/workflows/clang-tidy.yml | 2 +- CMakeLists.txt | 21 +------ Makefile | 4 +- simplecpp.cpp | 99 +++++--------------------------- simplecpp.h | 12 ---- 5 files changed, 19 insertions(+), 119 deletions(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 9a4f43c9..a2f7b6dc 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -30,7 +30,7 @@ jobs: - name: Prepare CMake run: | - cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DDISABLE_CPP03_SYNTAX_CHECK=ON + cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: CXX: clang-20 diff --git a/CMakeLists.txt b/CMakeLists.txt index 672e63bb..6ab0166e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,8 @@ cmake_minimum_required (VERSION 3.10) project (simplecpp LANGUAGES CXX) -option(DISABLE_CPP03_SYNTAX_CHECK "Disable the C++03 syntax check." OFF) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) include(CheckCXXCompilerFlag) @@ -70,25 +71,7 @@ endif() add_library(simplecpp_obj OBJECT simplecpp.cpp) add_executable(simplecpp $ main.cpp) -set_property(TARGET simplecpp PROPERTY CXX_STANDARD 11) - -if (NOT DISABLE_CPP03_SYNTAX_CHECK) - # it is not possible to set a standard older than C++14 with Visual Studio - if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - # we need to create a dummy library as -fsyntax-only will not produce any output files causing the build to fail - add_library(simplecpp-03-syntax OBJECT simplecpp.cpp) - target_compile_options(simplecpp-03-syntax PRIVATE -std=c++03) - if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - target_compile_options(simplecpp-03-syntax PRIVATE -Wno-long-long) - elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(simplecpp-03-syntax PRIVATE -Wno-c++11-long-long -Wno-c++11-compat) - endif() - add_dependencies(simplecpp simplecpp-03-syntax) - endif() -endif() - add_executable(testrunner $ test.cpp) -set_property(TARGET testrunner PROPERTY CXX_STANDARD 11) enable_testing() add_test(NAME testrunner COMMAND testrunner) diff --git a/Makefile b/Makefile index db1ca257..73977517 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++0x -g +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++11 -g LDFLAGS = -g %.o: %.cpp simplecpp.h @@ -11,8 +11,6 @@ testrunner: test.o simplecpp.o $(CXX) $(LDFLAGS) simplecpp.o test.o -o testrunner test: testrunner simplecpp - # The -std=c++03 makes sure that simplecpp.cpp is C++03 conformant. We don't require a C++11 compiler - g++ -std=c++03 -fsyntax-only simplecpp.cpp ./testrunner python3 run-tests.py diff --git a/simplecpp.cpp b/simplecpp.cpp index 97657d60..c29040bc 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -31,12 +31,10 @@ #include #include #include -#if __cplusplus >= 201103L #ifdef SIMPLECPP_WINDOWS #include #endif #include -#endif #include #include @@ -51,18 +49,6 @@ #undef ERROR #endif -#if __cplusplus >= 201103L -#define OVERRIDE override -#define EXPLICIT explicit -#else -#define OVERRIDE -#define EXPLICIT -#endif - -#if (__cplusplus < 201103L) && !defined(__APPLE__) -#define nullptr NULL -#endif - static bool isHex(const std::string &s) { return s.size()>2 && (s.compare(0,2,"0x")==0 || s.compare(0,2,"0X")==0); @@ -368,22 +354,22 @@ class simplecpp::TokenList::Stream { class StdIStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - EXPLICIT StdIStream(std::istream &istr) + explicit StdIStream(std::istream &istr) : istr(istr) { assert(istr.good()); init(); } - virtual int get() OVERRIDE { + virtual int get() override { return istr.get(); } - virtual int peek() OVERRIDE { + virtual int peek() override { return istr.peek(); } - virtual void unget() OVERRIDE { + virtual void unget() override { istr.unget(); } - virtual bool good() OVERRIDE { + virtual bool good() override { return istr.good(); } @@ -402,20 +388,20 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { init(); } - virtual int get() OVERRIDE { + virtual int get() override { if (pos >= size) return lastStatus = EOF; return str[pos++]; } - virtual int peek() OVERRIDE { + virtual int peek() override { if (pos >= size) return lastStatus = EOF; return str[pos]; } - virtual void unget() OVERRIDE { + virtual void unget() override { --pos; } - virtual bool good() OVERRIDE { + virtual bool good() override { return lastStatus != EOF; } @@ -429,7 +415,7 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { class FileStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - EXPLICIT FileStream(const std::string &filename, std::vector &files) + explicit FileStream(const std::string &filename, std::vector &files) : file(fopen(filename.c_str(), "rb")) , lastCh(0) , lastStatus(0) { @@ -440,25 +426,25 @@ class FileStream : public simplecpp::TokenList::Stream { init(); } - ~FileStream() OVERRIDE { + ~FileStream() override { fclose(file); file = nullptr; } - virtual int get() OVERRIDE { + virtual int get() override { lastStatus = lastCh = fgetc(file); return lastCh; } - virtual int peek() OVERRIDE{ + virtual int peek() override{ // keep lastCh intact const int ch = fgetc(file); unget_internal(ch); return ch; } - virtual void unget() OVERRIDE { + virtual void unget() override { unget_internal(lastCh); } - virtual bool good() OVERRIDE { + virtual bool good() override { return lastStatus != EOF; } @@ -519,12 +505,10 @@ simplecpp::TokenList::TokenList(const TokenList &other) : frontToken(nullptr), b *this = other; } -#if __cplusplus >= 201103L simplecpp::TokenList::TokenList(TokenList &&other) : frontToken(nullptr), backToken(nullptr), files(other.files) { *this = std::move(other); } -#endif simplecpp::TokenList::~TokenList() { @@ -543,7 +527,6 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(const TokenList &other) return *this; } -#if __cplusplus >= 201103L simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) { if (this != &other) { @@ -557,7 +540,6 @@ simplecpp::TokenList &simplecpp::TokenList::operator=(TokenList &&other) } return *this; } -#endif void simplecpp::TokenList::clear() { @@ -1477,11 +1459,7 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) namespace simplecpp { class Macro; -#if __cplusplus >= 201103L using MacroMap = std::unordered_map; -#else - typedef std::map MacroMap; -#endif class Macro { public: @@ -1791,13 +1769,8 @@ namespace simplecpp { tok = tok->next; } } -#if __cplusplus >= 201103L optExpandValue = new TokenList(std::move(expandValue)); optNoExpandValue = new TokenList(std::move(noExpandValue)); -#else - optExpandValue = new TokenList(expandValue); - optNoExpandValue = new TokenList(noExpandValue); -#endif } return true; @@ -2437,47 +2410,9 @@ namespace simplecpp { #ifdef SIMPLECPP_WINDOWS -#if __cplusplus >= 201103L using MyMutex = std::mutex; template using MyLock = std::lock_guard; -#else -class MyMutex { -public: - MyMutex() { - InitializeCriticalSection(&m_criticalSection); - } - - ~MyMutex() { - DeleteCriticalSection(&m_criticalSection); - } - - CRITICAL_SECTION* lock() { - return &m_criticalSection; - } -private: - CRITICAL_SECTION m_criticalSection; -}; - -template -class MyLock { -public: - explicit MyLock(T& m) - : m_mutex(m) { - EnterCriticalSection(m_mutex.lock()); - } - - ~MyLock() { - LeaveCriticalSection(m_mutex.lock()); - } - -private: - MyLock& operator=(const MyLock&); - MyLock(const MyLock&); - - T& m_mutex; -}; -#endif class RealFileNameMap { public: @@ -4099,7 +4034,3 @@ std::string simplecpp::getCppStdString(const std::string &std) { return getCppStdString(getCppStd(std)); } - -#if (__cplusplus < 201103L) && !defined(__APPLE__) -#undef nullptr -#endif diff --git a/simplecpp.h b/simplecpp.h index f5c69593..9fd95808 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -27,10 +27,6 @@ # define SIMPLECPP_LIB #endif -#if (__cplusplus < 201103L) && !defined(__APPLE__) -#define nullptr NULL -#endif - #if defined(_MSC_VER) # pragma warning(push) // suppress warnings about "conversion from 'type1' to 'type2', possible loss of data" @@ -214,14 +210,10 @@ namespace simplecpp { /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); -#if __cplusplus >= 201103L TokenList(TokenList &&other); -#endif ~TokenList(); TokenList &operator=(const TokenList &other); -#if __cplusplus >= 201103L TokenList &operator=(TokenList &&other); -#endif void clear(); bool empty() const { @@ -395,8 +387,4 @@ namespace simplecpp { # pragma warning(pop) #endif -#if (__cplusplus < 201103L) && !defined(__APPLE__) -#undef nullptr -#endif - #endif From c1f368832dbefa508480bbcdafea87f044645563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20Gunne=20Lindstr=C3=B6m?= Date: Thu, 3 Jul 2025 11:39:43 +0200 Subject: [PATCH 271/381] Fix #454: Accept __has_include for GNU C standards (#456) --- simplecpp.cpp | 9 ++++++--- test.cpp | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index c29040bc..d925773e 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2673,13 +2673,16 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::map= "201703L"); } +static bool isGnu(const simplecpp::DUI &dui) +{ + return dui.std.rfind("gnu", 0) != std::string::npos; +} static std::string currentDirectoryOSCalc() { const std::size_t size = 4096; @@ -2752,7 +2755,7 @@ static std::string extractRelativePathFromAbsolute(const std::string& absoluteSi static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { - if (!isCpp17OrLater(dui)) + if (!isCpp17OrLater(dui) && !isGnu(dui)) return; for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { @@ -3475,7 +3478,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL // use a dummy vector for the macros because as this is not part of the file and would add an empty entry - e.g. /usr/include/poll.h std::vector dummy; - const bool hasInclude = isCpp17OrLater(dui); + const bool hasInclude = isCpp17OrLater(dui) || isGnu(dui); MacroMap macros; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; diff --git a/test.cpp b/test.cpp index caa6137e..45498a08 100644 --- a/test.cpp +++ b/test.cpp @@ -1602,6 +1602,21 @@ static void has_include_5() ASSERT_EQUALS("", preprocess(code)); } +static void has_include_6() +{ + const char code[] = "#if defined( __has_include)\n" + " #if !__has_include()\n" + " A\n" + " #else\n" + " B\n" + " #endif\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "gnu99"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("", preprocess(code)); +} + static void ifdef1() { const char code[] = "#ifdef A\n" @@ -3164,6 +3179,7 @@ int main(int argc, char **argv) TEST_CASE(has_include_3); TEST_CASE(has_include_4); TEST_CASE(has_include_5); + TEST_CASE(has_include_6); TEST_CASE(ifdef1); TEST_CASE(ifdef2); From 4bbd1bf8e320471f2a46908c947251ed8aa18b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20Gunne=20Lindstr=C3=B6m?= Date: Thu, 3 Jul 2025 12:02:25 +0200 Subject: [PATCH 272/381] Fix #452: Undefined function-style macro does not cause an error (#453) --- simplecpp.cpp | 2 ++ test.cpp | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index d925773e..a69dc0c2 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2827,6 +2827,8 @@ static void simplifyName(simplecpp::TokenList &expr) if (alt) continue; } + if (tok->next && tok->next->str() == "(") + throw std::runtime_error("undefined function-like macro invocation: " + tok->str() + "( ... )"); tok->setstr("0"); } } diff --git a/test.cpp b/test.cpp index 45498a08..fb3e4b22 100644 --- a/test.cpp +++ b/test.cpp @@ -1879,6 +1879,15 @@ static void ifexpr() ASSERT_EQUALS("\n\n1", preprocess(code)); } +static void ifUndefFuncStyleMacro() +{ + const char code[] = "#if A()\n" + "#endif\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition, undefined function-like macro invocation: A( ... )\n", toString(outputList)); +} + static void location1() { const char *code; @@ -3202,6 +3211,7 @@ int main(int argc, char **argv) TEST_CASE(ifdiv0); TEST_CASE(ifalt); // using "and", "or", etc TEST_CASE(ifexpr); + TEST_CASE(ifUndefFuncStyleMacro); TEST_CASE(location1); TEST_CASE(location2); From f566848788c1445e56b543255e82ac3b0c952ee9 Mon Sep 17 00:00:00 2001 From: glankk Date: Mon, 7 Jul 2025 10:33:11 +0200 Subject: [PATCH 273/381] Add caching of conditional directive chains (#468) --- simplecpp.cpp | 15 ++++++++++++++- simplecpp.h | 5 +++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a69dc0c2..e9641b04 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3532,6 +3532,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL // AlwaysFalse => drop all code in #if and #else enum IfState { True, ElseIsTrue, AlwaysFalse }; std::stack ifstates; + std::stack iftokens; ifstates.push(True); std::stack includetokenstack; @@ -3855,15 +3856,24 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL ifstates.push(AlwaysFalse); else ifstates.push(conditionIsTrue ? True : ElseIsTrue); + iftokens.push(rawtok); } else if (ifstates.top() == True) { ifstates.top() = AlwaysFalse; + iftokens.top()->nextcond = rawtok; + iftokens.top() = rawtok; } else if (ifstates.top() == ElseIsTrue && conditionIsTrue) { ifstates.top() = True; + iftokens.top()->nextcond = rawtok; + iftokens.top() = rawtok; } } else if (rawtok->str() == ELSE) { ifstates.top() = (ifstates.top() == ElseIsTrue) ? True : AlwaysFalse; + iftokens.top()->nextcond = rawtok; + iftokens.top() = rawtok; } else if (rawtok->str() == ENDIF) { ifstates.pop(); + iftokens.top()->nextcond = rawtok; + iftokens.pop(); } else if (rawtok->str() == UNDEF) { if (ifstates.top() == True) { const Token *tok = rawtok->next; @@ -3875,7 +3885,10 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (ifstates.top() == True && rawtok->str() == PRAGMA && rawtok->next && rawtok->next->str() == ONCE && sameline(rawtok,rawtok->next)) { pragmaOnce.insert(rawtok->location.file()); } - rawtok = gotoNextLine(rawtok); + if (ifstates.top() != True && rawtok->nextcond) + rawtok = rawtok->nextcond->previous; + else + rawtok = gotoNextLine(rawtok); continue; } diff --git a/simplecpp.h b/simplecpp.h index 9fd95808..579e6e14 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -96,12 +96,12 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc, bool wsahead = false) : - whitespaceahead(wsahead), location(loc), previous(nullptr), next(nullptr), string(s) { + whitespaceahead(wsahead), location(loc), previous(nullptr), next(nullptr), nextcond(nullptr), string(s) { flags(); } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), nextcond(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { } void flags() { @@ -137,6 +137,7 @@ namespace simplecpp { Location location; Token *previous; Token *next; + mutable const Token *nextcond; const Token *previousSkipComments() const { const Token *tok = this->previous; From c7e99745d0ee079ee3041b7ff7bff03ced19307a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20Gunne=20Lindstr=C3=B6m?= Date: Mon, 7 Jul 2025 10:45:27 +0200 Subject: [PATCH 274/381] fix #459: Set __STRICT_ANSI__=1 for non-gnu standards (#460) --- simplecpp.cpp | 7 +++++++ test.cpp | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index e9641b04..c0f6e2c0 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3482,11 +3482,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool hasInclude = isCpp17OrLater(dui) || isGnu(dui); MacroMap macros; + bool strictAnsiDefined = false; for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { const std::string ¯ostr = *it; const std::string::size_type eq = macrostr.find('='); const std::string::size_type par = macrostr.find('('); const std::string macroname = macrostr.substr(0, std::min(eq,par)); + if (macroname == "__STRICT_ANSI__") + strictAnsiDefined = true; if (dui.undefined.find(macroname) != dui.undefined.end()) continue; const std::string lhs(macrostr.substr(0,eq)); @@ -3495,6 +3498,10 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::pair(macro.name(), macro)); } + const bool strictAnsiUndefined = dui.undefined.find("__STRICT_ANSI__") != dui.undefined.cend(); + if (!isGnu(dui) && !strictAnsiDefined && !strictAnsiUndefined) + macros.insert(std::pair("__STRICT_ANSI__", Macro("__STRICT_ANSI__", "1", dummy))); + macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", dummy))); macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", dummy))); macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", dummy))); diff --git a/test.cpp b/test.cpp index fb3e4b22..ba21d71b 100644 --- a/test.cpp +++ b/test.cpp @@ -1617,6 +1617,48 @@ static void has_include_6() ASSERT_EQUALS("", preprocess(code)); } +static void strict_ansi_1() +{ + const char code[] = "#if __STRICT_ANSI__\n" + " A\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "gnu99"; + ASSERT_EQUALS("", preprocess(code, dui)); +} + +static void strict_ansi_2() +{ + const char code[] = "#if __STRICT_ANSI__\n" + " A\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "c99"; + ASSERT_EQUALS("\nA", preprocess(code, dui)); +} + +static void strict_ansi_3() +{ + const char code[] = "#if __STRICT_ANSI__\n" + " A\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "c99"; + dui.undefined.insert("__STRICT_ANSI__"); + ASSERT_EQUALS("", preprocess(code, dui)); +} + +static void strict_ansi_4() +{ + const char code[] = "#if __STRICT_ANSI__\n" + " A\n" + "#endif"; + simplecpp::DUI dui; + dui.std = "gnu99"; + dui.defines.push_back("__STRICT_ANSI__"); + ASSERT_EQUALS("\nA", preprocess(code, dui)); +} + static void ifdef1() { const char code[] = "#ifdef A\n" @@ -3190,6 +3232,11 @@ int main(int argc, char **argv) TEST_CASE(has_include_5); TEST_CASE(has_include_6); + TEST_CASE(strict_ansi_1); + TEST_CASE(strict_ansi_2); + TEST_CASE(strict_ansi_3); + TEST_CASE(strict_ansi_4); + TEST_CASE(ifdef1); TEST_CASE(ifdef2); TEST_CASE(ifndef); From a0430f34e8b9e8a5980158eb9c7e5101b9f19473 Mon Sep 17 00:00:00 2001 From: glankk Date: Mon, 7 Jul 2025 11:22:23 +0200 Subject: [PATCH 275/381] Include and path handling optimization (#447) --- integration_test.py | 20 +- main.cpp | 3 +- simplecpp.cpp | 521 ++++++++++++-------------------------------- simplecpp.h | 131 ++++++++++- test.cpp | 173 ++++++++------- 5 files changed, 372 insertions(+), 476 deletions(-) diff --git a/integration_test.py b/integration_test.py index 27528e16..a59ae338 100644 --- a/integration_test.py +++ b/integration_test.py @@ -237,7 +237,13 @@ def test_same_name_header(record_property, tmpdir): assert stderr == "" def test_pragma_once_matching(record_property, tmpdir): - if platform.system() == "win32": + test_dir = os.path.join(tmpdir, "test_dir") + test_subdir = os.path.join(test_dir, "test_subdir") + + test_file = os.path.join(test_dir, "test.c") + once_header = os.path.join(test_dir, "once.h") + + if platform.system() == "Windows": names_to_test = [ '"once.h"', '"Once.h"', @@ -251,6 +257,10 @@ def test_pragma_once_matching(record_property, tmpdir): '"test_subdir/../Once.h"', '"Test_Subdir/../once.h"', '"Test_Subdir/../Once.h"', + f"\"{test_dir}/once.h\"", + f"\"{test_dir}/Once.h\"", + f"<{test_dir}/once.h>", + f"<{test_dir}/Once.h>", ] else: names_to_test = [ @@ -258,14 +268,10 @@ def test_pragma_once_matching(record_property, tmpdir): '', '"../test_dir/once.h"', '"test_subdir/../once.h"', + f"\"{test_dir}/once.h\"", + f"<{test_dir}/once.h>", ] - test_dir = os.path.join(tmpdir, "test_dir") - test_subdir = os.path.join(test_dir, "test_subdir") - - test_file = os.path.join(test_dir, "test.c") - once_header = os.path.join(test_dir, "once.h") - os.mkdir(test_dir) os.mkdir(test_subdir) diff --git a/main.cpp b/main.cpp index 424ef6fa..a6d14386 100644 --- a/main.cpp +++ b/main.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -128,7 +127,7 @@ int main(int argc, char **argv) } rawtokens->removeComments(); simplecpp::TokenList outputTokens(files); - std::map filedata; + simplecpp::FileDataCache filedata; simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList); simplecpp::cleanup(filedata); delete rawtokens; diff --git a/simplecpp.cpp b/simplecpp.cpp index c0f6e2c0..128da0bd 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -4,8 +4,10 @@ */ #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) -#define SIMPLECPP_WINDOWS -#define NOMINMAX +# define _WIN32_WINNT 0x0602 +# define NOMINMAX +# include +# undef ERROR #endif #include "simplecpp.h" @@ -32,21 +34,16 @@ #include #include #ifdef SIMPLECPP_WINDOWS -#include +# include #endif #include #include #include #ifdef _WIN32 -#include +# include #else -#include -#endif - -#ifdef SIMPLECPP_WINDOWS -#include -#undef ERROR +# include #endif static bool isHex(const std::string &s) @@ -139,12 +136,6 @@ static unsigned long long stringToULL(const std::string &s) return ret; } -// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild -static bool startsWith_(const std::string &s, const std::string &p) -{ - return (s.size() >= p.size()) && std::equal(p.begin(), p.end(), s.begin()); -} - static bool endsWith(const std::string &s, const std::string &e) { return (s.size() >= e.size()) && std::equal(e.rbegin(), e.rend(), s.rbegin()); @@ -435,7 +426,7 @@ class FileStream : public simplecpp::TokenList::Stream { lastStatus = lastCh = fgetc(file); return lastCh; } - virtual int peek() override{ + virtual int peek() override { // keep lastCh intact const int ch = fgetc(file); unget_internal(ch); @@ -2409,132 +2400,6 @@ namespace simplecpp { } #ifdef SIMPLECPP_WINDOWS - -using MyMutex = std::mutex; -template -using MyLock = std::lock_guard; - -class RealFileNameMap { -public: - RealFileNameMap() {} - - bool getCacheEntry(const std::string& path, std::string& returnPath) { - MyLock lock(m_mutex); - - const std::map::iterator it = m_fileMap.find(path); - if (it != m_fileMap.end()) { - returnPath = it->second; - return true; - } - return false; - } - - void addToCache(const std::string& path, const std::string& actualPath) { - MyLock lock(m_mutex); - m_fileMap[path] = actualPath; - } - -private: - std::map m_fileMap; - MyMutex m_mutex; -}; - -static RealFileNameMap realFileNameMap; - -static bool realFileName(const std::string &f, std::string &result) -{ - // are there alpha characters in last subpath? - bool alpha = false; - for (std::string::size_type pos = 1; pos <= f.size(); ++pos) { - const unsigned char c = f[f.size() - pos]; - if (c == '/' || c == '\\') - break; - if (std::isalpha(c)) { - alpha = true; - break; - } - } - - // do not convert this path if there are no alpha characters (either pointless or cause wrong results for . and ..) - if (!alpha) - return false; - - // Lookup filename or foldername on file system - if (!realFileNameMap.getCacheEntry(f, result)) { - - WIN32_FIND_DATAA FindFileData; - -#ifdef __CYGWIN__ - const std::string fConverted = simplecpp::convertCygwinToWindowsPath(f); - const HANDLE hFind = FindFirstFileExA(fConverted.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); -#else - HANDLE hFind = FindFirstFileExA(f.c_str(), FindExInfoBasic, &FindFileData, FindExSearchNameMatch, NULL, 0); -#endif - - if (INVALID_HANDLE_VALUE == hFind) - return false; - result = FindFileData.cFileName; - realFileNameMap.addToCache(f, result); - FindClose(hFind); - } - return true; -} - -static RealFileNameMap realFilePathMap; - -/** Change case in given path to match filesystem */ -static std::string realFilename(const std::string &f) -{ - std::string ret; - ret.reserve(f.size()); // this will be the final size - if (realFilePathMap.getCacheEntry(f, ret)) - return ret; - - // Current subpath - std::string subpath; - - for (std::string::size_type pos = 0; pos < f.size(); ++pos) { - const unsigned char c = f[pos]; - - // Separator.. add subpath and separator - if (c == '/' || c == '\\') { - // if subpath is empty just add separator - if (subpath.empty()) { - ret += c; - continue; - } - - const bool isDriveSpecification = - (pos == 2 && subpath.size() == 2 && std::isalpha(subpath[0]) && subpath[1] == ':'); - - // Append real filename (proper case) - std::string f2; - if (!isDriveSpecification && realFileName(f.substr(0, pos), f2)) - ret += f2; - else - ret += subpath; - - subpath.clear(); - - // Append separator - ret += c; - } else { - subpath += c; - } - } - - if (!subpath.empty()) { - std::string f2; - if (realFileName(f,f2)) - ret += f2; - else - ret += subpath; - } - - realFilePathMap.addToCache(f, ret); - return ret; -} - static bool isAbsolutePath(const std::string &path) { if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) @@ -2542,8 +2407,6 @@ static bool isAbsolutePath(const std::string &path) return path.length() > 1U && (path[0] == '/' || path[0] == '\\'); } #else -#define realFilename(f) f - static bool isAbsolutePath(const std::string &path) { return path.length() > 1U && path[0] == '/'; @@ -2621,8 +2484,7 @@ namespace simplecpp { if (unc) path = '/' + path; - // cppcheck-suppress duplicateExpressionTernary - platform-dependent implementation - return strpbrk(path.c_str(), "*?") == nullptr ? realFilename(path) : path; + return path; } } @@ -2684,37 +2546,8 @@ static bool isGnu(const simplecpp::DUI &dui) return dui.std.rfind("gnu", 0) != std::string::npos; } -static std::string currentDirectoryOSCalc() { - const std::size_t size = 4096; - char currentPath[size]; - -#ifndef _WIN32 - if (getcwd(currentPath, size) != nullptr) -#else - if (_getcwd(currentPath, size) != nullptr) -#endif - return std::string(currentPath); - - return ""; -} - -static const std::string& currentDirectory() { - static const std::string curdir = simplecpp::simplifyPath(currentDirectoryOSCalc()); - return curdir; -} - -static std::string toAbsolutePath(const std::string& path) { - if (path.empty()) { - return path;// preserve error file path that is indicated by an empty string - } - if (!isAbsolutePath(path)) { - return simplecpp::simplifyPath(currentDirectory() + "/" + path); - } - // otherwise - return simplecpp::simplifyPath(path); -} - -static std::string dirPath(const std::string& path, bool withTrailingSlash=true) { +static std::string dirPath(const std::string& path, bool withTrailingSlash=true) +{ const std::size_t lastSlash = path.find_last_of("\\/"); if (lastSlash == std::string::npos) { return ""; @@ -2722,36 +2555,6 @@ static std::string dirPath(const std::string& path, bool withTrailingSlash=true) return path.substr(0, lastSlash + (withTrailingSlash ? 1U : 0U)); } -static std::string omitPathTrailingSlash(const std::string& path) { - if (endsWith(path, "/")) { - return path.substr(0, path.size() - 1U); - } - return path; -} - -static std::string extractRelativePathFromAbsolute(const std::string& absoluteSimplifiedPath, const std::string& prefixSimplifiedAbsoluteDir = currentDirectory()) { - const std::string normalizedAbsolutePath = omitPathTrailingSlash(absoluteSimplifiedPath); - std::string currentPrefix = omitPathTrailingSlash(prefixSimplifiedAbsoluteDir); - std::string leadingParenting; - while (!startsWith_(normalizedAbsolutePath, currentPrefix)) { - leadingParenting = "../" + leadingParenting; - currentPrefix = dirPath(currentPrefix, false); - } - const std::size_t size = currentPrefix.size(); - std::string relativeFromMeetingPath = normalizedAbsolutePath.substr(size, normalizedAbsolutePath.size() - size); - if (currentPrefix.empty() && !(startsWith_(absoluteSimplifiedPath, "/") && startsWith_(prefixSimplifiedAbsoluteDir, "/"))) { - // In the case that there is no common prefix path, - // and at not both of the paths start with `/` (can happen only in Windows paths on distinct partitions), - // return the absolute simplified path as is because no relative path can match. - return absoluteSimplifiedPath; - } - if (startsWith_(relativeFromMeetingPath, "/")) { - // omit the leading slash - relativeFromMeetingPath = relativeFromMeetingPath.substr(1, relativeFromMeetingPath.size()); - } - return leadingParenting + relativeFromMeetingPath; -} - static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { @@ -2796,10 +2599,8 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI for (simplecpp::Token *headerToken = tok1->next; headerToken != tok3; headerToken = headerToken->next) header += headerToken->str(); - // cppcheck-suppress selfAssignment - platform-dependent implementation - header = realFilename(header); } else { - header = realFilename(tok1->str().substr(1U, tok1->str().size() - 2U)); + header = tok1->str().substr(1U, tok1->str().size() - 2U); } std::ifstream f; const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); @@ -3131,206 +2932,185 @@ class NonExistingFilesCache { NonExistingFilesCache() {} bool contains(const std::string& path) { - MyLock lock(m_mutex); + std::lock_guard lock(m_mutex); return (m_pathSet.find(path) != m_pathSet.end()); } void add(const std::string& path) { - MyLock lock(m_mutex); + std::lock_guard lock(m_mutex); m_pathSet.insert(path); } void clear() { - MyLock lock(m_mutex); + std::lock_guard lock(m_mutex); m_pathSet.clear(); } private: std::set m_pathSet; - MyMutex m_mutex; + std::mutex m_mutex; }; static NonExistingFilesCache nonExistingFilesCache; #endif -static std::string openHeader(std::ifstream &f, const std::string &path) +static std::string openHeaderDirect(std::ifstream &f, const std::string &path) { - std::string simplePath = simplecpp::simplifyPath(path); #ifdef SIMPLECPP_WINDOWS - if (nonExistingFilesCache.contains(simplePath)) + if (nonExistingFilesCache.contains(path)) return ""; // file is known not to exist, skip expensive file open call #endif - f.open(simplePath.c_str()); + f.open(path.c_str()); if (f.is_open()) - return simplePath; + return path; #ifdef SIMPLECPP_WINDOWS - nonExistingFilesCache.add(simplePath); + nonExistingFilesCache.add(path); #endif return ""; } -static std::string getRelativeFileName(const std::string &baseFile, const std::string &header, bool returnAbsolutePath) -{ - const std::string baseFileSimplified = simplecpp::simplifyPath(baseFile); - const std::string baseFileAbsolute = isAbsolutePath(baseFileSimplified) ? - baseFileSimplified : - simplecpp::simplifyPath(currentDirectory() + "/" + baseFileSimplified); - - const std::string headerSimplified = simplecpp::simplifyPath(header); - const std::string path = isAbsolutePath(headerSimplified) ? - headerSimplified : - simplecpp::simplifyPath(dirPath(baseFileAbsolute) + headerSimplified); - - return returnAbsolutePath ? toAbsolutePath(path) : extractRelativePathFromAbsolute(path); -} - -static std::string openHeaderRelative(std::ifstream &f, const std::string &sourcefile, const std::string &header) -{ - return openHeader(f, getRelativeFileName(sourcefile, header, isAbsolutePath(sourcefile))); -} - -// returns the simplified header path: -// * If the header path is absolute, returns it in absolute path -// * Otherwise, returns it in relative path with respect to the current directory -static std::string getIncludePathFileName(const std::string &includePath, const std::string &header) +static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { - std::string simplifiedHeader = simplecpp::simplifyPath(header); + if (isAbsolutePath(header)) + return openHeaderDirect(f, simplecpp::simplifyPath(header)); - if (isAbsolutePath(simplifiedHeader)) { - return simplifiedHeader; + // prefer first to search the header relatively to source file if found, when not a system header + if (!systemheader) { + std::string path = openHeaderDirect(f, simplecpp::simplifyPath(dirPath(sourcefile) + header)); + if (!path.empty()) { + return path; + } } - std::string basePath = toAbsolutePath(includePath); - if (!basePath.empty() && basePath[basePath.size()-1U]!='/' && basePath[basePath.size()-1U]!='\\') - basePath += '/'; - const std::string absoluteSimplifiedHeaderPath = simplecpp::simplifyPath(basePath + simplifiedHeader); - // preserve absoluteness/relativieness of the including dir - return isAbsolutePath(includePath) ? absoluteSimplifiedHeaderPath : extractRelativePathFromAbsolute(absoluteSimplifiedHeaderPath); -} - -static std::string openHeaderIncludePath(std::ifstream &f, const simplecpp::DUI &dui, const std::string &header) -{ - for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - std::string path = openHeader(f, getIncludePathFileName(*it, header)); + // search the header on the include paths (provided by the flags "-I...") + for (const auto &includePath : dui.includePaths) { + std::string path = openHeaderDirect(f, simplecpp::simplifyPath(includePath + "/" + header)); if (!path.empty()) return path; } return ""; } -static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) +std::pair simplecpp::FileDataCache::tryload(FileDataCache::name_map_type::iterator &name_it, const simplecpp::DUI &dui, std::vector &filenames, simplecpp::OutputList *outputList) { - if (isAbsolutePath(header)) - return openHeader(f, header); + const std::string &path = name_it->first; + FileID fileId; - // prefer first to search the header relatively to source file if found, when not a system header - if (!systemheader) { - std::string relativeHeader = openHeaderRelative(f, sourcefile, header); - if (!relativeHeader.empty()) { - return relativeHeader; - } + if (!getFileId(path, fileId)) + return {nullptr, false}; + + const auto id_it = mIdMap.find(fileId); + if (id_it != mIdMap.end()) { + name_it->second = id_it->second; + return {id_it->second, false}; } - // search the header on the include paths (provided by the flags "-I...") - return openHeaderIncludePath(f, dui, header); -} + std::ifstream f(path); + FileData *const data = new FileData {path, TokenList(f, filenames, path, outputList)}; -static std::string findPathInMapBothRelativeAndAbsolute(const std::map &filedata, const std::string& path) { - // here there are two possibilities - either we match this from absolute path or from a relative one - if (filedata.find(path) != filedata.end()) {// try first to respect the exact match - return path; - } + if (dui.removeComments) + data->tokens.removeComments(); - // otherwise - try to use the normalize to the correct representation - std::string alternativePath; - if (isAbsolutePath(path)) { - alternativePath = extractRelativePathFromAbsolute(simplecpp::simplifyPath(path)); - } else { - alternativePath = toAbsolutePath(path); - } + name_it->second = data; + mIdMap.emplace(fileId, data); + mData.emplace_back(data); - if (filedata.find(alternativePath) != filedata.end()) { - return alternativePath; - } - return ""; + return {data, true}; } -static std::string getFileIdPath(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +std::pair simplecpp::FileDataCache::get(const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader, std::vector &filenames, simplecpp::OutputList *outputList) { - if (filedata.empty()) { - return ""; - } if (isAbsolutePath(header)) { - const std::string simplifiedHeaderPath = simplecpp::simplifyPath(header); - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, simplifiedHeaderPath); - if (!match.empty()) { - return match; + auto ins = mNameMap.emplace(simplecpp::simplifyPath(header), nullptr); + + if (ins.second) { + const auto ret = tryload(ins.first, dui, filenames, outputList); + if (ret.first != nullptr) { + return ret; + } + } else { + return {ins.first->second, false}; } + + return {nullptr, false}; } if (!systemheader) { - const std::string relativeFilename = getRelativeFileName(sourcefile, header, true); - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, relativeFilename); - if (!match.empty()) { - return match; - } - // if the file exists but hasn't been loaded yet then we need to stop searching here or we could get a false match - std::ifstream f; - openHeader(f, relativeFilename); - if (f.is_open()) { - f.close(); - return ""; + auto ins = mNameMap.emplace(simplecpp::simplifyPath(dirPath(sourcefile) + header), nullptr); + + if (ins.second) { + const auto ret = tryload(ins.first, dui, filenames, outputList); + if (ret.first != nullptr) { + return ret; + } + } else if (ins.first->second != nullptr) { + return {ins.first->second, false}; } - } else if (filedata.find(header) != filedata.end()) { - return header;// system header that its file is already in the filedata - return that as is } - for (std::list::const_iterator it = dui.includePaths.begin(); it != dui.includePaths.end(); ++it) { - const std::string match = findPathInMapBothRelativeAndAbsolute(filedata, getIncludePathFileName(*it, header)); - if (!match.empty()) { - return match; + for (const auto &includePath : dui.includePaths) { + auto ins = mNameMap.emplace(simplecpp::simplifyPath(includePath + "/" + header), nullptr); + + if (ins.second) { + const auto ret = tryload(ins.first, dui, filenames, outputList); + if (ret.first != nullptr) { + return ret; + } + } else if (ins.first->second != nullptr) { + return {ins.first->second, false}; } } - return ""; + return {nullptr, false}; } -static bool hasFile(const std::map &filedata, const std::string &sourcefile, const std::string &header, const simplecpp::DUI &dui, bool systemheader) +bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) { - return !getFileIdPath(filedata, sourcefile, header, dui, systemheader).empty(); -} +#ifdef SIMPLECPP_WINDOWS + HANDLE hFile = CreateFileA(path.c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); -static void safeInsertTokenListToMap(std::map &filedata, const std::string &header2, simplecpp::TokenList *tokens, const std::string &header, const std::string &sourcefile, bool systemheader, const char* contextDesc) -{ - const bool inserted = filedata.insert(std::make_pair(header2, tokens)).second; - if (!inserted) { - std::cerr << "error in " << contextDesc << " - attempt to add a tokenized file to the file map, but this file is already in the map! Details:" << - "header: " << header << " header2: " << header2 << " source: " << sourcefile << " systemheader: " << systemheader << std::endl; - std::abort(); - } + if (hFile == INVALID_HANDLE_VALUE) + return false; + + const BOOL ret = GetFileInformationByHandleEx(hFile, FileIdInfo, &id.fileIdInfo, sizeof(id.fileIdInfo)); + + CloseHandle(hFile); + + return ret == TRUE; +#else + struct stat statbuf; + + if (stat(path.c_str(), &statbuf) != 0) + return false; + + id.dev = statbuf.st_dev; + id.ino = statbuf.st_ino; + + return true; +#endif } -std::map simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) +simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { #ifdef SIMPLECPP_WINDOWS if (dui.clearIncludeCache) nonExistingFilesCache.clear(); #endif - std::map ret; + FileDataCache cache; std::list filelist; // -include files for (std::list::const_iterator it = dui.includes.begin(); it != dui.includes.end(); ++it) { - const std::string &filename = realFilename(*it); + const std::string &filename = *it; - if (ret.find(filename) != ret.end()) - continue; + const auto loadResult = cache.get("", filename, dui, false, filenames, outputList); + const bool loaded = loadResult.second; + FileData *const filedata = loadResult.first; - std::ifstream fin(filename.c_str()); - if (!fin.is_open()) { + if (filedata == nullptr) { if (outputList) { simplecpp::Output err(filenames); err.type = simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND; @@ -3340,18 +3120,17 @@ std::map simplecpp::load(const simplecpp::To } continue; } - fin.close(); - TokenList *tokenlist = new TokenList(filename, filenames, outputList); - if (!tokenlist->front()) { - delete tokenlist; + if (!loaded) + continue; + + if (!filedata->tokens.front()) continue; - } if (dui.removeComments) - tokenlist->removeComments(); - ret[filename] = tokenlist; - filelist.push_back(tokenlist->front()); + filedata->tokens.removeComments(); + + filelist.push_back(filedata->tokens.front()); } for (const Token *rawtok = rawtokens.cfront(); rawtok || !filelist.empty(); rawtok = rawtok ? rawtok->next : nullptr) { @@ -3374,25 +3153,20 @@ std::map simplecpp::load(const simplecpp::To continue; const bool systemheader = (htok->str()[0] == '<'); - const std::string header(realFilename(htok->str().substr(1U, htok->str().size() - 2U))); - if (hasFile(ret, sourcefile, header, dui, systemheader)) - continue; + const std::string header(htok->str().substr(1U, htok->str().size() - 2U)); - std::ifstream f; - const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); - if (!f.is_open()) + FileData *const filedata = cache.get(sourcefile, header, dui, systemheader, filenames, outputList).first; + if (!filedata) continue; - f.close(); - TokenList *tokens = new TokenList(header2, filenames, outputList); if (dui.removeComments) - tokens->removeComments(); - safeInsertTokenListToMap(ret, header2, tokens, header, rawtok->location.file(), systemheader, "simplecpp::load"); - if (tokens->front()) - filelist.push_back(tokens->front()); + filedata->tokens.removeComments(); + + if (filedata->tokens.front()) + filelist.push_back(filedata->tokens.front()); } - return ret; + return cache; } static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) @@ -3448,7 +3222,7 @@ static std::string getTimeDefine(const struct tm *timep) return std::string("\"").append(buf).append("\""); } -void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, std::map &filedata, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage, std::list *ifCond) +void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenList &rawtokens, std::vector &files, simplecpp::FileDataCache &cache, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage, std::list *ifCond) { #ifdef SIMPLECPP_WINDOWS if (dui.clearIncludeCache) @@ -3548,9 +3322,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL includetokenstack.push(rawtokens.cfront()); for (std::list::const_iterator it = dui.includes.begin(); it != dui.includes.end(); ++it) { - const std::map::const_iterator f = filedata.find(*it); - if (f != filedata.end()) - includetokenstack.push(f->second->cfront()); + const FileData *const filedata = cache.get("", *it, dui, false, files, outputList).first; + if (filedata != nullptr && filedata->tokens.cfront() != nullptr) + includetokenstack.push(filedata->tokens.cfront()); } std::map > maybeUsedMacros; @@ -3681,21 +3455,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const Token * const inctok = inc2.cfront(); const bool systemheader = (inctok->str()[0] == '<'); - const std::string header(realFilename(inctok->str().substr(1U, inctok->str().size() - 2U))); - std::string header2 = getFileIdPath(filedata, rawtok->location.file(), header, dui, systemheader); - if (header2.empty()) { - // try to load file.. - std::ifstream f; - header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); - if (f.is_open()) { - f.close(); - TokenList * const tokens = new TokenList(header2, files, outputList); - if (dui.removeComments) - tokens->removeComments(); - safeInsertTokenListToMap(filedata, header2, tokens, header, rawtok->location.file(), systemheader, "simplecpp::preprocess"); - } - } - if (header2.empty()) { + const std::string header(inctok->str().substr(1U, inctok->str().size() - 2U)); + const FileData *const filedata = cache.get(rawtok->location.file(), header, dui, systemheader, files, outputList).first; + if (filedata == nullptr) { if (outputList) { simplecpp::Output out(files); out.type = Output::MISSING_HEADER; @@ -3711,10 +3473,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL out.msg = "#include nested too deeply"; outputList->push_back(out); } - } else if (pragmaOnce.find(header2) == pragmaOnce.end()) { + } else if (pragmaOnce.find(filedata->filename) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); - const TokenList * const includetokens = filedata.find(header2)->second; - rawtok = includetokens ? includetokens->cfront() : nullptr; + rawtok = filedata->tokens.cfront(); continue; } } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { @@ -3791,12 +3552,10 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (systemheader) { while ((tok = tok->next) && tok->op != '>') header += tok->str(); - // cppcheck-suppress selfAssignment - platform-dependent implementation - header = realFilename(header); if (tok && tok->op == '>') closingAngularBracket = true; } else { - header = realFilename(tok->str().substr(1U, tok->str().size() - 2U)); + header = tok->str().substr(1U, tok->str().size() - 2U); closingAngularBracket = true; } std::ifstream f; @@ -3956,11 +3715,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } -void simplecpp::cleanup(std::map &filedata) +void simplecpp::cleanup(FileDataCache &cache) { - for (std::map::iterator it = filedata.begin(); it != filedata.end(); ++it) - delete it->second; - filedata.clear(); + cache.clear(); } simplecpp::cstd_t simplecpp::getCStd(const std::string &std) diff --git a/simplecpp.h b/simplecpp.h index 579e6e14..76487d6c 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -6,13 +6,19 @@ #ifndef simplecppH #define simplecppH +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) +# define SIMPLECPP_WINDOWS +#endif + #include #include #include #include #include +#include #include #include +#include #include #ifdef _WIN32 @@ -27,6 +33,12 @@ # define SIMPLECPP_LIB #endif +#ifdef SIMPLECPP_WINDOWS +# include +#else +# include +#endif + #if defined(_MSC_VER) # pragma warning(push) // suppress warnings about "conversion from 'type1' to 'type2', possible loss of data" @@ -43,6 +55,7 @@ namespace simplecpp { typedef std::string TokenString; class Macro; + class FileDataCache; /** * Location in source code @@ -342,7 +355,7 @@ namespace simplecpp { SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); - SIMPLECPP_LIB std::map load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr); + SIMPLECPP_LIB FileDataCache load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr); /** * Preprocess @@ -350,18 +363,18 @@ namespace simplecpp { * @param output TokenList that receives the preprocessing output * @param rawtokens Raw tokenlist for top sourcefile * @param files internal data of simplecpp - * @param filedata output from simplecpp::load() + * @param cache output from simplecpp::load() * @param dui defines, undefs, and include paths * @param outputList output: list that will receive output messages * @param macroUsage output: macro usage * @param ifCond output: #if/#elif expressions */ - SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, std::map &filedata, const DUI &dui, OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr); + SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, FileDataCache &cache, const DUI &dui, OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr); /** * Deallocate data */ - SIMPLECPP_LIB void cleanup(std::map &filedata); + SIMPLECPP_LIB void cleanup(FileDataCache &cache); /** Simplify path */ SIMPLECPP_LIB std::string simplifyPath(std::string path); @@ -382,6 +395,116 @@ namespace simplecpp { /** Returns the __cplusplus value for a given standard */ SIMPLECPP_LIB std::string getCppStdString(const std::string &std); SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); + + struct SIMPLECPP_LIB FileData { + /** The canonical filename associated with this data */ + std::string filename; + /** The tokens associated with this file */ + TokenList tokens; + }; + + class SIMPLECPP_LIB FileDataCache { + public: + FileDataCache() = default; + + FileDataCache(const FileDataCache &) = delete; + FileDataCache(FileDataCache &&) = default; + + FileDataCache &operator=(const FileDataCache &) = delete; + FileDataCache &operator=(FileDataCache &&) = default; + + /** Get the cached data for a file, or load and then return it if it isn't cached. + * returns the file data and true if the file was loaded, false if it was cached. */ + std::pair get(const std::string &sourcefile, const std::string &header, const DUI &dui, bool systemheader, std::vector &filenames, OutputList *outputList); + + void insert(FileData data) { + FileData *const newdata = new FileData(std::move(data)); + + mData.emplace_back(newdata); + mNameMap.emplace(newdata->filename, newdata); + } + + void clear() { + mNameMap.clear(); + mIdMap.clear(); + mData.clear(); + } + + typedef std::vector> container_type; + typedef container_type::iterator iterator; + typedef container_type::const_iterator const_iterator; + typedef container_type::size_type size_type; + + size_type size() const { + return mData.size(); + } + iterator begin() { + return mData.begin(); + } + iterator end() { + return mData.end(); + } + const_iterator begin() const { + return mData.begin(); + } + const_iterator end() const { + return mData.end(); + } + const_iterator cbegin() const { + return mData.cbegin(); + } + const_iterator cend() const { + return mData.cend(); + } + + private: + struct FileID { +#ifdef SIMPLECPP_WINDOWS + struct { + std::uint64_t VolumeSerialNumber; + struct { + std::uint64_t IdentifierHi; + std::uint64_t IdentifierLo; + } FileId; + } fileIdInfo; + + bool operator==(const FileID &that) const noexcept { + return fileIdInfo.VolumeSerialNumber == that.fileIdInfo.VolumeSerialNumber && + fileIdInfo.FileId.IdentifierHi == that.fileIdInfo.FileId.IdentifierHi && + fileIdInfo.FileId.IdentifierLo == that.fileIdInfo.FileId.IdentifierLo; + } +#else + dev_t dev; + ino_t ino; + + bool operator==(const FileID& that) const noexcept { + return dev == that.dev && ino == that.ino; + } +#endif + struct Hasher { + std::size_t operator()(const FileID &id) const { +#ifdef SIMPLECPP_WINDOWS + return static_cast(id.fileIdInfo.FileId.IdentifierHi ^ id.fileIdInfo.FileId.IdentifierLo ^ + id.fileIdInfo.VolumeSerialNumber); +#else + return static_cast(id.dev) ^ static_cast(id.ino); +#endif + } + }; + }; + + using name_map_type = std::unordered_map; + using id_map_type = std::unordered_map; + + static bool getFileId(const std::string &path, FileID &id); + + std::pair tryload(name_map_type::iterator &name_it, const DUI &dui, std::vector &filenames, OutputList *outputList); + + container_type mData; + name_map_type mNameMap; + id_map_type mIdMap; + + }; } #if defined(_MSC_VER) diff --git a/test.cpp b/test.cpp index ba21d71b..e1968569 100644 --- a/test.cpp +++ b/test.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -100,13 +99,12 @@ static std::string readfile(const char code[], std::size_t size, simplecpp::Outp static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList) { std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokens = makeTokenList(code,files); tokens.removeComments(); simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, tokens, files, filedata, dui, outputList); - for (auto &i : filedata) - delete i.second; + simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList); + simplecpp::cleanup(cache); return tokens2.stringify(); } @@ -1077,11 +1075,11 @@ static void error4() // "#error x\n1" const char code[] = "\xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtoken = makeTokenList(code, sizeof(code),files,"test.c"); - simplecpp::preprocess(tokens2, rawtoken, files, filedata, simplecpp::DUI(), &outputList); + simplecpp::preprocess(tokens2, rawtoken, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -1090,11 +1088,11 @@ static void error5() // "#error x\n1" const char code[] = "\xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code, sizeof(code),files,"test.c"); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); + simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI(), &outputList); ASSERT_EQUALS("file0,1,#error,#error x\n", toString(outputList)); } @@ -2001,12 +1999,14 @@ static void missingHeader2() { const char code[] = "#include \"foo.h\"\n"; // this file exists std::vector files; - std::map filedata; - filedata["foo.h"] = nullptr; + simplecpp::FileDataCache cache; + cache.insert({"foo.h", simplecpp::TokenList(files)}); simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -2030,13 +2030,15 @@ static void nestedInclude() { const char code[] = "#include \"test.h\"\n"; std::vector files; - simplecpp::TokenList rawtokens = makeTokenList(code,files,"test.h"); - std::map filedata; - filedata["test.h"] = &rawtokens; + const simplecpp::TokenList rawtokens = makeTokenList(code,files,"test.h"); + simplecpp::FileDataCache cache; + cache.insert({"test.h", rawtokens}); simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); } @@ -2045,14 +2047,16 @@ static void systemInclude() { const char code[] = "#include \n"; std::vector files; - simplecpp::TokenList rawtokens = makeTokenList(code,files,"local/limits.h"); - std::map filedata; - filedata["limits.h"] = nullptr; - filedata["local/limits.h"] = &rawtokens; + const simplecpp::TokenList rawtokens = makeTokenList(code,files,"local/limits.h"); + simplecpp::FileDataCache cache; + cache.insert({"include/limits.h", simplecpp::TokenList(files)}); + cache.insert({"local/limits.h", rawtokens}); simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI(), &outputList); + simplecpp::DUI dui; + dui.includePaths.push_back("include"); + simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -2074,9 +2078,9 @@ static void multiline2() simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /**/ 1\n\nA", rawtokens.stringify()); rawtokens.removeComments(); - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -2089,9 +2093,9 @@ static void multiline3() // #28 - macro with multiline comment simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /* */ 1\n\nA", rawtokens.stringify()); rawtokens.removeComments(); - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n1", tokens2.stringify()); } @@ -2105,9 +2109,9 @@ static void multiline4() // #28 - macro with multiline comment simplecpp::TokenList rawtokens = makeTokenList(code,files); ASSERT_EQUALS("# define A /* */ 1\n\n\nA", rawtokens.stringify()); rawtokens.removeComments(); - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokens2, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("\n\n\n1", tokens2.stringify()); } @@ -2221,19 +2225,21 @@ static void include3() // #16 - crash when expanding macro from header std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("A.c", files[0]); ASSERT_EQUALS("A.h", files[1]); - std::map filedata; - filedata["A.c"] = &rawtokens_c; - filedata["A.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"A.c", rawtokens_c}); + cache.insert({"A.h", rawtokens_h}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens_c, files, filedata, simplecpp::DUI()); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n1234", out.stringify()); } @@ -2246,21 +2252,22 @@ static void include4() // #27 - -include std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "27.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "27.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "27.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "27.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("27.c", files[0]); ASSERT_EQUALS("27.h", files[1]); - std::map filedata; - filedata["27.c"] = &rawtokens_c; - filedata["27.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"27.c", rawtokens_c}); + cache.insert({"27.h", rawtokens_h}); simplecpp::TokenList out(files); simplecpp::DUI dui; + dui.includePaths.push_back("."); dui.includes.push_back("27.h"); - simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("123", out.stringify()); } @@ -2272,19 +2279,21 @@ static void include5() // #3 - handle #include MACRO std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("3.c", files[0]); ASSERT_EQUALS("3.h", files[1]); - std::map filedata; - filedata["3.c"] = &rawtokens_c; - filedata["3.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"3.c", rawtokens_c}); + cache.insert({"3.h", rawtokens_h}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens_c, files, filedata, simplecpp::DUI()); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -2295,16 +2304,16 @@ static void include6() // #57 - incomplete macro #include MACRO(,) std::vector files; - simplecpp::TokenList rawtokens = makeTokenList(code, files, "57.c"); + const simplecpp::TokenList rawtokens = makeTokenList(code, files, "57.c"); ASSERT_EQUALS(1U, files.size()); ASSERT_EQUALS("57.c", files[0]); - std::map filedata; - filedata["57.c"] = &rawtokens; + simplecpp::FileDataCache cache; + cache.insert({"57.c", rawtokens}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(out, rawtokens, files, cache, simplecpp::DUI()); } @@ -2316,21 +2325,21 @@ static void include7() // #include MACRO std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "3.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "3.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("3.c", files[0]); ASSERT_EQUALS("3.h", files[1]); - std::map filedata; - filedata["3.c"] = &rawtokens_c; - filedata["3.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"3.c", rawtokens_c}); + cache.insert({"3.h", rawtokens_h}); simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.push_back("."); - simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); } @@ -2354,21 +2363,21 @@ static void include9() std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "1.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "1.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "1.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "1.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("1.c", files[0]); ASSERT_EQUALS("1.h", files[1]); - std::map filedata; - filedata["1.c"] = &rawtokens_c; - filedata["1.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"1.c", rawtokens_c}); + cache.insert({"1.h", rawtokens_h}); simplecpp::TokenList out(files); simplecpp::DUI dui; dui.includePaths.push_back("."); - simplecpp::preprocess(out, rawtokens_c, files, filedata, dui); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify()); } @@ -2536,19 +2545,21 @@ static void stringify1() std::vector files; - simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); - simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); + const simplecpp::TokenList rawtokens_c = makeTokenList(code_c, files, "A.c"); + const simplecpp::TokenList rawtokens_h = makeTokenList(code_h, files, "A.h"); ASSERT_EQUALS(2U, files.size()); ASSERT_EQUALS("A.c", files[0]); ASSERT_EQUALS("A.h", files[1]); - std::map filedata; - filedata["A.c"] = &rawtokens_c; - filedata["A.h"] = &rawtokens_h; + simplecpp::FileDataCache cache; + cache.insert({"A.c", rawtokens_c}); + cache.insert({"A.h", rawtokens_h}); simplecpp::TokenList out(files); - simplecpp::preprocess(out, rawtokens_c, files, filedata, simplecpp::DUI()); + simplecpp::DUI dui; + dui.includePaths.push_back("."); + simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); } @@ -2558,10 +2569,10 @@ static void tokenMacro1() const char code[] = "#define A 123\n" "A"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); ASSERT_EQUALS("A", tokenList.cback()->macro); } @@ -2570,10 +2581,10 @@ static void tokenMacro2() const char code[] = "#define ADD(X,Y) X+Y\n" "ADD(1,2)"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("", tok->macro); @@ -2591,10 +2602,10 @@ static void tokenMacro3() "#define FRED 1\n" "ADD(FRED,2)"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); const simplecpp::Token *tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("FRED", tok->macro); @@ -2612,10 +2623,10 @@ static void tokenMacro4() "#define B 1\n" "A"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); const simplecpp::Token * const tok = tokenList.cfront(); ASSERT_EQUALS("1", tok->str()); ASSERT_EQUALS("A", tok->macro); @@ -2627,10 +2638,10 @@ static void tokenMacro5() "#define SET_BPF_JUMP(code) SET_BPF(D | code)\n" "SET_BPF_JUMP(A | B | C);"; std::vector files; - std::map filedata; + simplecpp::FileDataCache cache; simplecpp::TokenList tokenList(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); - simplecpp::preprocess(tokenList, rawtokens, files, filedata, simplecpp::DUI()); + simplecpp::preprocess(tokenList, rawtokens, files, cache, simplecpp::DUI()); const simplecpp::Token * const tok = tokenList.cfront()->next; ASSERT_EQUALS("D", tok->str()); ASSERT_EQUALS("SET_BPF_JUMP", tok->macro); @@ -3064,8 +3075,8 @@ static void preprocess_files() ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("", *files.cbegin()); - std::map filedata; - simplecpp::preprocess(tokens2, tokens, files, filedata, simplecpp::DUI(), nullptr); + simplecpp::FileDataCache cache; + simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("", *files.cbegin()); } @@ -3081,8 +3092,8 @@ static void preprocess_files() ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("test.cpp", *files.cbegin()); - std::map filedata; - simplecpp::preprocess(tokens2, tokens, files, filedata, simplecpp::DUI(), nullptr); + simplecpp::FileDataCache cache; + simplecpp::preprocess(tokens2, tokens, files, cache, simplecpp::DUI(), nullptr); ASSERT_EQUALS(1, files.size()); ASSERT_EQUALS("test.cpp", *files.cbegin()); } From 6dd82cb039f669f83150aff400721b8432134b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 8 Jul 2025 16:42:31 +0200 Subject: [PATCH 276/381] fixed #466 - CI-unixish.yml: added missing `UBSAN_OPTIONS` (#467) --- .github/workflows/CI-unixish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index f5a78ea5..ff7c3f65 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -101,6 +101,8 @@ jobs: run: | make clean make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + env: + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1:report_error_type=1 # TODO: requires instrumented libc++ - name: Run MemorySanitizer From bd068aeaae1414104458b9fd79911dfb8462ebdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 14 Jul 2025 12:12:20 +0200 Subject: [PATCH 277/381] fixed #464 - added integration test to `test` make target (#465) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 73977517..4a6ae6b7 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ testrunner: test.o simplecpp.o test: testrunner simplecpp ./testrunner python3 run-tests.py + python3 -m pytest integration_test.py -vv selfcheck: simplecpp ./selfcheck.sh From 5bf471de82684afe7af168739c037fbf6a248d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 14 Jul 2025 12:40:45 +0200 Subject: [PATCH 278/381] added test for #346 (#457) --- test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test.cpp b/test.cpp index e1968569..3c5f5488 100644 --- a/test.cpp +++ b/test.cpp @@ -3106,6 +3106,11 @@ static void fuzz_crash() "n\n"; (void)preprocess(code, simplecpp::DUI()); // do not crash } + { // #346 + const char code[] = "#define foo(intp)f##oo(intp\n" + "foo(f##oo(intp))\n"; + (void)preprocess(code, simplecpp::DUI()); // do not crash + } } int main(int argc, char **argv) From 29abbc6bdb29eb72d37892dab776e48e9998d8bc Mon Sep 17 00:00:00 2001 From: clock999 Date: Tue, 15 Jul 2025 19:50:26 +0800 Subject: [PATCH 279/381] fix #337 - line splicing in comment not handled properly (#431) --- simplecpp.cpp | 33 +++++++++++++++++++++++++-------- test.cpp | 25 ++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 128da0bd..721bbdeb 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -758,17 +758,34 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, // comment else if (ch == '/' && stream.peekChar() == '/') { - while (stream.good() && ch != '\r' && ch != '\n') { + while (stream.good() && ch != '\n') { currentToken += ch; ch = stream.readChar(); + if(ch == '\\') { + TokenString tmp; + char tmp_ch = ch; + while((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { + tmp += tmp_ch; + tmp_ch = stream.readChar(); + } + if(!stream.good()) { + break; + } + + if(tmp_ch != '\n') { + currentToken += tmp; + } else { + TokenString check_portability = currentToken + tmp; + const std::string::size_type pos = check_portability.find_last_not_of(" \t"); + if (pos < check_portability.size() - 1U && check_portability[pos] == '\\') + portabilityBackslash(outputList, files, location); + ++multiline; + tmp_ch = stream.readChar(); + } + ch = tmp_ch; + } } - const std::string::size_type pos = currentToken.find_last_not_of(" \t"); - if (pos < currentToken.size() - 1U && currentToken[pos] == '\\') - portabilityBackslash(outputList, files, location); - if (currentToken[currentToken.size() - 1U] == '\\') { - ++multiline; - currentToken.erase(currentToken.size() - 1U); - } else { + if (ch == '\n') { stream.ungetChar(); } } diff --git a/test.cpp b/test.cpp index 3c5f5488..a7578823 100644 --- a/test.cpp +++ b/test.cpp @@ -434,7 +434,30 @@ static void comment_multiline() const char code[] = "#define ABC {// \\\n" "}\n" "void f() ABC\n"; - ASSERT_EQUALS("\n\nvoid f ( ) { }", preprocess(code)); + ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code)); + + const char code1[] = "#define ABC {// \\\r\n" + "}\n" + "void f() ABC\n"; + ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code1)); + + const char code2[] = "#define A 1// \\\r" + "\r" + "2\r" + "A\r"; + ASSERT_EQUALS("\n\n2\n1", preprocess(code2)); + + const char code3[] = "void f() {// \\ \n}\n"; + ASSERT_EQUALS("void f ( ) {", preprocess(code3)); + + const char code4[] = "void f() {// \\\\\\\t\t\n}\n"; + ASSERT_EQUALS("void f ( ) {", preprocess(code4)); + + const char code5[] = "void f() {// \\\\\\a\n}\n"; + ASSERT_EQUALS("void f ( ) {\n}", preprocess(code5)); + + const char code6[] = "void f() {// \\\n\n\n}\n"; + ASSERT_EQUALS("void f ( ) {\n\n\n}", preprocess(code6)); } From d0f2b99d656cb6dab6fc99a105a6c59cbcfbb13a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 3 Aug 2025 07:42:06 +0200 Subject: [PATCH 280/381] Fix #471 (preserve line splicing information in '// ..' comments) (#472) --- simplecpp.cpp | 3 ++- test.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 721bbdeb..5093b4b7 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -775,12 +775,13 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if(tmp_ch != '\n') { currentToken += tmp; } else { - TokenString check_portability = currentToken + tmp; + const TokenString check_portability = currentToken + tmp; const std::string::size_type pos = check_portability.find_last_not_of(" \t"); if (pos < check_portability.size() - 1U && check_portability[pos] == '\\') portabilityBackslash(outputList, files, location); ++multiline; tmp_ch = stream.readChar(); + currentToken += '\n'; } ch = tmp_ch; } diff --git a/test.cpp b/test.cpp index a7578823..de9f250b 100644 --- a/test.cpp +++ b/test.cpp @@ -458,6 +458,9 @@ static void comment_multiline() const char code6[] = "void f() {// \\\n\n\n}\n"; ASSERT_EQUALS("void f ( ) {\n\n\n}", preprocess(code6)); + + // #471 ensure there is newline in comment so that line-splicing can be detected by tools + ASSERT_EQUALS("// abc\ndef", readfile("// abc\\\ndef")); } From 435a74cc192e64499ddf96193becf8073c50376c Mon Sep 17 00:00:00 2001 From: glankk Date: Mon, 4 Aug 2025 15:14:18 +0200 Subject: [PATCH 281/381] Fix #391 (`__TIME__` replacement might be empty depending on compiler) (#441) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5093b4b7..addac46f 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3236,7 +3236,7 @@ static std::string getDateDefine(const struct tm *timep) static std::string getTimeDefine(const struct tm *timep) { char buf[] = "??:??:??"; - strftime(buf, sizeof(buf), "%T", timep); + strftime(buf, sizeof(buf), "%H:%M:%S", timep); return std::string("\"").append(buf).append("\""); } From 2b4f727da30c87ef2de79cfe81760e3b1b2ca772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 11 Aug 2025 15:59:06 +0200 Subject: [PATCH 282/381] simplecpp.cpp: fixed Visual Studio C4800 compiler warnings (#481) --- simplecpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index addac46f..25d0d3c3 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -745,7 +745,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, // number or name if (isNameChar(ch)) { - const bool num = std::isdigit(ch); + const bool num = !!std::isdigit(ch); while (stream.good() && isNameChar(ch)) { currentToken += ch; ch = stream.readChar(); @@ -886,7 +886,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } if (prefix.empty()) - push_back(new Token(s, location, std::isspace(stream.peekChar()))); // push string without newlines + push_back(new Token(s, location, !!std::isspace(stream.peekChar()))); // push string without newlines else back()->setstr(prefix + s); @@ -916,7 +916,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } } - push_back(new Token(currentToken, location, std::isspace(stream.peekChar()))); + push_back(new Token(currentToken, location, !!std::isspace(stream.peekChar()))); if (multiline) location.col += currentToken.size(); From 5783afac7dded04a5e4bb2c9b6b6b593ea2a4c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 11 Aug 2025 16:00:16 +0200 Subject: [PATCH 283/381] Makefile: added `CXXOPTS` and `LDOPTS` to extend `CXXFLAGS` and `LDFLAGS` (#480) --- .github/workflows/CI-unixish.yml | 10 +++++----- Makefile | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index ff7c3f65..cb498b5e 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -80,19 +80,19 @@ jobs: if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-g3 -D_GLIBCXX_DEBUG" + make -j$(nproc) test selfcheck CXXOPTS="-g3 -D_GLIBCXX_DEBUG" - name: Run with libc++ hardening mode if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDFLAGS="-lc++" + make -j$(nproc) test selfcheck CXXOPTS="-stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" - name: Run AddressSanitizer if: matrix.os == 'ubuntu-24.04' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address" + make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -fsanitize=address" LDOPTS="-fsanitize=address" env: ASAN_OPTIONS: detect_stack_use_after_return=1 @@ -100,7 +100,7 @@ jobs: if: matrix.os == 'ubuntu-24.04' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDOPTS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" env: UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1:report_error_type=1 @@ -109,4 +109,4 @@ jobs: if: false && matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory" + make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDOPTS="-lc++ -fsanitize=memory" diff --git a/Makefile b/Makefile index 4a6ae6b7..d899d2cd 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: testrunner simplecpp -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++11 -g -LDFLAGS = -g +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++11 -g $(CXXOPTS) +LDFLAGS = -g $(LDOPTS) %.o: %.cpp simplecpp.h $(CXX) $(CXXFLAGS) -c $< From 1678b7d229a7bcf833055766afa1496d68e1397c Mon Sep 17 00:00:00 2001 From: glankk Date: Thu, 14 Aug 2025 17:53:41 +0200 Subject: [PATCH 284/381] Remove execute bit from simplecpp.cpp/h (#494) --- simplecpp.cpp | 0 simplecpp.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 simplecpp.cpp mode change 100755 => 100644 simplecpp.h diff --git a/simplecpp.cpp b/simplecpp.cpp old mode 100755 new mode 100644 diff --git a/simplecpp.h b/simplecpp.h old mode 100755 new mode 100644 From f790009b5f19a20b6254c03e8d02c8d3e60f1244 Mon Sep 17 00:00:00 2001 From: glankk Date: Thu, 21 Aug 2025 11:12:21 +0200 Subject: [PATCH 285/381] Fix infinite loop with circular includes (#497) --- simplecpp.cpp | 18 ++++++----- simplecpp.h | 87 +++++++++++++++++++++++++-------------------------- test.cpp | 45 ++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 51 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 25d0d3c3..fd327549 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3109,15 +3109,13 @@ bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) #endif } -simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList) +simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, std::vector &filenames, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, FileDataCache cache) { #ifdef SIMPLECPP_WINDOWS if (dui.clearIncludeCache) nonExistingFilesCache.clear(); #endif - FileDataCache cache; - std::list filelist; // -include files @@ -3173,15 +3171,21 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, const bool systemheader = (htok->str()[0] == '<'); const std::string header(htok->str().substr(1U, htok->str().size() - 2U)); - FileData *const filedata = cache.get(sourcefile, header, dui, systemheader, filenames, outputList).first; - if (!filedata) + const auto loadResult = cache.get(sourcefile, header, dui, systemheader, filenames, outputList); + const bool loaded = loadResult.second; + + if (!loaded) + continue; + + FileData *const filedata = loadResult.first; + + if (!filedata->tokens.front()) continue; if (dui.removeComments) filedata->tokens.removeComments(); - if (filedata->tokens.front()) - filelist.push_back(filedata->tokens.front()); + filelist.push_back(filedata->tokens.front()); } return cache; diff --git a/simplecpp.h b/simplecpp.h index 76487d6c..8268fa8d 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -353,49 +353,6 @@ namespace simplecpp { bool removeComments; /** remove comment tokens from included files */ }; - SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); - - SIMPLECPP_LIB FileDataCache load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr); - - /** - * Preprocess - * @todo simplify interface - * @param output TokenList that receives the preprocessing output - * @param rawtokens Raw tokenlist for top sourcefile - * @param files internal data of simplecpp - * @param cache output from simplecpp::load() - * @param dui defines, undefs, and include paths - * @param outputList output: list that will receive output messages - * @param macroUsage output: macro usage - * @param ifCond output: #if/#elif expressions - */ - SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, FileDataCache &cache, const DUI &dui, OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr); - - /** - * Deallocate data - */ - SIMPLECPP_LIB void cleanup(FileDataCache &cache); - - /** Simplify path */ - SIMPLECPP_LIB std::string simplifyPath(std::string path); - - /** Convert Cygwin path to Windows path */ - SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); - - /** Returns the C version a given standard */ - SIMPLECPP_LIB cstd_t getCStd(const std::string &std); - - /** Returns the C++ version a given standard */ - SIMPLECPP_LIB cppstd_t getCppStd(const std::string &std); - - /** Returns the __STDC_VERSION__ value for a given standard */ - SIMPLECPP_LIB std::string getCStdString(const std::string &std); - SIMPLECPP_LIB std::string getCStdString(cstd_t std); - - /** Returns the __cplusplus value for a given standard */ - SIMPLECPP_LIB std::string getCppStdString(const std::string &std); - SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); - struct SIMPLECPP_LIB FileData { /** The canonical filename associated with this data */ std::string filename; @@ -503,8 +460,50 @@ namespace simplecpp { container_type mData; name_map_type mNameMap; id_map_type mIdMap; - }; + + SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); + + SIMPLECPP_LIB FileDataCache load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr, FileDataCache cache = {}); + + /** + * Preprocess + * @todo simplify interface + * @param output TokenList that receives the preprocessing output + * @param rawtokens Raw tokenlist for top sourcefile + * @param files internal data of simplecpp + * @param cache output from simplecpp::load() + * @param dui defines, undefs, and include paths + * @param outputList output: list that will receive output messages + * @param macroUsage output: macro usage + * @param ifCond output: #if/#elif expressions + */ + SIMPLECPP_LIB void preprocess(TokenList &output, const TokenList &rawtokens, std::vector &files, FileDataCache &cache, const DUI &dui, OutputList *outputList = nullptr, std::list *macroUsage = nullptr, std::list *ifCond = nullptr); + + /** + * Deallocate data + */ + SIMPLECPP_LIB void cleanup(FileDataCache &cache); + + /** Simplify path */ + SIMPLECPP_LIB std::string simplifyPath(std::string path); + + /** Convert Cygwin path to Windows path */ + SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); + + /** Returns the C version a given standard */ + SIMPLECPP_LIB cstd_t getCStd(const std::string &std); + + /** Returns the C++ version a given standard */ + SIMPLECPP_LIB cppstd_t getCppStd(const std::string &std); + + /** Returns the __STDC_VERSION__ value for a given standard */ + SIMPLECPP_LIB std::string getCStdString(const std::string &std); + SIMPLECPP_LIB std::string getCStdString(cstd_t std); + + /** Returns the __cplusplus value for a given standard */ + SIMPLECPP_LIB std::string getCppStdString(const std::string &std); + SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); } #if defined(_MSC_VER) diff --git a/test.cpp b/test.cpp index de9f250b..ccb653ca 100644 --- a/test.cpp +++ b/test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #define STRINGIZE_(x) #x @@ -2087,6 +2088,49 @@ static void systemInclude() ASSERT_EQUALS("", toString(outputList)); } +static void circularInclude() +{ + std::vector files; + simplecpp::FileDataCache cache; + + { + const char *const path = "test.h"; + const char code[] = + "#ifndef TEST_H\n" + "#define TEST_H\n" + "#include \"a/a.h\"\n" + "#endif\n" + ; + cache.insert({path, makeTokenList(code, files, path)}); + } + + { + const char *const path = "a/a.h"; + const char code[] = + "#ifndef A_H\n" + "#define A_H\n" + "#include \"../test.h\"\n" + "#endif\n" + ; + cache.insert({path, makeTokenList(code, files, path)}); + } + + simplecpp::OutputList outputList; + simplecpp::TokenList tokens2(files); + { + std::vector filenames; + const simplecpp::DUI dui; + + const char code[] = "#include \"test.h\"\n"; + const simplecpp::TokenList rawtokens = makeTokenList(code, files, "test.cpp"); + + cache = simplecpp::load(rawtokens, filenames, dui, &outputList, std::move(cache)); + simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); + } + + ASSERT_EQUALS("", toString(outputList)); +} + static void multiline1() { const char code[] = "#define A \\\n" @@ -3314,6 +3358,7 @@ int main(int argc, char **argv) TEST_CASE(missingHeader4); TEST_CASE(nestedInclude); TEST_CASE(systemInclude); + TEST_CASE(circularInclude); TEST_CASE(nullDirective1); TEST_CASE(nullDirective2); From fead0b280a3b242b15191c3ebaefc42b6159eb16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 22 Aug 2025 15:18:19 +0200 Subject: [PATCH 286/381] CI-unixish.yml: removed duplicated execution of integration test (#503) --- .github/workflows/CI-unixish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index cb498b5e..c343e279 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -51,10 +51,6 @@ jobs: run: | make -j$(nproc) selfcheck - - name: integration test - run: | - python3 -m pytest integration_test.py -vv - - name: Run CMake run: | cmake -S . -B cmake.output From 7bca11f0ec435df3c9ccefd37c9a21a3c575355b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 22 Aug 2025 18:24:41 +0200 Subject: [PATCH 287/381] CI-unixish.yml: do not run with `g++` on `macos-*` as it is just an alias for `clang++` (#483) --- .github/workflows/CI-unixish.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index c343e279..718414d8 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -7,8 +7,13 @@ jobs: strategy: matrix: - compiler: [clang++, g++] os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14, macos-15] + compiler: [clang++] + include: + - os: ubuntu-22.04 + compiler: g++ + - os: ubuntu-24.04 + compiler: g++ fail-fast: false runs-on: ${{ matrix.os }} From 285998182edd0280a9c1b5fe877f074fe441fd16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 22 Aug 2025 20:51:46 +0200 Subject: [PATCH 288/381] fixed #478 - fail builds in CI on compiler warnings (#479) --- .github/workflows/CI-unixish.yml | 16 ++++++++-------- .github/workflows/CI-windows.yml | 2 +- .github/workflows/clang-tidy.yml | 2 +- CMakeLists.txt | 4 ++++ appveyor.yml | 3 ++- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 718414d8..c80aaba2 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -47,10 +47,10 @@ jobs: python3 -m pip install pytest - name: make simplecpp - run: make -j$(nproc) + run: make -j$(nproc) CXXOPTS="-Werror" - name: make test - run: make -j$(nproc) test + run: make -j$(nproc) test CXXOPTS="-Werror" - name: selfcheck run: | @@ -58,7 +58,7 @@ jobs: - name: Run CMake run: | - cmake -S . -B cmake.output + cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On - name: CMake simplecpp run: | @@ -81,19 +81,19 @@ jobs: if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++' run: | make clean - make -j$(nproc) test selfcheck CXXOPTS="-g3 -D_GLIBCXX_DEBUG" + make -j$(nproc) test selfcheck CXXOPTS="-Werror -g3 -D_GLIBCXX_DEBUG" - name: Run with libc++ hardening mode if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXXOPTS="-stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" + make -j$(nproc) test selfcheck CXXOPTS="-Werror -stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" - name: Run AddressSanitizer if: matrix.os == 'ubuntu-24.04' run: | make clean - make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -fsanitize=address" LDOPTS="-fsanitize=address" + make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=address" LDOPTS="-fsanitize=address" env: ASAN_OPTIONS: detect_stack_use_after_return=1 @@ -101,7 +101,7 @@ jobs: if: matrix.os == 'ubuntu-24.04' run: | make clean - make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDOPTS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" + make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDOPTS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" env: UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1:report_error_type=1 @@ -110,4 +110,4 @@ jobs: if: false && matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' run: | make clean - make -j$(nproc) test selfcheck CXXOPTS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDOPTS="-lc++ -fsanitize=memory" + make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -stdlib=libc++ -fsanitize=memory" LDOPTS="-lc++ -fsanitize=memory" diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 971f3827..d4c99388 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -40,7 +40,7 @@ jobs: - name: Run CMake run: | - cmake -G "Visual Studio 17 2022" -A x64 . || exit /b !errorlevel! + cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_COMPILE_WARNING_AS_ERROR=On . || exit /b !errorlevel! - name: Build run: | diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index a2f7b6dc..41d2ee6f 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -30,7 +30,7 @@ jobs: - name: Prepare CMake run: | - cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: CXX: clang-20 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab0166e..f9e3eb6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,10 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options_safe(-Wuseless-cast) elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") add_compile_definitions(_CRT_SECURE_NO_WARNINGS) + # TODO: bump warning level + #add_compile_options(/W4) # Warning Level + # TODO: enable warning + add_compile_options(/wd4267) # warning C4267: '...': conversion from 'size_t' to 'unsigned int', possible loss of data elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Weverything) # no need for c++98 compatibility diff --git a/appveyor.yml b/appveyor.yml index ea8dd1df..09aa6cbe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,9 +10,10 @@ environment: build_script: - ECHO Building %configuration% %platform% with MSVC %VisualStudioVersion% using %PlatformToolset% PlatformToolset - - cmake -G "Visual Studio 14" . + - cmake -DCMAKE_COMPILE_WARNING_AS_ERROR=On -G "Visual Studio 14" . - dir - 'CALL "C:\Program Files (x86)\Microsoft Visual Studio %VisualStudioVersion%\VC\vcvarsall.bat" %vcvarsall_platform%' + - set _CL_=/WX - msbuild "simplecpp.sln" /consoleloggerparameters:Verbosity=minimal /target:Build /property:Configuration="%configuration%";Platform=%platform% /p:PlatformToolset=%PlatformToolset% /maxcpucount /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" test_script: From 538c5c4cd8baf806835c0d51ce2a9814ca883a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 23 Aug 2025 10:47:23 +0200 Subject: [PATCH 289/381] fixed #485 - addressed zizmor findings in GitHub Actions (#486) --- .github/workflows/CI-unixish.yml | 5 +++++ .github/workflows/CI-windows.yml | 5 +++++ .github/workflows/clang-tidy.yml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index c80aaba2..60361389 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -2,6 +2,9 @@ name: CI-unixish on: [push, pull_request] +permissions: + contents: read + jobs: build: @@ -23,6 +26,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Install missing software on ubuntu if: matrix.os == 'ubuntu-24.04' diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index d4c99388..767bba6c 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -6,6 +6,9 @@ name: CI-windows on: [push,pull_request] +permissions: + contents: read + defaults: run: shell: cmd @@ -23,6 +26,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 41d2ee6f..fd71bdfe 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -4,6 +4,9 @@ name: clang-tidy on: [push, pull_request] +permissions: + contents: read + jobs: build: @@ -11,6 +14,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Install missing software run: | From 5fcb07307bd08226831667a85984bd5a392fc640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 28 Aug 2025 17:34:20 +0200 Subject: [PATCH 290/381] fixed #476 - fixed MSYS2 compilation with MSYS msystem (#513) --- simplecpp.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index fd327549..74ab66e1 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2385,12 +2385,17 @@ namespace simplecpp { namespace simplecpp { #ifdef __CYGWIN__ + static bool startsWith(const std::string &s, const std::string &p) + { + return (s.size() >= p.size()) && std::equal(p.begin(), p.end(), s.begin()); + } + std::string convertCygwinToWindowsPath(const std::string &cygwinPath) { std::string windowsPath; std::string::size_type pos = 0; - if (cygwinPath.size() >= 11 && startsWith_(cygwinPath, "/cygdrive/")) { + if (cygwinPath.size() >= 11 && startsWith(cygwinPath, "/cygdrive/")) { const unsigned char driveLetter = cygwinPath[10]; if (std::isalpha(driveLetter)) { if (cygwinPath.size() == 11) { From cfd179711f413aa8e0da9c2f437ad4f8938d5f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Thu, 28 Aug 2025 19:49:29 +0200 Subject: [PATCH 291/381] Switch to uncrustify (#517) --- .uncrustify.cfg | 170 ++++++++++++++++++++++++++++++++++++++++++++++++ runastyle | 23 ------- runastyle.bat | 26 -------- runformat | 39 +++++++++++ simplecpp.cpp | 18 ++--- simplecpp.h | 9 ++- test.cpp | 24 +++---- 7 files changed, 234 insertions(+), 75 deletions(-) create mode 100644 .uncrustify.cfg delete mode 100755 runastyle delete mode 100644 runastyle.bat create mode 100755 runformat diff --git a/.uncrustify.cfg b/.uncrustify.cfg new file mode 100644 index 00000000..81722ff7 --- /dev/null +++ b/.uncrustify.cfg @@ -0,0 +1,170 @@ +# Uncrustify-0.80.1_f + +# The original size of tabs in the input. +# +# Default: 8 +input_tab_size = 4 # unsigned number + +# The size of tabs in the output (only used if align_with_tabs=true). +# +# Default: 8 +output_tab_size = 4 # unsigned number + +# Add or remove space between 'while' and '('. +sp_while_paren_open = add # ignore/add/remove/force + +# Add or remove space around boolean operators '&&' and '||'. +sp_bool = force # ignore/add/remove/force + +# Add or remove space inside '(' and ')'. +sp_inside_paren = remove # ignore/add/remove/force + +# Add or remove space between nested parentheses, i.e. '((' vs. ') )'. +sp_paren_paren = remove # ignore/add/remove/force + +# Add or remove space between ')' and '{'. +sp_paren_brace = force # ignore/add/remove/force + +# Add or remove space between pointer stars '*'. +sp_between_ptr_star = remove # ignore/add/remove/force + +# Add or remove space before '<'. +sp_before_angle = remove # ignore/add/remove/force + +# Add or remove space inside '<' and '>'. +sp_inside_angle = remove # ignore/add/remove/force + +# Add or remove space after '>'. +sp_after_angle = add # ignore/add/remove/force + +# Add or remove space between '>' and '(' as found in 'new List(foo);'. +sp_angle_paren = remove # ignore/add/remove/force + +# Add or remove space between '>' and a word as in 'List m;' or +# 'template static ...'. +sp_angle_word = add # ignore/add/remove/force + +# Add or remove space between '>' and '>' in '>>' (template stuff). +# +# Default: add +sp_angle_shift = ignore # ignore/add/remove/force + +# (C++11) Permit removal of the space between '>>' in 'foo >'. Note +# that sp_angle_shift cannot remove the space without this option. +sp_permit_cpp11_shift = true # true/false + +# Add or remove space before '(' of control statements ('if', 'for', 'switch', +# 'while', etc.). +sp_before_sparen = force # ignore/add/remove/force + +# Add or remove space inside '(' and ')' of control statements. +sp_inside_sparen = remove # ignore/add/remove/force + +# Add or remove space after ')' of control statements. +sp_after_sparen = force # ignore/add/remove/force + +# Add or remove space between ')' and '{' of of control statements. +sp_sparen_brace = force # ignore/add/remove/force + +# Add or remove space before ';' in non-empty 'for' statements. +sp_before_semi_for = remove # ignore/add/remove/force + +# Add or remove space after the final semicolon of an empty part of a for +# statement, as in 'for ( ; ; )'. +sp_after_semi_for_empty = remove # ignore/add/remove/force + +# Add or remove space before '[]'. +sp_before_squares = remove # ignore/add/remove/force + +# Add or remove space before C++17 structured bindings. +sp_cpp_before_struct_binding = ignore # ignore/add/remove/force + +# Add or remove space inside a non-empty '[' and ']'. +sp_inside_square = remove # ignore/add/remove/force + +# Add or remove space after class ':'. +sp_after_class_colon = force # ignore/add/remove/force + +# Add or remove space before class ':'. +sp_before_class_colon = force # ignore/add/remove/force + +# Add or remove space inside '{}'. +sp_inside_braces_empty = remove # ignore/add/remove/force + +# Add or remove space between 'else' and '{' if on the same line. +sp_else_brace = force # ignore/add/remove/force + +# Add or remove space between '}' and 'else' if on the same line. +sp_brace_else = force # ignore/add/remove/force + +# Add or remove space before the '{' of a 'catch' statement, if the '{' and +# 'catch' are on the same line, as in 'catch (decl) {'. +sp_catch_brace = force # ignore/add/remove/force + +# Add or remove space between '}' and 'catch' if on the same line. +sp_brace_catch = force # ignore/add/remove/force + +# The number of columns to indent per level. Usually 2, 3, 4, or 8. +# +# Default: 8 +indent_columns = 4 # unsigned number + +# How to use tabs when indenting code. +# +# 0: Spaces only +# 1: Indent with tabs to brace level, align with spaces (default) +# 2: Indent and align with tabs, using spaces when not on a tabstop +# +# Default: 1 +indent_with_tabs = 0 # unsigned number + +# Whether to indent the body of a 'namespace'. +indent_namespace = true # true/false + +# Whether the 'class' body is indented. +indent_class = true # true/false + +# How to indent access specifiers that are followed by a +# colon. +# +# >0: Absolute column where 1 is the leftmost column +# <=0: Subtract from brace indent +# +# Default: 1 +indent_access_spec = -4 # number + +# Whether to collapse empty blocks between '{' and '}' except for functions. +# Use nl_collapse_empty_body_functions to specify how empty function braces +# should be formatted. +nl_collapse_empty_body = true # true/false + +# Whether to collapse empty blocks between '{' and '}' for functions only. +# If true, overrides nl_inside_empty_func. +nl_collapse_empty_body_functions = true # true/false + +# Whether to convert all tabs to spaces in comments. If false, tabs in +# comments are left alone, unless used for indenting. +cmt_convert_tab_to_spaces = true # true/false + +# An offset value that controls the indentation of the body of a multiline #define. +# 'body' refers to all the lines of a multiline #define except the first line. +# Requires 'pp_ignore_define_body = false'. +# +# <0: Absolute column: the body indentation starts off at the specified column +# (ex. -3 ==> the body is indented starting from column 3) +# >=0: Relative to the column of the '#' of '#define' +# (ex. 3 ==> the body is indented starting 3 columns at the right of '#') +# +# Default: 8 +pp_multiline_define_body_indent = 4 # number + +# The value might be used twice: +# - at the assignment +# - at the opening brace +# +# To prevent the double use of the indentation value, use this option with the +# value 'true'. +# +# true: indentation will be used only once +# false: indentation will be used every time (default) +indent_cpp_lambda_only_once = true # true/false diff --git a/runastyle b/runastyle deleted file mode 100755 index 64298273..00000000 --- a/runastyle +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# The version check in this script is used to avoid commit battles -# between different developers that use different astyle versions as -# different versions might have different output (this has happened in -# the past). - -# If project management wishes to take a newer astyle version into use -# just change this string to match the start of astyle version string. -ASTYLE_VERSION="Artistic Style Version 3.0.1" -ASTYLE="astyle" - -DETECTED_VERSION=`$ASTYLE --version 2>&1` -if [[ "$DETECTED_VERSION" != ${ASTYLE_VERSION}* ]]; then - echo "You should use: ${ASTYLE_VERSION}"; - echo "Detected: ${DETECTED_VERSION}" - exit 1; -fi - -style="--style=kr --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0" -options="--options=none --pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines --attach-classes --attach-namespaces" - -$ASTYLE $style $options *.cpp -$ASTYLE $style $options *.h diff --git a/runastyle.bat b/runastyle.bat deleted file mode 100644 index b8f11561..00000000 --- a/runastyle.bat +++ /dev/null @@ -1,26 +0,0 @@ -@REM Script to run AStyle on the sources -@REM The version check in this script is used to avoid commit battles -@REM between different developers that use different astyle versions as -@REM different versions might have different output (this has happened in -@REM the past). - -@REM If project management wishes to take a newer astyle version into use -@REM just change this string to match the start of astyle version string. -@SET ASTYLE_VERSION="Artistic Style Version 3.0.1" -@SET ASTYLE="astyle" - -@SET DETECTED_VERSION="" -@FOR /F "tokens=*" %%i IN ('%ASTYLE% --version') DO SET DETECTED_VERSION=%%i -@ECHO %DETECTED_VERSION% | FINDSTR /B /C:%ASTYLE_VERSION% > nul && ( - ECHO "%DETECTED_VERSION%" matches %ASTYLE_VERSION% -) || ( - ECHO You should use: %ASTYLE_VERSION% - ECHO Detected: "%DETECTED_VERSION%" - GOTO EXIT_ERROR -) - -@SET STYLE=--style=kr --indent=spaces=4 --indent-namespaces --lineend=linux --min-conditional-indent=0 -@SET OPTIONS=--pad-header --unpad-paren --suffix=none --convert-tabs --attach-inlines --attach-classes --attach-namespaces - -%ASTYLE% %STYLE% %OPTIONS% *.cpp -%ASTYLE% %STYLE% %OPTIONS% *.h \ No newline at end of file diff --git a/runformat b/runformat new file mode 100755 index 00000000..2f883a76 --- /dev/null +++ b/runformat @@ -0,0 +1,39 @@ +#!/bin/bash +# +# uncrustify-0.72 is used to format simplecpp and cppcheck source code. +# +# 1. Download source code: https://github.com/uncrustify/uncrustify/archive/refs/tags/uncrustify-0.72.0.zip +# It's important that all developers use the exact same version so we don't get a "format battle". +# 2. Building: +# - Linux: mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. && make +# - Windows: mkdir build && cd build && cmake -G"NMake Makefiles" -DCMAKE_BUILD_TYPE=Release .. && nmake +# 3. Ensure that the binary "uncrustify" is found by runformat. Either: +# - you can put uncrustify in your PATH +# - you can create an environment variable UNCRUSTIFY that has the full path of the binary + +UNCRUSTIFY_VERSION="0.72.0" +UNCRUSTIFY="${UNCRUSTIFY-uncrustify}" + +DETECTED_VERSION=$("$UNCRUSTIFY" --version 2>&1 | grep -o -E '[0-9.]+') +if [ "$DETECTED_VERSION" != "${UNCRUSTIFY_VERSION}" ]; then + echo "You should use version: ${UNCRUSTIFY_VERSION}" + echo "Detected version: ${DETECTED_VERSION}" + exit 1 +fi + +# OS variables +[ $(uname -s) = "Darwin" ] && export OSX=1 && export UNIX=1 +[ $(uname -s) = "Linux" ] && export LINUX=1 && export UNIX=1 +uname -s | grep -q "_NT-" && export WINDOWS=1 + +if [ $OSX ] +then + export CPUCOUNT=$(sysctl -n hw.ncpu) +elif [ $LINUX ] +then + export CPUCOUNT=$(nproc) +else + export CPUCOUNT="1" +fi + +$UNCRUSTIFY -c .uncrustify.cfg --no-backup *.cpp *.h diff --git a/simplecpp.cpp b/simplecpp.cpp index 74ab66e1..22a45e7a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -761,18 +761,18 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, while (stream.good() && ch != '\n') { currentToken += ch; ch = stream.readChar(); - if(ch == '\\') { + if (ch == '\\') { TokenString tmp; char tmp_ch = ch; - while((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { + while ((stream.good()) && (tmp_ch == '\\' || tmp_ch == ' ' || tmp_ch == '\t')) { tmp += tmp_ch; tmp_ch = stream.readChar(); } - if(!stream.good()) { + if (!stream.good()) { break; } - if(tmp_ch != '\n') { + if (tmp_ch != '\n') { currentToken += tmp; } else { const TokenString check_portability = currentToken + tmp; @@ -1667,7 +1667,7 @@ namespace simplecpp { } invalidHashHash(const Location &loc, const std::string ¯oName, const std::string &message) - : Error(loc, format(macroName, message)) { } + : Error(loc, format(macroName, message)) {} static inline invalidHashHash unexpectedToken(const Location &loc, const std::string ¯oName, const Token *tokenA) { return invalidHashHash(loc, macroName, "Unexpected token '"+ tokenA->str()+"'"); @@ -2672,7 +2672,7 @@ static unsigned long long stringToULLbounded( int base = 0, std::ptrdiff_t minlen = 1, std::size_t maxlen = std::string::npos -) + ) { const std::string sub = s.substr(pos, maxlen); const char * const start = sub.c_str(); @@ -3354,7 +3354,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL includetokenstack.push(filedata->tokens.cfront()); } - std::map > maybeUsedMacros; + std::map> maybeUsedMacros; for (const Token *rawtok = nullptr; rawtok || !includetokenstack.empty();) { if (rawtok == nullptr) { @@ -3751,11 +3751,11 @@ simplecpp::cstd_t simplecpp::getCStd(const std::string &std) { if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") return C89; - if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99"|| std == "gnu9x") + if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99" || std == "gnu9x") return C99; if (std == "c11" || std == "c1x" || std == "iso9899:2011" || std == "gnu11" || std == "gnu1x") return C11; - if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") + if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17" || std == "gnu18") return C17; if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") return C23; diff --git a/simplecpp.h b/simplecpp.h index 8268fa8d..05de07dd 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -114,8 +114,7 @@ namespace simplecpp { } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), nextcond(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) { - } + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), nextcond(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} void flags() { name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') @@ -325,9 +324,9 @@ namespace simplecpp { struct SIMPLECPP_LIB MacroUsage { explicit MacroUsage(const std::vector &f, bool macroValueKnown_) : macroLocation(f), useLocation(f), macroValueKnown(macroValueKnown_) {} std::string macroName; - Location macroLocation; - Location useLocation; - bool macroValueKnown; + Location macroLocation; + Location useLocation; + bool macroValueKnown; }; /** Tracking #if/#elif expressions */ diff --git a/test.cpp b/test.cpp index ccb653ca..7b108638 100644 --- a/test.cpp +++ b/test.cpp @@ -22,7 +22,7 @@ static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) -#define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while(false) +#define ASSERT_THROW_EQUALS(stmt, e, expected) do { try { stmt; assertThrowFailed(__LINE__); } catch (const e& ex) { assertEquals((expected), (ex.what()), __LINE__); } } while (false) static std::string pprint(const std::string &in) { @@ -250,10 +250,10 @@ static void characterLiteral() ASSERT_EQUALS('\u0012', simplecpp::characterLiteralToLL("'\\u0012'")); ASSERT_EQUALS('\U00000012', simplecpp::characterLiteralToLL("'\\U00000012'")); - ASSERT_EQUALS((static_cast(static_cast('b')) << 8) | static_cast('c'), simplecpp::characterLiteralToLL("'bc'")); + ASSERT_EQUALS((static_cast(static_cast('b')) << 8) | static_cast('c'), simplecpp::characterLiteralToLL("'bc'")); ASSERT_EQUALS((static_cast(static_cast('\x23')) << 8) | static_cast('\x45'), simplecpp::characterLiteralToLL("'\\x23\\x45'")); - ASSERT_EQUALS((static_cast(static_cast('\11')) << 8) | static_cast('\222'), simplecpp::characterLiteralToLL("'\\11\\222'")); - ASSERT_EQUALS((static_cast(static_cast('\a')) << 8) | static_cast('\b'), simplecpp::characterLiteralToLL("'\\a\\b'")); + ASSERT_EQUALS((static_cast(static_cast('\11')) << 8) | static_cast('\222'), simplecpp::characterLiteralToLL("'\\11\\222'")); + ASSERT_EQUALS((static_cast(static_cast('\a')) << 8) | static_cast('\b'), simplecpp::characterLiteralToLL("'\\a\\b'")); if (sizeof(int) <= 4) ASSERT_EQUALS(-1, simplecpp::characterLiteralToLL("'\\xff\\xff\\xff\\xff'")); else @@ -438,14 +438,14 @@ static void comment_multiline() ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code)); const char code1[] = "#define ABC {// \\\r\n" - "}\n" - "void f() ABC\n"; + "}\n" + "void f() ABC\n"; ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code1)); const char code2[] = "#define A 1// \\\r" - "\r" - "2\r" - "A\r"; + "\r" + "2\r" + "A\r"; ASSERT_EQUALS("\n\n2\n1", preprocess(code2)); const char code3[] = "void f() {// \\ \n}\n"; @@ -1960,7 +1960,7 @@ static void location1() const char *code; code = "# 1 \"main.c\"\n\n\n" - "x"; + "x"; ASSERT_EQUALS("\n#line 3 \"main.c\"\nx", preprocess(code)); } @@ -2290,7 +2290,7 @@ static void include2() static void include3() // #16 - crash when expanding macro from header { const char code_c[] = "#include \"A.h\"\n" - "glue(1,2,3,4)\n" ; + "glue(1,2,3,4)\n"; const char code_h[] = "#define glue(a,b,c,d) a##b##c##d\n"; std::vector files; @@ -2317,7 +2317,7 @@ static void include3() // #16 - crash when expanding macro from header static void include4() // #27 - -include { - const char code_c[] = "X\n" ; + const char code_c[] = "X\n"; const char code_h[] = "#define X 123\n"; std::vector files; From f282eec74b776fe66bebf1de2c1202a5c767ae6c Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Fri, 29 Aug 2025 02:59:18 -0400 Subject: [PATCH 292/381] chore: Improve runformat script and integrate CI checks (#520) --- .github/workflows/format.yml | 62 +++++++++++ .gitignore | 1 + runformat | 201 +++++++++++++++++++++++++++++------ 3 files changed, 232 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/format.yml diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 00000000..4d5657f8 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,62 @@ +# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions +# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners +name: format + +on: + push: + branches: + - 'master' + - 'releases/**' + - '1.*' + tags: + - '1.*' + pull_request: + +permissions: + contents: read + +jobs: + format: + + runs-on: ubuntu-22.04 + + defaults: + run: + shell: bash -euo pipefail {0} + + env: + UNCRUSTIFY_INSTALL_DIR: ${{ github.workspace }}/runformat-uncrustify + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Determine uncrustify version + id: get-uncrustify-version + run: | + version="$(./runformat --expected-uncrustify-version)" + echo "Expected uncrustify version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Set UNCRUSTIFY_VERSION env variable + run: | + version=$(./runformat --expected-uncrustify-version) + echo "version [$version]" + echo "UNCRUSTIFY_VERSION=${version}" >> "$GITHUB_ENV" + + - name: Cache uncrustify + uses: actions/cache@v4 + id: cache-uncrustify + with: + path: ${{ env.UNCRUSTIFY_INSTALL_DIR }} + key: ${{ runner.os }}-uncrustify-${{ steps.get-uncrustify-version.outputs.version }} + + - name: Install uncrustify + if: steps.cache-uncrustify.outputs.cache-hit != 'true' + run: | + ./runformat --install --install-dir "${UNCRUSTIFY_INSTALL_DIR}" + + - name: Uncrustify check + run: | + ./runformat diff --git a/.gitignore b/.gitignore index 113cf360..34d9c55d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ *.app simplecpp testrunner +/.runformat-uncrustify # CLion /.idea diff --git a/runformat b/runformat index 2f883a76..0dc1ad52 100755 --- a/runformat +++ b/runformat @@ -1,39 +1,176 @@ #!/bin/bash # -# uncrustify-0.72 is used to format simplecpp and cppcheck source code. +# runformat - format this project's C++ sources with Uncrustify. # -# 1. Download source code: https://github.com/uncrustify/uncrustify/archive/refs/tags/uncrustify-0.72.0.zip -# It's important that all developers use the exact same version so we don't get a "format battle". -# 2. Building: -# - Linux: mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. && make -# - Windows: mkdir build && cd build && cmake -G"NMake Makefiles" -DCMAKE_BUILD_TYPE=Release .. && nmake -# 3. Ensure that the binary "uncrustify" is found by runformat. Either: -# - you can put uncrustify in your PATH -# - you can create an environment variable UNCRUSTIFY that has the full path of the binary - -UNCRUSTIFY_VERSION="0.72.0" -UNCRUSTIFY="${UNCRUSTIFY-uncrustify}" - -DETECTED_VERSION=$("$UNCRUSTIFY" --version 2>&1 | grep -o -E '[0-9.]+') -if [ "$DETECTED_VERSION" != "${UNCRUSTIFY_VERSION}" ]; then - echo "You should use version: ${UNCRUSTIFY_VERSION}" - echo "Detected version: ${DETECTED_VERSION}" - exit 1 +# Usage: +# ./runformat # format using the configured Uncrustify +# ./runformat --install # download, build, and use Uncrustify locally +# ./runformat --install --install-dir /abs/path +# ./runformat --expected-uncrustify-version # print ONLY the expected Uncrustify version +# +# You may also set: +# UNCRUSTIFY=/abs/path/to/uncrustify # use a specific binary +# UNCRUSTIFY_INSTALL_DIR=/abs/path # where `--install` will install +# +# Requirements: +# - All developers must use the *exact* same Uncrustify version to avoid format churn. +# - Either: +# * Have `uncrustify` in PATH, or +# * Set env var UNCRUSTIFY=/absolute/path/to/uncrustify, or +# * Run `./runformat --install` to fetch & build the pinned version locally. +# +# Notes: +# - The local install lives under: ./.runformat-uncrustify/uncrustify--install +# - The config file is expected at: ./.uncrustify.cfg +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +UNCRUSTIFY_VERSION="0.80.1" +UNCRUSTIFY_HASH="6bf662e05c4140dd4df5e45d6690cad96b4ef23c293b85813f5c725bbf1894d0" + +UNCRUSTIFY_WORK_DIR="${SCRIPT_DIR}/.runformat-uncrustify" + +# Allow external install dir override (arg or env). If not set, default under work dir. +DEFAULT_INSTALL_DIR="${UNCRUSTIFY_WORK_DIR}/uncrustify-${UNCRUSTIFY_VERSION}-install" +UNCRUSTIFY_INSTALL_DIR="${UNCRUSTIFY_INSTALL_DIR:-$DEFAULT_INSTALL_DIR}" +UNCRUSTIFY_BIN="${UNCRUSTIFY_INSTALL_DIR}/bin/uncrustify" + +# Allow override via env; default to local pinned build path. +UNCRUSTIFY="${UNCRUSTIFY:-$UNCRUSTIFY_BIN}" +UNCRUSTIFY_CONFIG="${SCRIPT_DIR}/.uncrustify.cfg" + +err() { echo -e >&2 "ERROR: $@\n"; } +die() { err $@; exit 1; } + +install_uncrustify() { + local root="uncrustify-${UNCRUSTIFY_VERSION}" + local file="${root}.tar.gz" + local url="https://github.com/uncrustify/uncrustify/releases/download/${root}/${file}" + + mkdir -p "${UNCRUSTIFY_WORK_DIR}" + + echo "Downloading ${file}..." + curl -fsSL -o "${UNCRUSTIFY_WORK_DIR}/${file}" "${url}" + + ( + cd "${UNCRUSTIFY_WORK_DIR}" + + echo "${UNCRUSTIFY_HASH} ${file}" > "${file}.sha256" + sha256sum -c "${file}.sha256" + rm -f "${file}.sha256" + + command -v cmake >/dev/null 2>&1 || die "cmake executable not found." + + echo "Extracting archive..." + rm -rf "${root}" "${root}-build" + mkdir -p "${root}" + tar -xzf "${file}" --strip-components=1 -C "${root}" + + echo "Configuring (prefix: ${UNCRUSTIFY_INSTALL_DIR})..." + cmake \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:PATH="${UNCRUSTIFY_INSTALL_DIR}" \ + -S "${root}" -B "${root}-build" + + echo "Building & installing..." + cmake --build "${root}-build" --config Release --target install --parallel + ) + + echo "Installed Uncrustify to: ${UNCRUSTIFY_INSTALL_DIR}" +} + +print_usage_and_exit() { + sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//' + exit 0 +} + +# Print ONLY expected Uncrustify version (no extra text). +print_expected_uncrustify_version_and_exit() { + printf '%s\n' "$UNCRUSTIFY_VERSION" + exit 0 +} + +# -------------------------- +# Argument parsing +# -------------------------- +DO_INSTALL=0 +PRINT_EXPECTED_UNCRUSTIFY_VERSION=0 +# Accept: --install, --install-dir , -h/--help +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + print_usage_and_exit + ;; + --install) + DO_INSTALL=1 + shift + ;; + --install-dir) + [[ $# -ge 2 ]] || die "$1 requires a path argument" + UNCRUSTIFY_INSTALL_DIR="$(readlink -m "$2" 2>/dev/null || realpath -m "$2")" + UNCRUSTIFY_BIN="${UNCRUSTIFY_INSTALL_DIR}/bin/uncrustify" + # Only update UNCRUSTIFY default if user hasn't explicitly set it + if [[ "${UNCRUSTIFY:-}" != "${UNCRUSTIFY_BIN}" ]]; then + UNCRUSTIFY="${UNCRUSTIFY_BIN}" + fi + shift 2 + ;; + --expected-uncrustify-version) + PRINT_EXPECTED_UNCRUSTIFY_VERSION=1 + shift + ;; + *) + # ignore unrecognized positional args for now + shift + ;; + esac +done + +if [[ "$DO_INSTALL" -eq 1 ]]; then + install_uncrustify + # Ensure we use the freshly installed binary for this run + UNCRUSTIFY="$UNCRUSTIFY_BIN" fi -# OS variables -[ $(uname -s) = "Darwin" ] && export OSX=1 && export UNIX=1 -[ $(uname -s) = "Linux" ] && export LINUX=1 && export UNIX=1 -uname -s | grep -q "_NT-" && export WINDOWS=1 - -if [ $OSX ] -then - export CPUCOUNT=$(sysctl -n hw.ncpu) -elif [ $LINUX ] -then - export CPUCOUNT=$(nproc) -else - export CPUCOUNT="1" +# If requested, print ONLY the expected Uncrustify version and exit. +if [[ "$PRINT_EXPECTED_UNCRUSTIFY_VERSION" -eq 1 ]]; then + print_expected_uncrustify_version_and_exit fi -$UNCRUSTIFY -c .uncrustify.cfg --no-backup *.cpp *.h +# -------------------------- +# Validate & run +# -------------------------- + +# Check Uncrustify availability +if ! command -v "$UNCRUSTIFY" >/dev/null 2>&1; then + err "Uncrustify executable not found: $UNCRUSTIFY" + die "Add it to PATH, set UNCRUSTIFY=/path/to/uncrustify, or run: $0 --install [--install-dir DIR]" +fi + +# Version check +DETECTED_VERSION="$("$UNCRUSTIFY" --version 2>&1 | grep -oE '[0-9]+(\.[0-9]+)*' | head -n1 || true)" +echo "Detected Uncrustify: ${DETECTED_VERSION:-unknown}" +if [[ "$DETECTED_VERSION" != "${UNCRUSTIFY_VERSION}" ]]; then + die "Expected Uncrustify ${UNCRUSTIFY_VERSION}. Re-run with --install (and optionally --install-dir) or set UNCRUSTIFY." +fi + +# Config check +[[ -f "$UNCRUSTIFY_CONFIG" ]] || die "Uncrustify config not found at: $UNCRUSTIFY_CONFIG" + +# Run formatter +echo "Running formatter..." +$UNCRUSTIFY -c "$UNCRUSTIFY_CONFIG" -l CPP --no-backup --replace *.cpp *.h + +# Show diff and fail if changes exist +echo "Checking for formatting changes..." +git diff --exit-code || { + echo + echo "Formatting changes were applied. Please review and commit." + exit 1 +} + +echo "Formatting is clean." From 871dc30b45d401780a8d6f38d88db8fe56a5d5f1 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Fri, 29 Aug 2025 03:19:27 -0400 Subject: [PATCH 293/381] chore: Improve attribution by ignoring bulk formatting (#518) --- .git-blame-ignore-revs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..bd256eb3 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,16 @@ +# +# This file lists revisions that should be ignored when considering +# attribution for the actual code written. Code style changes should +# not be considered as modifications with regards to attribution. +# +# To see clean and meaningful blame information. +# $ git blame important.py --ignore-revs-file .git-blame-ignore-revs +# +# To configure git to automatically ignore revisions listed in a file +# on every call to git blame. +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# Ignore changes introduced when doing global file format changes + +# Switch to uncrustify (#517) +cfd179711f413aa8e0da9c2f437ad4f8938d5f70 From cd2dd49cf1c183da0b8fc0746dba953fcfbb9b18 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Fri, 29 Aug 2025 06:11:50 -0400 Subject: [PATCH 294/381] tests: Fix execution of testrunner for out-of-source builds (#510) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Oliver Stöneberg --- .github/workflows/CI-unixish.yml | 3 +++ CMakeLists.txt | 4 ++++ Makefile | 10 ++++++++-- test.cpp | 12 +++++++++++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 60361389..f76fdd15 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -73,6 +73,9 @@ jobs: run: | cmake --build cmake.output --target testrunner -- -j $(nproc) ./cmake.output/testrunner + # Re-run tests from within the build directory to validate that + # SIMPLECPP_TEST_SOURCE_DIR is correctly defined and resolved + (cd cmake.output && ./testrunner) - name: Run valgrind if: matrix.os == 'ubuntu-24.04' diff --git a/CMakeLists.txt b/CMakeLists.txt index f9e3eb6d..8799b7a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,10 @@ add_library(simplecpp_obj OBJECT simplecpp.cpp) add_executable(simplecpp $ main.cpp) add_executable(testrunner $ test.cpp) +target_compile_definitions(testrunner + PRIVATE + SIMPLECPP_TEST_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" +) enable_testing() add_test(NAME testrunner COMMAND testrunner) diff --git a/Makefile b/Makefile index d899d2cd..7489ec83 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,17 @@ all: testrunner simplecpp +CPPFLAGS ?= CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++11 -g $(CXXOPTS) LDFLAGS = -g $(LDOPTS) -%.o: %.cpp simplecpp.h - $(CXX) $(CXXFLAGS) -c $< +# Define test source dir macro for compilation (preprocessor flags) +TEST_CPPFLAGS = -DSIMPLECPP_TEST_SOURCE_DIR=\"$(CURDIR)\" + +# Only test.o gets the define +test.o: CPPFLAGS += $(TEST_CPPFLAGS) +%.o: %.cpp simplecpp.h + $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $< testrunner: test.o simplecpp.o $(CXX) $(LDFLAGS) simplecpp.o test.o -o testrunner diff --git a/test.cpp b/test.cpp index 7b108638..c44524a3 100644 --- a/test.cpp +++ b/test.cpp @@ -16,9 +16,14 @@ #include #include +#ifndef SIMPLECPP_TEST_SOURCE_DIR +#error "SIMPLECPP_TEST_SOURCE_DIR is not defined." +#endif + #define STRINGIZE_(x) #x #define STRINGIZE(x) STRINGIZE_(x) +static const std::string testSourceDir = SIMPLECPP_TEST_SOURCE_DIR; static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) @@ -1556,6 +1561,7 @@ static void has_include_1() " #endif\n" "#endif"; simplecpp::DUI dui; + dui.includePaths.push_back(testSourceDir); dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); dui.std = "c++14"; @@ -1573,6 +1579,7 @@ static void has_include_2() " #endif\n" "#endif"; simplecpp::DUI dui; + dui.includePaths.push_back(testSourceDir); dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); @@ -1592,7 +1599,7 @@ static void has_include_3() // Test file not found... ASSERT_EQUALS("\n\n\n\nB", preprocess(code, dui)); // Unless -I is set (preferably, we should differentiate -I and -isystem...) - dui.includePaths.push_back("./testsuite"); + dui.includePaths.push_back(testSourceDir + "/testsuite"); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } @@ -1608,6 +1615,7 @@ static void has_include_4() "#endif"; simplecpp::DUI dui; dui.std = "c++17"; + dui.includePaths.push_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } @@ -1623,6 +1631,7 @@ static void has_include_5() "#endif"; simplecpp::DUI dui; dui.std = "c++17"; + dui.includePaths.push_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } @@ -1638,6 +1647,7 @@ static void has_include_6() "#endif"; simplecpp::DUI dui; dui.std = "gnu99"; + dui.includePaths.push_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); ASSERT_EQUALS("", preprocess(code)); } From 9db6a20f9d35108b1f81a2344c7bebf647151bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 13:45:22 +0200 Subject: [PATCH 295/381] CI-unixish.yml: cleaned up prerequisites for `macos-*` (#519) --- .github/workflows/CI-unixish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index f76fdd15..c81f023c 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -41,10 +41,11 @@ jobs: sudo apt-get update sudo apt-get install libc++-18-dev + # coreutils contains "nproc" - name: Install missing software on macos if: contains(matrix.os, 'macos') run: | - brew install python3 + brew install coreutils - name: Install missing Python packages run: | From 0689d834e49a50a2fc542f7262d2b660ac6575b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 13:45:36 +0200 Subject: [PATCH 296/381] fixed #500 - added callgrind step to CI (#501) --- .github/workflows/CI-unixish.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index c81f023c..b045bcbd 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -120,3 +120,22 @@ jobs: run: | make clean make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -stdlib=libc++ -fsanitize=memory" LDOPTS="-lc++ -fsanitize=memory" + + - name: Run callgrind + if: matrix.os == 'ubuntu-24.04' + run: | + wget https://github.com/danmar/simplecpp/archive/refs/tags/1.5.1.tar.gz + tar xvf 1.5.1.tar.gz + make clean + make -j$(nproc) CXXOPTS="-O2 -g3" + valgrind --tool=callgrind ./simplecpp -e simplecpp-1.5.1/simplecpp.cpp 2>callgrind.log || (cat callgrind.log && false) + cat callgrind.log + callgrind_annotate --auto=no > callgrind.annotated.log + head -50 callgrind.annotated.log + + - uses: actions/upload-artifact@v4 + if: matrix.os == 'ubuntu-24.04' + with: + name: Callgrind Output - ${{ matrix.compiler }} + path: | + ./callgrind.* From 3911fdd284653ea52087f44574c986110f09ae5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 13:56:58 +0200 Subject: [PATCH 297/381] fixed some `Variable copied when it could be moved` Coverity warnings (#495) --- simplecpp.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 22a45e7a..d576f6e4 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -611,7 +611,7 @@ static void portabilityBackslash(simplecpp::OutputList *outputList, const std::v err.type = simplecpp::Output::PORTABILITY_BACKSLASH; err.location = location; err.msg = "Combination 'backslash space newline' is not portable."; - outputList->push_back(err); + outputList->push_back(std::move(err)); } static bool isStringLiteralPrefix(const std::string &str) @@ -855,7 +855,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, err.type = Output::SYNTAX_ERROR; err.location = location; err.msg = "Raw string missing terminating delimiter."; - outputList->push_back(err); + outputList->push_back(std::move(err)); } return; } @@ -1397,7 +1397,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca err.type = Output::SYNTAX_ERROR; err.location = location; err.msg = std::string("No pair for character (") + start + "). Can't process file. File is either invalid or unicode, which is currently not supported."; - outputList->push_back(err); + outputList->push_back(std::move(err)); } return ""; } @@ -2178,7 +2178,7 @@ namespace simplecpp { if (it != macros.end() && !partok->isExpandedFrom(&it->second) && (partok->str() == name() || expandedmacros.find(partok->str()) == expandedmacros.end())) { std::set expandedmacros2(expandedmacros); // temporary amnesia to allow reexpansion of currently expanding macros during argument evaluation expandedmacros2.erase(name()); - partok = it->second.expand(output, loc, partok, macros, expandedmacros2); + partok = it->second.expand(output, loc, partok, macros, std::move(expandedmacros2)); } else { output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); output->back()->macro = partok->macro; @@ -3137,7 +3137,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, err.type = simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND; err.location = Location(filenames); err.msg = "Can not open include file '" + filename + "' that is explicitly included."; - outputList->push_back(err); + outputList->push_back(std::move(err)); } continue; } @@ -3210,7 +3210,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token out.type = simplecpp::Output::SYNTAX_ERROR; out.location = err.location; out.msg = "failed to expand \'" + tok->str() + "\', " + err.what; - outputList->push_back(out); + outputList->push_back(std::move(out)); } return false; } @@ -3397,7 +3397,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL err.msg += tok->str(); } err.msg = '#' + rawtok->str() + ' ' + err.msg; - outputList->push_back(err); + outputList->push_back(std::move(err)); } if (rawtok->str() == ERROR) { output.clear(); @@ -3557,7 +3557,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL out.type = Output::SYNTAX_ERROR; out.location = rawtok->location; out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; - outputList->push_back(out); + outputList->push_back(std::move(out)); } output.clear(); return; @@ -3736,7 +3736,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL mu.macroName = macro.name(); mu.macroLocation = macro.defineLocation(); mu.useLocation = *usageIt; - macroUsage->push_back(mu); + macroUsage->push_back(std::move(mu)); } } } From 4056cd5fcb1d002b3e0ce3b7a3dc80aac720e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 13:59:14 +0200 Subject: [PATCH 298/381] added `TokenList` constructors with modern buffer wrappers and hide "unsafe" ones - if available (#496) --- .github/workflows/CI-unixish.yml | 10 ++++++ simplecpp.cpp | 9 +---- simplecpp.h | 62 ++++++++++++++++++++++++++++++-- test.cpp | 40 +++++++++++++++++++++ 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index b045bcbd..adb91138 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -62,6 +62,16 @@ jobs: run: | make -j$(nproc) selfcheck + - name: make testrunner (c++17) + run: | + make clean + make -j$(nproc) testrunner CXXOPTS="-std=c++17" + + - name: make testrunner (c++20) + run: | + make clean + make -j$(nproc) testrunner CXXOPTS="-std=c++20" + - name: Run CMake run: | cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On diff --git a/simplecpp.cpp b/simplecpp.cpp index d576f6e4..23aa7387 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -466,20 +466,13 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector &fi readfile(stream,filename,outputList); } -simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) +simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList, int /*unused*/) : frontToken(nullptr), backToken(nullptr), files(filenames) { StdCharBufStream stream(data, size); readfile(stream,filename,outputList); } -simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) - : frontToken(nullptr), backToken(nullptr), files(filenames) -{ - StdCharBufStream stream(reinterpret_cast(data), size); - readfile(stream,filename,outputList); -} - simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { diff --git a/simplecpp.h b/simplecpp.h index 05de07dd..f035ea95 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -20,6 +20,16 @@ #include #include #include +#if __cplusplus >= 202002L +# include +#endif + +#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) +#include +#endif +#ifdef __cpp_lib_span +#include +#endif #ifdef _WIN32 # ifdef SIMPLECPP_EXPORT @@ -46,6 +56,15 @@ # pragma warning(disable : 4244) #endif +// provide legacy (i.e. raw pointer) API for TokenList +// note: std::istream has an overhead compared to raw pointers +#ifndef SIMPLECPP_TOKENLIST_ALLOW_PTR +// still provide the legacy API in case we lack the performant wrappers +# if !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) +# define SIMPLECPP_TOKENLIST_ALLOW_PTR +# endif +#endif + namespace simplecpp { /** C code standard */ enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 }; @@ -216,10 +235,45 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); +#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR + /** generates a token list from the given buffer */ + template + TokenList(const char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(reinterpret_cast(data), size-1, filenames, filename, outputList, 0) + {} + /** generates a token list from the given buffer */ + template + TokenList(const unsigned char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(data, size-1, filenames, filename, outputList, 0) + {} + /** generates a token list from the given buffer */ - TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(data, size, filenames, filename, outputList, 0) + {} /** generates a token list from the given buffer */ - TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(reinterpret_cast(data), size, filenames, filename, outputList, 0) + {} +#endif +#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) + /** generates a token list from the given buffer */ + TokenList(std::string_view data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(reinterpret_cast(data.data()), data.size(), filenames, filename, outputList, 0) + {} +#endif +#ifdef __cpp_lib_span + /** generates a token list from the given buffer */ + TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(reinterpret_cast(data.data()), data.size(), filenames, filename, outputList, 0) + {} + + /** generates a token list from the given buffer */ + TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + : TokenList(data.data(), data.size(), filenames, filename, outputList, 0) + {} +#endif + /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); @@ -295,6 +349,8 @@ namespace simplecpp { } private: + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList, int unused); + void combineOperators(); void constFoldUnaryNotPosNeg(Token *tok); @@ -505,6 +561,8 @@ namespace simplecpp { SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); } +#undef SIMPLECPP_TOKENLIST_ALLOW_PTR + #if defined(_MSC_VER) # pragma warning(pop) #endif diff --git a/test.cpp b/test.cpp index c44524a3..0ecaa3b1 100644 --- a/test.cpp +++ b/test.cpp @@ -3179,6 +3179,44 @@ static void preprocess_files() } } +static void safe_api() +{ + // this test is to make sure the safe APIs are compiling +#if defined(__cpp_lib_string_view) || defined(__cpp_lib_span) + std::vector filenames; +# if defined(__cpp_lib_string_view) + { + const char input[] = "code"; + const std::string_view sv = input; + // std::string_view can be implicitly converted into a std::span + simplecpp::TokenList(sv,filenames,""); + } +# endif +# ifdef __cpp_lib_span + { + char input[] = "code"; + const std::span sp = input; + simplecpp::TokenList(sp,filenames,""); + } + { + const char input[] = "code"; + const std::span sp = input; + simplecpp::TokenList(sp,filenames,""); + } + { + unsigned char input[] = "code"; + const std::span sp = input; + simplecpp::TokenList(sp,filenames,""); + } + { + const unsigned char input[] = "code"; + const std::span sp = input; + simplecpp::TokenList(sp,filenames,""); + } +# endif +#endif +} + static void fuzz_crash() { { @@ -3445,6 +3483,8 @@ int main(int argc, char **argv) TEST_CASE(preprocess_files); + TEST_CASE(safe_api); + TEST_CASE(fuzz_crash); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; From 4db4304739e8634e62ec3a6ba8eae3d732ab4275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 17:11:33 +0200 Subject: [PATCH 299/381] simplecpp.h: fixed formatting (#522) --- simplecpp.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index f035ea95..ac367154 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -244,7 +244,7 @@ namespace simplecpp { /** generates a token list from the given buffer */ template TokenList(const unsigned char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) - : TokenList(data, size-1, filenames, filename, outputList, 0) + : TokenList(data, size-1, filenames, filename, outputList, 0) {} /** generates a token list from the given buffer */ @@ -265,12 +265,12 @@ namespace simplecpp { #ifdef __cpp_lib_span /** generates a token list from the given buffer */ TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) - : TokenList(reinterpret_cast(data.data()), data.size(), filenames, filename, outputList, 0) + : TokenList(reinterpret_cast(data.data()), data.size(), filenames, filename, outputList, 0) {} /** generates a token list from the given buffer */ TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) - : TokenList(data.data(), data.size(), filenames, filename, outputList, 0) + : TokenList(data.data(), data.size(), filenames, filename, outputList, 0) {} #endif From 740b2b94a6eb50a1b81822f52d8347c2514cffd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 29 Aug 2025 18:30:13 +0200 Subject: [PATCH 300/381] do not use stream to read file in `FileDataCache::tryload()` (#523) --- simplecpp.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 23aa7387..84e4b54b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3022,8 +3022,7 @@ std::pair simplecpp::FileDataCache::tryload(FileDat return {id_it->second, false}; } - std::ifstream f(path); - FileData *const data = new FileData {path, TokenList(f, filenames, path, outputList)}; + FileData *const data = new FileData {path, TokenList(path, filenames, outputList)}; if (dui.removeComments) data->tokens.removeComments(); From 37fa4f49b33aca8b4e86f42b43c3d57060e725b5 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Fillion-Robin Date: Sat, 30 Aug 2025 13:26:00 -0400 Subject: [PATCH 301/381] Improve test runner script with refactoring and prerequisite checks (#509) --- run-tests.py | 64 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/run-tests.py b/run-tests.py index 5017f4c1..8810bf2b 100644 --- a/run-tests.py +++ b/run-tests.py @@ -1,17 +1,30 @@ - import glob import os +import shutil import subprocess import sys -def cleanup(out): - ret = '' - for s in out.decode('utf-8').split('\n'): - if len(s) > 1 and s[0] == '#': + +def cleanup(out: str) -> str: + parts = [] + for line in out.splitlines(): + if len(line) > 1 and line[0] == '#': continue - s = "".join(s.split()) - ret = ret + s - return ret + parts.append("".join(line.split())) + return "".join(parts) + + +# Check for required compilers and exit if any are missing +CLANG_EXE = shutil.which('clang') +if not CLANG_EXE: + sys.exit('Failed to run tests: clang compiler not found') + +GCC_EXE = shutil.which('gcc') +if not GCC_EXE: + sys.exit('Failed to run tests: gcc compiler not found') + +SIMPLECPP_EXE = './simplecpp' + commands = [] @@ -78,6 +91,21 @@ def cleanup(out): 'pr57580.c', ] + +def run(compiler_executable: str, compiler_args: list[str]) -> tuple[int, str, str]: + """Execute a compiler command and capture its exit code, stdout, and stderr.""" + compiler_cmd = [compiler_executable] + compiler_cmd.extend(compiler_args) + + with subprocess.Popen(compiler_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") as process: + stdout, stderr = process.communicate() + exit_code = process.returncode + + output = cleanup(stdout) + error = (stderr or "").strip() + return (exit_code, output, error) + + numberOfSkipped = 0 numberOfFailed = 0 numberOfFixed = 0 @@ -89,26 +117,12 @@ def cleanup(out): numberOfSkipped = numberOfSkipped + 1 continue - clang_cmd = ['clang'] - clang_cmd.extend(cmd.split(' ')) - p = subprocess.Popen(clang_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - comm = p.communicate() - clang_output = cleanup(comm[0]) + _, clang_output, _ = run(CLANG_EXE, cmd.split(' ')) - gcc_cmd = ['gcc'] - gcc_cmd.extend(cmd.split(' ')) - p = subprocess.Popen(gcc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - comm = p.communicate() - gcc_output = cleanup(comm[0]) + _, gcc_output, _ = run(GCC_EXE, cmd.split(' ')) - simplecpp_cmd = ['./simplecpp'] # -E is not supported and we bail out on unknown options - simplecpp_cmd.extend(cmd.replace('-E ', '', 1).split(' ')) - p = subprocess.Popen(simplecpp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - comm = p.communicate() - simplecpp_ec = p.returncode - simplecpp_output = cleanup(comm[0]) - simplecpp_err = comm[0].decode('utf-8').strip() + simplecpp_ec, simplecpp_output, simplecpp_err = run(SIMPLECPP_EXE, cmd.replace('-E ', '', 1).split(' ')) if simplecpp_output != clang_output and simplecpp_output != gcc_output: filename = cmd[cmd.rfind('/')+1:] From 04f4dfa2c62432d9bd4e36711fa06ff78395e4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 7 Sep 2025 23:41:58 +0200 Subject: [PATCH 302/381] main.cpp: added missing help entry for `-f` (#528) --- main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/main.cpp b/main.cpp index a6d14386..2397e847 100644 --- a/main.cpp +++ b/main.cpp @@ -106,6 +106,7 @@ int main(int argc, char **argv) std::cout << " -q Quiet mode (no output)." << std::endl; std::cout << " -is Use std::istream interface." << std::endl; std::cout << " -e Output errors only." << std::endl; + std::cout << " -f Fail when errors were encountered (exitcode 1)." << std::endl; std::exit(0); } From 68e9c3f4884bd6939a90faff766929a519e3eebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Sep 2025 11:41:33 +0200 Subject: [PATCH 303/381] simplecpp.h: made `Token::flags()` private (#526) --- simplecpp.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index ac367154..11accc74 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -135,14 +135,6 @@ namespace simplecpp { Token(const Token &tok) : macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), nextcond(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} - void flags() { - name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') - && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); - comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); - number = isNumberLike(string); - op = (string.size() == 1U && !name && !comment && !number) ? string[0] : '\0'; - } - const TokenString& str() const { return string; } @@ -197,6 +189,14 @@ namespace simplecpp { void printAll() const; void printOut() const; private: + void flags() { + name = (std::isalpha(static_cast(string[0])) || string[0] == '_' || string[0] == '$') + && (std::memchr(string.c_str(), '\'', string.size()) == nullptr); + comment = string.size() > 1U && string[0] == '/' && (string[1] == '/' || string[1] == '*'); + number = isNumberLike(string); + op = (string.size() == 1U && !name && !comment && !number) ? string[0] : '\0'; + } + TokenString string; std::set mExpandedFrom; From fd60cfe05948253c27c274c9198678b49d00ab32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Sep 2025 11:41:47 +0200 Subject: [PATCH 304/381] avoid impossible macro lookups in `preprocessToken()` (#532) --- simplecpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 84e4b54b..27b05519 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3191,7 +3191,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) { const simplecpp::Token * const tok = *tok1; - const simplecpp::MacroMap::const_iterator it = macros.find(tok->str()); + const simplecpp::MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); if (it != macros.end()) { simplecpp::TokenList value(files); try { From 919a25b0ab1edb0635802a236bdf2ee7e58bbe2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Sep 2025 12:09:49 +0200 Subject: [PATCH 305/381] added support for `c2y` and `gnu2y` as standards (#521) --- simplecpp.cpp | 8 +++++++- simplecpp.h | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 27b05519..5b8fde81 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3751,6 +3751,8 @@ simplecpp::cstd_t simplecpp::getCStd(const std::string &std) return C17; if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") return C23; + if (std == "c2y" || std == "gnu2y") + return C2Y; return CUnknown; } @@ -3772,6 +3774,10 @@ std::string simplecpp::getCStdString(cstd_t std) // Clang 14, 15, 16, 17 return "202000L" // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" return "202311L"; + case C2Y: + // supported by GCC 15+ and Clang 19+ + // Clang 19, 20, 21, 22 return "202400L" + return "202500L"; case CUnknown: return ""; } @@ -3823,7 +3829,7 @@ std::string simplecpp::getCppStdString(cppstd_t std) // Clang 17, 18 return "202302L" return "202302L"; case CPP26: - // supported by Clang 17+ + // supported by GCC 14+ and Clang 17+ return "202400L"; case CPPUnknown: return ""; diff --git a/simplecpp.h b/simplecpp.h index 11accc74..43a03730 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -67,7 +67,7 @@ namespace simplecpp { /** C code standard */ - enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 }; + enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23, C2Y }; /** C++ code standard */ enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; From 6c9eaf437a43467d56f10d57496e15acbca3a784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Sep 2025 12:10:01 +0200 Subject: [PATCH 306/381] pass `TokenList` by reference internally (#530) --- simplecpp.cpp | 148 +++++++++++++++++++++++++------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 5b8fde81..21c4bd82 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1528,7 +1528,7 @@ namespace simplecpp { * @return token after macro * @throw Can throw wrongNumberOfParameters or invalidHashHash */ - const Token * expand(TokenList * const output, + const Token * expand(TokenList & output, const Token * rawtok, const MacroMap ¯os, std::vector &inputFiles) const { @@ -1559,10 +1559,10 @@ namespace simplecpp { rawtokens2.push_back(new Token(rawtok->str(), rawtok1->location, rawtok->whitespaceahead)); rawtok = rawtok->next; } - if (expand(&output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros)) + if (expand(output2, rawtok1->location, rawtokens2.cfront(), macros, expandedmacros)) rawtok = rawtok1->next; } else { - rawtok = expand(&output2, rawtok->location, rawtok, macros, expandedmacros); + rawtok = expand(output2, rawtok->location, rawtok, macros, expandedmacros); } while (output2.cback() && rawtok) { unsigned int par = 0; @@ -1610,11 +1610,11 @@ namespace simplecpp { } if (!rawtok2 || par != 1U) break; - if (macro->second.expand(&output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != nullptr) + if (macro->second.expand(output2, rawtok->location, rawtokens2.cfront(), macros, expandedmacros) != nullptr) break; rawtok = rawtok2->next; } - output->takeTokens(output2); + output.takeTokens(output2); return rawtok; } @@ -1810,7 +1810,7 @@ namespace simplecpp { return parametertokens; } - const Token *appendTokens(TokenList *tokens, + const Token *appendTokens(TokenList &tokens, const Location &rawloc, const Token * const lpar, const MacroMap ¯os, @@ -1828,9 +1828,9 @@ namespace simplecpp { tok = expandHash(tokens, rawloc, tok, expandedmacros, parametertokens); } else { if (!expandArg(tokens, tok, rawloc, macros, expandedmacros, parametertokens)) { - tokens->push_back(new Token(*tok)); + tokens.push_back(new Token(*tok)); if (tok->macro.empty() && (par > 0 || tok->str() != "(")) - tokens->back()->macro = name(); + tokens.back()->macro = name(); } if (tok->op == '(') @@ -1843,12 +1843,12 @@ namespace simplecpp { tok = tok->next; } } - for (Token *tok2 = tokens->front(); tok2; tok2 = tok2->next) + for (Token *tok2 = tokens.front(); tok2; tok2 = tok2->next) tok2->location = lpar->location; return sameline(lpar,tok) ? tok : nullptr; } - const Token * expand(TokenList * const output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros) const { + const Token * expand(TokenList & output, const Location &loc, const Token * const nameTokInst, const MacroMap ¯os, std::set expandedmacros) const { expandedmacros.insert(nameTokInst->str()); #ifdef SIMPLECPP_DEBUG_MACRO_EXPANSION @@ -1858,15 +1858,15 @@ namespace simplecpp { usageList.push_back(loc); if (nameTokInst->str() == "__FILE__") { - output->push_back(new Token('\"'+loc.file()+'\"', loc)); + output.push_back(new Token('\"'+loc.file()+'\"', loc)); return nameTokInst->next; } if (nameTokInst->str() == "__LINE__") { - output->push_back(new Token(toString(loc.line), loc)); + output.push_back(new Token(toString(loc.line), loc)); return nameTokInst->next; } if (nameTokInst->str() == "__COUNTER__") { - output->push_back(new Token(toString(usageList.size()-1U), loc)); + output.push_back(new Token(toString(usageList.size()-1U), loc)); return nameTokInst->next; } @@ -1878,7 +1878,7 @@ namespace simplecpp { if (functionLike()) { // No arguments => not macro expansion if (nameTokInst->next && nameTokInst->next->op != '(') { - output->push_back(new Token(nameTokInst->str(), loc)); + output.push_back(new Token(nameTokInst->str(), loc)); return nameTokInst->next; } @@ -1927,7 +1927,7 @@ namespace simplecpp { } } - Token * const output_end_1 = output->back(); + Token * const output_end_1 = output.back(); const Token *valueToken2; const Token *endToken2; @@ -1952,20 +1952,20 @@ namespace simplecpp { throw invalidHashHash::unexpectedNewline(tok->location, name()); if (variadic && tok->op == ',' && tok->next->next->next->str() == args.back()) { Token *const comma = newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok); - output->push_back(comma); + output.push_back(comma); tok = expandToken(output, loc, tok->next->next->next, macros, expandedmacros, parametertokens2); - if (output->back() == comma) - output->deleteToken(comma); + if (output.back() == comma) + output.deleteToken(comma); continue; } TokenList new_output(files); - if (!expandArg(&new_output, tok, parametertokens2)) - output->push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok)); + if (!expandArg(new_output, tok, parametertokens2)) + output.push_back(newMacroToken(tok->str(), loc, isReplaced(expandedmacros), tok)); else if (new_output.empty()) // placemarker token - output->push_back(newMacroToken("", loc, isReplaced(expandedmacros))); + output.push_back(newMacroToken("", loc, isReplaced(expandedmacros))); else for (const Token *tok2 = new_output.cfront(); tok2; tok2 = tok2->next) - output->push_back(newMacroToken(tok2->str(), loc, isReplaced(expandedmacros), tok2)); + output.push_back(newMacroToken(tok2->str(), loc, isReplaced(expandedmacros), tok2)); tok = tok->next; } else { tok = expandToken(output, loc, tok, macros, expandedmacros, parametertokens2); @@ -1981,20 +1981,20 @@ namespace simplecpp { } if (numberOfHash == 4 && tok->next->location.col + 1 == tok->next->next->location.col) { // # ## # => ## - output->push_back(newMacroToken("##", loc, isReplaced(expandedmacros))); + output.push_back(newMacroToken("##", loc, isReplaced(expandedmacros))); tok = hashToken; continue; } if (numberOfHash >= 2 && tok->location.col + 1 < tok->next->location.col) { - output->push_back(new Token(*tok)); + output.push_back(new Token(*tok)); tok = tok->next; continue; } tok = tok->next; if (tok == endToken2) { - output->push_back(new Token(*tok->previous)); + output.push_back(new Token(*tok->previous)); break; } if (tok->op == '#') { @@ -2007,7 +2007,7 @@ namespace simplecpp { } if (!functionLike()) { - for (Token *tok = output_end_1 ? output_end_1->next : output->front(); tok; tok = tok->next) { + for (Token *tok = output_end_1 ? output_end_1->next : output.front(); tok; tok = tok->next) { tok->macro = nameTokInst->str(); } } @@ -2018,55 +2018,55 @@ namespace simplecpp { return functionLike() ? parametertokens2.back()->next : nameTokInst->next; } - const Token *recursiveExpandToken(TokenList *output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *recursiveExpandToken(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!(temp.cback() && temp.cback()->name && tok->next && tok->next->op == '(')) { - output->takeTokens(temp); + output.takeTokens(temp); return tok->next; } if (!sameline(tok, tok->next)) { - output->takeTokens(temp); + output.takeTokens(temp); return tok->next; } const MacroMap::const_iterator it = macros.find(temp.cback()->str()); if (it == macros.end() || expandedmacros.find(temp.cback()->str()) != expandedmacros.end()) { - output->takeTokens(temp); + output.takeTokens(temp); return tok->next; } const Macro &calledMacro = it->second; if (!calledMacro.functionLike()) { - output->takeTokens(temp); + output.takeTokens(temp); return tok->next; } TokenList temp2(files); temp2.push_back(new Token(temp.cback()->str(), tok->location)); - const Token * const tok2 = appendTokens(&temp2, loc, tok->next, macros, expandedmacros, parametertokens); + const Token * const tok2 = appendTokens(temp2, loc, tok->next, macros, expandedmacros, parametertokens); if (!tok2) return tok->next; - output->takeTokens(temp); - output->deleteToken(output->back()); + output.takeTokens(temp); + output.deleteToken(output.back()); calledMacro.expand(output, loc, temp2.cfront(), macros, expandedmacros); return tok2->next; } - const Token *expandToken(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandToken(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { // Not name.. if (!tok->name) { - output->push_back(newMacroToken(tok->str(), loc, true, tok)); + output.push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } // Macro parameter.. { TokenList temp(files); - if (expandArg(&temp, tok, loc, macros, expandedmacros, parametertokens)) { - if (tok->str() == "__VA_ARGS__" && temp.empty() && output->cback() && output->cback()->str() == "," && + if (expandArg(temp, tok, loc, macros, expandedmacros, parametertokens)) { + if (tok->str() == "__VA_ARGS__" && temp.empty() && output.cback() && output.cback()->str() == "," && tok->nextSkipComments() && tok->nextSkipComments()->str() == ")") - output->deleteToken(output->back()); + output.deleteToken(output.back()); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros, parametertokens); } } @@ -2080,29 +2080,29 @@ namespace simplecpp { const Macro &calledMacro = it->second; if (!calledMacro.functionLike()) { TokenList temp(files); - calledMacro.expand(&temp, loc, tok, macros, expandedmacros); + calledMacro.expand(temp, loc, tok, macros, expandedmacros); return recursiveExpandToken(output, temp, loc, tok, macros, expandedmacros2, parametertokens); } if (!sameline(tok, tok->next)) { - output->push_back(newMacroToken(tok->str(), loc, true, tok)); + output.push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } TokenList tokens(files); tokens.push_back(new Token(*tok)); const Token * tok2 = nullptr; if (tok->next->op == '(') - tok2 = appendTokens(&tokens, loc, tok->next, macros, expandedmacros, parametertokens); - else if (expandArg(&tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { + tok2 = appendTokens(tokens, loc, tok->next, macros, expandedmacros, parametertokens); + else if (expandArg(tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { tokens.front()->location = loc; if (tokens.cfront()->next && tokens.cfront()->next->op == '(') tok2 = tok->next; } if (!tok2) { - output->push_back(newMacroToken(tok->str(), loc, true, tok)); + output.push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } TokenList temp(files); - calledMacro.expand(&temp, loc, tokens.cfront(), macros, expandedmacros); + calledMacro.expand(temp, loc, tokens.cfront(), macros, expandedmacros); return recursiveExpandToken(output, temp, loc, tok2, macros, expandedmacros, parametertokens); } @@ -2122,25 +2122,25 @@ namespace simplecpp { std::string macroName = defToken->str(); if (defToken->next && defToken->next->op == '#' && defToken->next->next && defToken->next->next->op == '#' && defToken->next->next->next && defToken->next->next->next->name && sameline(defToken,defToken->next->next->next)) { TokenList temp(files); - if (expandArg(&temp, defToken, parametertokens)) + if (expandArg(temp, defToken, parametertokens)) macroName = temp.cback()->str(); - if (expandArg(&temp, defToken->next->next->next, parametertokens)) + if (expandArg(temp, defToken->next->next->next, parametertokens)) macroName += temp.cback() ? temp.cback()->str() : ""; else macroName += defToken->next->next->next->str(); lastToken = defToken->next->next->next; } const bool def = (macros.find(macroName) != macros.end()); - output->push_back(newMacroToken(def ? "1" : "0", loc, true)); + output.push_back(newMacroToken(def ? "1" : "0", loc, true)); return lastToken->next; } } - output->push_back(newMacroToken(tok->str(), loc, true, tok)); + output.push_back(newMacroToken(tok->str(), loc, true, tok)); return tok->next; } - bool expandArg(TokenList *output, const Token *tok, const std::vector ¶metertokens) const { + bool expandArg(TokenList &output, const Token *tok, const std::vector ¶metertokens) const { if (!tok->name) return false; @@ -2153,12 +2153,12 @@ namespace simplecpp { return true; for (const Token *partok = parametertokens[argnr]->next; partok != parametertokens[argnr + 1U]; partok = partok->next) - output->push_back(new Token(*partok)); + output.push_back(new Token(*partok)); return true; } - bool expandArg(TokenList *output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { + bool expandArg(TokenList &output, const Token *tok, const Location &loc, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { if (!tok->name) return false; const unsigned int argnr = getArgNum(tok->str()); @@ -2173,13 +2173,13 @@ namespace simplecpp { expandedmacros2.erase(name()); partok = it->second.expand(output, loc, partok, macros, std::move(expandedmacros2)); } else { - output->push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); - output->back()->macro = partok->macro; + output.push_back(newMacroToken(partok->str(), loc, isReplaced(expandedmacros), partok)); + output.back()->macro = partok->macro; partok = partok->next; } } - if (tok->whitespaceahead && output->back()) - output->back()->whitespaceahead = true; + if (tok->whitespaceahead && output.back()) + output.back()->whitespaceahead = true; return true; } @@ -2192,10 +2192,10 @@ namespace simplecpp { * @param parametertokens parameters given when expanding this macro * @return token after the X */ - const Token *expandHash(TokenList *output, const Location &loc, const Token *tok, const std::set &expandedmacros, const std::vector ¶metertokens) const { + const Token *expandHash(TokenList &output, const Location &loc, const Token *tok, const std::set &expandedmacros, const std::vector ¶metertokens) const { TokenList tokenListHash(files); const MacroMap macros2; // temporarily bypass macro expansion - tok = expandToken(&tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens); + tok = expandToken(tokenListHash, loc, tok->next, macros2, expandedmacros, parametertokens); std::ostringstream ostr; ostr << '\"'; for (const Token *hashtok = tokenListHash.cfront(), *next; hashtok; hashtok = next) { @@ -2205,7 +2205,7 @@ namespace simplecpp { ostr << ' '; } ostr << '\"'; - output->push_back(newMacroToken(escapeString(ostr.str()), loc, isReplaced(expandedmacros))); + output.push_back(newMacroToken(escapeString(ostr.str()), loc, isReplaced(expandedmacros))); return tok; } @@ -2221,8 +2221,8 @@ namespace simplecpp { * @param expandResult expand ## result i.e. "AB"? * @return token after B */ - const Token *expandHashHash(TokenList *output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, bool expandResult=true) const { - Token *A = output->back(); + const Token *expandHashHash(TokenList &output, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens, bool expandResult=true) const { + Token *A = output.back(); if (!A) throw invalidHashHash(tok->location, name(), "Missing first argument"); if (!sameline(tok, tok->next) || !sameline(tok, tok->next->next)) @@ -2253,20 +2253,20 @@ namespace simplecpp { // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. // TODO The question is whether the ## or varargs may still apply, and how to provoke? - if (expandArg(&tokensB, B, parametertokens)) { + if (expandArg(tokensB, B, parametertokens)) { for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; } else { tokensB.push_back(new Token(*B)); tokensB.back()->location = loc; } - output->takeTokens(tokensB); + output.takeTokens(tokensB); } else { std::string strAB; const bool varargs = variadic && !args.empty() && B->str() == args[args.size()-1U]; - if (expandArg(&tokensB, B, parametertokens)) { + if (expandArg(tokensB, B, parametertokens)) { if (tokensB.empty()) strAB = A->str(); else if (varargs && A->op == ',') @@ -2292,27 +2292,27 @@ namespace simplecpp { } if (varargs && tokensB.empty() && tok->previous->str() == ",") - output->deleteToken(A); + output.deleteToken(A); else if (strAB != "," && macros.find(strAB) == macros.end()) { A->setstr(strAB); for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; - output->takeTokens(tokensB); + output.takeTokens(tokensB); } else if (sameline(B, nextTok) && sameline(B, nextTok->next) && nextTok->op == '#' && nextTok->next->op == '#') { TokenList output2(files); output2.push_back(new Token(strAB, tok->location)); - nextTok = expandHashHash(&output2, loc, nextTok, macros, expandedmacros, parametertokens); - output->deleteToken(A); - output->takeTokens(output2); + nextTok = expandHashHash(output2, loc, nextTok, macros, expandedmacros, parametertokens); + output.deleteToken(A); + output.takeTokens(output2); } else { - output->deleteToken(A); + output.deleteToken(A); TokenList tokens(files); tokens.push_back(new Token(strAB, tok->location)); // for function like macros, push the (...) if (tokensB.empty() && sameline(B,B->next) && B->next->op=='(') { const MacroMap::const_iterator it = macros.find(strAB); if (it != macros.end() && expandedmacros.find(strAB) == expandedmacros.end() && it->second.functionLike()) { - const Token * const tok2 = appendTokens(&tokens, loc, B->next, macros, expandedmacros, parametertokens); + const Token * const tok2 = appendTokens(tokens, loc, B->next, macros, expandedmacros, parametertokens); if (tok2) nextTok = tok2->next; } @@ -2320,10 +2320,10 @@ namespace simplecpp { if (expandResult) expandToken(output, loc, tokens.cfront(), macros, expandedmacros, parametertokens); else - output->takeTokens(tokens); + output.takeTokens(tokens); for (Token *b = tokensB.front(); b; b = b->next) b->location = loc; - output->takeTokens(tokensB); + output.takeTokens(tokensB); } } @@ -3195,7 +3195,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token if (it != macros.end()) { simplecpp::TokenList value(files); try { - *tok1 = it->second.expand(&value, tok, macros, files); + *tok1 = it->second.expand(value, tok, macros, files); } catch (simplecpp::Macro::Error &err) { if (outputList) { simplecpp::Output out(files); From ad24c6e5b5d8df9320cac179068c24adb798d9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Sep 2025 14:41:54 +0200 Subject: [PATCH 307/381] fixes #352 - internally default to latest standard if none is provided (#356) --- simplecpp.cpp | 2 +- test.cpp | 53 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 21c4bd82..b1505cb6 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2554,7 +2554,7 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::map= "201703L"); + return std_ver.empty() || (std_ver >= "201703L"); } static bool isGnu(const simplecpp::DUI &dui) diff --git a/test.cpp b/test.cpp index 0ecaa3b1..f1fe2d1c 100644 --- a/test.cpp +++ b/test.cpp @@ -1562,11 +1562,13 @@ static void has_include_1() "#endif"; simplecpp::DUI dui; dui.includePaths.push_back(testSourceDir); - dui.std = "c++17"; - ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++14"; ASSERT_EQUALS("", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); + dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + dui.std = "c++20"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); } static void has_include_2() @@ -1580,9 +1582,13 @@ static void has_include_2() "#endif"; simplecpp::DUI dui; dui.includePaths.push_back(testSourceDir); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); dui.std = "c++17"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); + dui.std = "c++20"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); } static void has_include_3() @@ -1595,13 +1601,24 @@ static void has_include_3() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.std = "c++17"; + // Test file not found... + ASSERT_EQUALS("\n\n\n\nB", preprocess(code, dui)); // we default to latest standard internally + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); + dui.std = "c++17"; ASSERT_EQUALS("\n\n\n\nB", preprocess(code, dui)); + // Unless -I is set (preferably, we should differentiate -I and -isystem...) dui.includePaths.push_back(testSourceDir + "/testsuite"); + dui.std = ""; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); + dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + dui.std = "c++20"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); } static void has_include_4() @@ -1614,10 +1631,14 @@ static void has_include_4() " #endif\n" "#endif"; simplecpp::DUI dui; + dui.includePaths.push_back(testSourceDir); // we default to latest standard internally + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); dui.std = "c++17"; - dui.includePaths.push_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); + dui.std = "c++20"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); } static void has_include_5() @@ -1630,10 +1651,14 @@ static void has_include_5() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.includePaths.push_back(testSourceDir); + dui.std = "c++14"; + ASSERT_EQUALS("", preprocess(code, dui)); + dui.std = "c++17"; + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); + dui.std = "c++20"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); } static void has_include_6() @@ -1646,10 +1671,12 @@ static void has_include_6() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.std = "gnu99"; dui.includePaths.push_back(testSourceDir); + ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally + dui.std = "c++99"; + ASSERT_EQUALS("", preprocess(code, dui)); + dui.std = "gnu99"; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); - ASSERT_EQUALS("", preprocess(code)); } static void strict_ansi_1() @@ -2983,6 +3010,7 @@ static void stdcVersionDefine() " __STDC_VERSION__\n" "#endif\n"; simplecpp::DUI dui; + ASSERT_EQUALS("", preprocess(code, dui)); dui.std = "c11"; ASSERT_EQUALS("\n201112L", preprocess(code, dui)); } @@ -2993,6 +3021,7 @@ static void cpluscplusDefine() " __cplusplus\n" "#endif\n"; simplecpp::DUI dui; + ASSERT_EQUALS("", preprocess(code, dui)); dui.std = "c++11"; ASSERT_EQUALS("\n201103L", preprocess(code, dui)); } From 8c80e7c0f441c95f1bb3433bf5bcd181f244314e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Sep 2025 10:54:16 +0200 Subject: [PATCH 308/381] disabled `-Wpoison-system-directories` warnings on macOS for now (#542) --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8799b7a4..0cc62e88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,11 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # use force DWARF 4 debug format since not all tools might be able to handle DWARF 5 yet - e.g. valgrind on ubuntu 20.04 add_compile_options(-gdwarf-4) endif() + if (APPLE) + # CMake is sometimes chosing the wrong compiler on macos-* runners + # see https://github.com/actions/runner/issues/4034 + add_compile_options(-Wno-poison-system-directories) + endif() endif() add_library(simplecpp_obj OBJECT simplecpp.cpp) From 4fc9ec92161ff9840fd2039986f8933c80be069a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Sep 2025 11:04:09 +0200 Subject: [PATCH 309/381] uninstall `man-db` package on ubuntu to avoid potential stalls in package installations (#543) --- .github/workflows/CI-unixish.yml | 8 ++++++++ .github/workflows/clang-tidy.yml | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index adb91138..74100265 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -29,6 +29,14 @@ jobs: with: persist-credentials: false + # the man-db trigger causes package installations to stall for several minutes at times. so just drop the package. + # see https://github.com/actions/runner/issues/4030 + - name: Remove man-db package on ubuntu + if: matrix.os == 'ubuntu-24.04' + run: | + sudo apt-get update + sudo apt-get remove man-db + - name: Install missing software on ubuntu if: matrix.os == 'ubuntu-24.04' run: | diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index fd71bdfe..9ebb6683 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -17,6 +17,13 @@ jobs: with: persist-credentials: false + # the man-db trigger causes package installations to stall for several minutes at times. so just drop the package. + # see https://github.com/actions/runner/issues/4030 + - name: Remove man-db package + run: | + sudo apt-get update + sudo apt-get remove man-db + - name: Install missing software run: | sudo apt-get update From de1d67bcd6129858261050bed80e0f6e4e487dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 21 Sep 2025 20:08:14 +0200 Subject: [PATCH 310/381] fixed #524 - use `CreateFileA()` for `_WIN32` only (fixes includes not found on MinGW) (#541) Co-authored-by: glank --- simplecpp.cpp | 13 ++++++++++--- simplecpp.h | 10 +++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index b1505cb6..4d64d582 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3,15 +3,22 @@ * Copyright (C) 2016-2023 simplecpp team */ -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) -# define _WIN32_WINNT 0x0602 +#if defined(_WIN32) +# ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0602 +# endif # define NOMINMAX +# define WIN32_LEAN_AND_MEAN # include # undef ERROR #endif #include "simplecpp.h" +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) +# define SIMPLECPP_WINDOWS +#endif + #include #include #include @@ -3082,7 +3089,7 @@ std::pair simplecpp::FileDataCache::get(const std:: bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) { -#ifdef SIMPLECPP_WINDOWS +#ifdef _WIN32 HANDLE hFile = CreateFileA(path.c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) diff --git a/simplecpp.h b/simplecpp.h index 43a03730..ef748b3e 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -6,10 +6,6 @@ #ifndef simplecppH #define simplecppH -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) -# define SIMPLECPP_WINDOWS -#endif - #include #include #include @@ -43,7 +39,7 @@ # define SIMPLECPP_LIB #endif -#ifdef SIMPLECPP_WINDOWS +#ifdef _WIN32 # include #else # include @@ -471,7 +467,7 @@ namespace simplecpp { private: struct FileID { -#ifdef SIMPLECPP_WINDOWS +#ifdef _WIN32 struct { std::uint64_t VolumeSerialNumber; struct { @@ -495,7 +491,7 @@ namespace simplecpp { #endif struct Hasher { std::size_t operator()(const FileID &id) const { -#ifdef SIMPLECPP_WINDOWS +#ifdef _WIN32 return static_cast(id.fileIdInfo.FileId.IdentifierHi ^ id.fileIdInfo.FileId.IdentifierLo ^ id.fileIdInfo.VolumeSerialNumber); #else From 32cccf5d74b8b0356fcb161e3ada23d7b3ccb43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Sep 2025 09:22:11 +0200 Subject: [PATCH 311/381] main.cpp: improved handling of empty options (#540) --- main.cpp | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/main.cpp b/main.cpp index 2397e847..2ddf78cd 100644 --- a/main.cpp +++ b/main.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include int main(int argc, char **argv) @@ -30,49 +31,76 @@ int main(int argc, char **argv) const char c = arg[1]; switch (c) { case 'D': { // define symbol + found = true; const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; + if (!value) { + std::cout << "error: option -D with no value." << std::endl; + error = true; + break; + } dui.defines.push_back(value); - found = true; break; } case 'U': { // undefine symbol + found = true; const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; + if (!value) { + std::cout << "error: option -U with no value." << std::endl; + error = true; + break; + } dui.undefined.insert(value); - found = true; break; } case 'I': { // include path - const char * const value = arg[2] ? (argv[i] + 2) : argv[++i]; - dui.includePaths.push_back(value); found = true; + const char * const value = arg[2] ? (arg + 2) : argv[++i]; + if (!value) { + std::cout << "error: option -I with no value." << std::endl; + error = true; + break; + } + dui.includePaths.push_back(value); break; } case 'i': if (std::strncmp(arg, "-include=",9)==0) { - dui.includes.push_back(arg+9); found = true; + std::string value = arg + 9; + if (value.empty()) { + std::cout << "error: option -include with no value." << std::endl; + error = true; + break; + } + dui.includes.push_back(std::move(value)); } else if (std::strncmp(arg, "-is",3)==0) { - use_istream = true; found = true; + use_istream = true; } break; case 's': if (std::strncmp(arg, "-std=",5)==0) { - dui.std = arg + 5; found = true; + std::string value = arg + 5; + if (value.empty()) { + std::cout << "error: option -std with no value." << std::endl; + error = true; + break; + } + dui.std = std::move(value); } break; case 'q': - quiet = true; found = true; + quiet = true; break; case 'e': - error_only = true; found = true; + error_only = true; break; case 'f': - fail_on_error = true; found = true; + fail_on_error = true; break; } if (!found) { From 1420eb61d45b6e89d7be898052a455791184913a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Sep 2025 11:06:45 +0200 Subject: [PATCH 312/381] fixed #462 - added CLI option `-l` to print line numbers (#463) --- main.cpp | 8 +++++++- simplecpp.cpp | 15 ++++++++++++--- simplecpp.h | 4 ++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/main.cpp b/main.cpp index 2ddf78cd..cbae6ac8 100644 --- a/main.cpp +++ b/main.cpp @@ -19,6 +19,7 @@ int main(int argc, char **argv) const char *filename = nullptr; bool use_istream = false; bool fail_on_error = false; + bool linenrs = false; // Settings.. simplecpp::DUI dui; @@ -102,6 +103,10 @@ int main(int argc, char **argv) found = true; fail_on_error = true; break; + case 'l': + linenrs = true; + found = true; + break; } if (!found) { std::cout << "error: option '" << arg << "' is unknown." << std::endl; @@ -135,6 +140,7 @@ int main(int argc, char **argv) std::cout << " -is Use std::istream interface." << std::endl; std::cout << " -e Output errors only." << std::endl; std::cout << " -f Fail when errors were encountered (exitcode 1)." << std::endl; + std::cout << " -l Print lines numbers." << std::endl; std::exit(0); } @@ -165,7 +171,7 @@ int main(int argc, char **argv) // Output if (!quiet) { if (!error_only) - std::cout << outputTokens.stringify() << std::endl; + std::cout << outputTokens.stringify(linenrs) << std::endl; for (const simplecpp::Output &output : outputList) { std::cerr << output.location.file() << ':' << output.location.line << ": "; diff --git a/simplecpp.cpp b/simplecpp.cpp index 4d64d582..0621fe06 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -553,24 +553,33 @@ void simplecpp::TokenList::push_back(Token *tok) backToken = tok; } -void simplecpp::TokenList::dump() const +void simplecpp::TokenList::dump(bool linenrs) const { - std::cout << stringify() << std::endl; + std::cout << stringify(linenrs) << std::endl; } -std::string simplecpp::TokenList::stringify() const +std::string simplecpp::TokenList::stringify(bool linenrs) const { std::ostringstream ret; Location loc(files); + bool filechg = true; for (const Token *tok = cfront(); tok; tok = tok->next) { if (tok->location.line < loc.line || tok->location.fileIndex != loc.fileIndex) { ret << "\n#line " << tok->location.line << " \"" << tok->location.file() << "\"\n"; loc = tok->location; + filechg = true; + } + + if (linenrs && filechg) { + ret << loc.line << ": "; + filechg = false; } while (tok->location.line > loc.line) { ret << '\n'; loc.line++; + if (linenrs) + ret << loc.line << ": "; } if (sameline(tok->previous, tok)) diff --git a/simplecpp.h b/simplecpp.h index ef748b3e..a014a6fc 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -284,8 +284,8 @@ namespace simplecpp { } void push_back(Token *tok); - void dump() const; - std::string stringify() const; + void dump(bool linenrs = false) const; + std::string stringify(bool linenrs = false) const; void readfile(Stream &stream, const std::string &filename=std::string(), OutputList *outputList = nullptr); void constFold(); From 67f8e0e2436c99eeabbc5115af12ca40479ba4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Sep 2025 11:07:34 +0200 Subject: [PATCH 313/381] run-tests.py: improved output when test failed (#545) --- run-tests.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/run-tests.py b/run-tests.py index 8810bf2b..20f59a49 100644 --- a/run-tests.py +++ b/run-tests.py @@ -92,7 +92,7 @@ def cleanup(out: str) -> str: ] -def run(compiler_executable: str, compiler_args: list[str]) -> tuple[int, str, str]: +def run(compiler_executable: str, compiler_args: list[str]) -> tuple[int, str, str, str]: """Execute a compiler command and capture its exit code, stdout, and stderr.""" compiler_cmd = [compiler_executable] compiler_cmd.extend(compiler_args) @@ -103,7 +103,7 @@ def run(compiler_executable: str, compiler_args: list[str]) -> tuple[int, str, s output = cleanup(stdout) error = (stderr or "").strip() - return (exit_code, output, error) + return (exit_code, output, stdout, error) numberOfSkipped = 0 @@ -117,20 +117,32 @@ def run(compiler_executable: str, compiler_args: list[str]) -> tuple[int, str, s numberOfSkipped = numberOfSkipped + 1 continue - _, clang_output, _ = run(CLANG_EXE, cmd.split(' ')) + _, clang_output_c, clang_output, _ = run(CLANG_EXE, cmd.split(' ')) - _, gcc_output, _ = run(GCC_EXE, cmd.split(' ')) + _, gcc_output_c, gcc_output, _ = run(GCC_EXE, cmd.split(' ')) # -E is not supported and we bail out on unknown options - simplecpp_ec, simplecpp_output, simplecpp_err = run(SIMPLECPP_EXE, cmd.replace('-E ', '', 1).split(' ')) + simplecpp_ec, simplecpp_output_c, simplecpp_output, simplecpp_err = run(SIMPLECPP_EXE, cmd.replace('-E ', '', 1).split(' ')) - if simplecpp_output != clang_output and simplecpp_output != gcc_output: + if simplecpp_output_c != clang_output_c and simplecpp_output_c != gcc_output_c: filename = cmd[cmd.rfind('/')+1:] if filename in todo: print('TODO ' + cmd) usedTodos.append(filename) else: print('FAILED ' + cmd) + print('---expected (clang):') + print(clang_output_c) + print('---expected (gcc):') + print(gcc_output_c) + print('---actual:') + print(simplecpp_output_c) + print('---output (clang):') + print(clang_output) + print('---output (gcc):') + print(gcc_output) + print('---output (simplecpp):') + print(simplecpp_output) if simplecpp_ec: print('simplecpp failed - ' + simplecpp_err) numberOfFailed = numberOfFailed + 1 From 3deeb916acdb04be19811289dc13a2dc09da06c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 23 Sep 2025 19:25:00 +0200 Subject: [PATCH 314/381] CI-windows.yml: added `windows-11-arm` (#544) --- .github/workflows/CI-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 767bba6c..7985995f 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -18,7 +18,7 @@ jobs: build: strategy: matrix: - os: [windows-2022, windows-2025] + os: [windows-2022, windows-2025, windows-11-arm] config: [Release, Debug] fail-fast: false From 7f59eec64312aec7456ebdabb39e351280c11ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 25 Sep 2025 08:21:12 +0200 Subject: [PATCH 315/381] fixed #498 - avoid potential leak in `simplecpp::Macro::parseDefine()` (#527) --- simplecpp.cpp | 6 ++++-- test.cpp | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0621fe06..799d6c9b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1709,7 +1709,9 @@ namespace simplecpp { nameTokDef = nametoken; variadic = false; variadicOpt = false; + delete optExpandValue; optExpandValue = nullptr; + delete optNoExpandValue; optNoExpandValue = nullptr; if (!nameTokDef) { valueToken = endToken = nullptr; @@ -2383,8 +2385,8 @@ namespace simplecpp { bool variadicOpt; /** Expansion value for varadic macros with __VA_OPT__ expanded and discarded respectively */ - const TokenList *optExpandValue; - const TokenList *optNoExpandValue; + const TokenList *optExpandValue{}; + const TokenList *optNoExpandValue{}; /** was the value of this macro actually defined in the code? */ bool valueDefinedInCode_; diff --git a/test.cpp b/test.cpp index f1fe2d1c..b9869b5b 100644 --- a/test.cpp +++ b/test.cpp @@ -3246,6 +3246,7 @@ static void safe_api() #endif } +// crashes detected by fuzzer static void fuzz_crash() { { @@ -3260,6 +3261,16 @@ static void fuzz_crash() } } +// memory leaks detected by LSAN/valgrind +static void leak() +{ + { // #498 + const char code[] = "#define e(...)__VA_OPT__()\n" + "#define e\n"; + (void)preprocess(code, simplecpp::DUI()); + } +} + int main(int argc, char **argv) { TEST_CASE(backslash); @@ -3516,5 +3527,7 @@ int main(int argc, char **argv) TEST_CASE(fuzz_crash); + TEST_CASE(leak); + return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 52fb0b914fe4bc74df5c34b4093aa78505dba180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 29 Sep 2025 02:20:48 +0200 Subject: [PATCH 316/381] simplecpp.h: prevent internal define `SIMPLECPP_LIB` from spilling (#525) --- simplecpp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/simplecpp.h b/simplecpp.h index a014a6fc..4675d1f3 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -563,4 +563,6 @@ namespace simplecpp { # pragma warning(pop) #endif +#undef SIMPLECPP_LIB + #endif From 60e743a208f76c7cd68d3e8501347da9f9b39035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 29 Sep 2025 02:21:42 +0200 Subject: [PATCH 317/381] bail out on CMake related issues in the CI (#550) --- .github/workflows/CI-unixish.yml | 2 +- .github/workflows/CI-windows.yml | 2 +- .github/workflows/clang-tidy.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 74100265..07ee9731 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -82,7 +82,7 @@ jobs: - name: Run CMake run: | - cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On + cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On - name: CMake simplecpp run: | diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 7985995f..ccf208fa 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -45,7 +45,7 @@ jobs: - name: Run CMake run: | - cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_COMPILE_WARNING_AS_ERROR=On . || exit /b !errorlevel! + cmake -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On . || exit /b !errorlevel! - name: Build run: | diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 9ebb6683..8bc15d1a 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -42,7 +42,7 @@ jobs: - name: Prepare CMake run: | - cmake -S . -B cmake.output -G "Unix Makefiles" -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: CXX: clang-20 From d86671f47544882a1cb2860a7e007d9a20437a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 29 Sep 2025 02:35:31 +0200 Subject: [PATCH 318/381] clang-tidy.yml: updated to Clang 21 (#421) --- .clang-tidy | 1 + .github/workflows/clang-tidy.yml | 10 +++++----- CMakeLists.txt | 17 +++++++++++++---- simplecpp.cpp | 27 +++++++++++++++++---------- simplecpp.h | 1 + 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index c03da523..806a8608 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -49,6 +49,7 @@ Checks: > -readability-magic-numbers, -readability-redundant-inline-specifier, -readability-simplify-boolean-expr, + -readability-use-concise-preprocessor-directives, -readability-uppercase-literal-suffix, -performance-avoid-endl, -performance-enum-size, diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 8bc15d1a..4b52d7bc 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -33,19 +33,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 20 - sudo apt-get install clang-tidy-20 + sudo ./llvm.sh 21 + sudo apt-get install clang-tidy-21 - name: Verify clang-tidy configuration run: | - clang-tidy-20 --verify-config + clang-tidy-21 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: - CXX: clang-20 + CXX: clang-21 - name: Clang-Tidy run: | - run-clang-tidy-20 -q -j $(nproc) -p=cmake.output + run-clang-tidy-21 -q -j $(nproc) -p=cmake.output diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cc62e88..efdc0d96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,16 +49,25 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # no need for c++98 compatibility add_compile_options(-Wno-c++98-compat-pedantic) # these are not really fixable - add_compile_options(-Wno-exit-time-destructors -Wno-global-constructors -Wno-weak-vtables) + add_compile_options(-Wno-exit-time-destructors) + add_compile_options(-Wno-global-constructors) + add_compile_options(-Wno-weak-vtables) add_compile_options_safe(-Wno-unsafe-buffer-usage) + add_compile_options_safe(-Wno-nrvo) # we are not interested in these - add_compile_options(-Wno-multichar -Wno-four-char-constants) + add_compile_options(-Wno-multichar) + add_compile_options(-Wno-four-char-constants) # ignore C++11-specific warning - add_compile_options(-Wno-suggest-override -Wno-suggest-destructor-override) + add_compile_options(-Wno-suggest-override) + add_compile_options(-Wno-suggest-destructor-override) # contradicts -Wcovered-switch-default add_compile_options(-Wno-switch-default) # TODO: fix these? - add_compile_options(-Wno-padded -Wno-sign-conversion -Wno-implicit-int-conversion -Wno-shorten-64-to-32 -Wno-shadow-field-in-constructor) + add_compile_options(-Wno-padded) + add_compile_options(-Wno-sign-conversion) + add_compile_options(-Wno-implicit-int-conversion) + add_compile_options(-Wno-shorten-64-to-32) + add_compile_options(-Wno-shadow-field-in-constructor) if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 14) # TODO: verify this regression still exists in clang-15 diff --git a/simplecpp.cpp b/simplecpp.cpp index 799d6c9b..320836be 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -874,7 +874,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, back()->setstr(currentToken); location.adjust(currentToken); if (currentToken.find_first_of("\r\n") == std::string::npos) - location.col += 2 + 2 * delim.size(); + location.col += 2 + (2 * delim.size()); else location.col += 1 + delim.size(); @@ -1329,6 +1329,7 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) void simplecpp::TokenList::constFoldQuestionOp(Token **tok1) { bool gotoTok1 = false; + // NOLINTNEXTLINE(misc-const-correctness) - technically correct but used to access non-const data for (Token *tok = *tok1; tok && tok->op != ')'; tok = gotoTok1 ? *tok1 : tok->next) { gotoTok1 = false; if (tok->str() != "?") @@ -1508,7 +1509,12 @@ namespace simplecpp { } Macro(const Macro &other) : nameTokDef(nullptr), files(other.files), tokenListDefine(other.files), valueDefinedInCode_(other.valueDefinedInCode_) { - *this = other; + // TODO: remove the try-catch - see #537 + // avoid bugprone-exception-escape clang-tidy warning + try { + *this = other; + } + catch (const Error&) {} // NOLINT(bugprone-empty-catch) } ~Macro() { @@ -1945,6 +1951,7 @@ namespace simplecpp { } } + // NOLINTNEXTLINE(misc-const-correctness) - technically correct but used to access non-const data Token * const output_end_1 = output.back(); const Token *valueToken2; @@ -2250,7 +2257,7 @@ namespace simplecpp { const bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); const bool unexpectedA = (!A->name && !A->number && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar); - Token * const B = tok->next->next; + const Token * const B = tok->next->next; if (!B->name && !B->number && B->op && !B->isOneOf("#=")) throw invalidHashHash::unexpectedToken(tok->location, name(), B); @@ -2528,11 +2535,11 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::mapnext) { if (tok->str() != "sizeof") continue; - simplecpp::Token *tok1 = tok->next; + const simplecpp::Token *tok1 = tok->next; if (!tok1) { throw std::runtime_error("missing sizeof argument"); } - simplecpp::Token *tok2 = tok1->next; + const simplecpp::Token *tok2 = tok1->next; if (!tok2) { throw std::runtime_error("missing sizeof argument"); } @@ -2547,7 +2554,7 @@ static void simplifySizeof(simplecpp::TokenList &expr, const std::mapnext) { + for (const simplecpp::Token *typeToken = tok1; typeToken != tok2; typeToken = typeToken->next) { if ((typeToken->str() == "unsigned" || typeToken->str() == "signed") && typeToken->next->name) continue; if (typeToken->str() == "*" && type.find('*') != std::string::npos) @@ -2598,11 +2605,11 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->str() != HAS_INCLUDE) continue; - simplecpp::Token *tok1 = tok->next; + const simplecpp::Token *tok1 = tok->next; if (!tok1) { throw std::runtime_error("missing __has_include argument"); } - simplecpp::Token *tok2 = tok1->next; + const simplecpp::Token *tok2 = tok1->next; if (!tok2) { throw std::runtime_error("missing __has_include argument"); } @@ -2620,7 +2627,7 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI const bool systemheader = (tok1 && tok1->op == '<'); std::string header; if (systemheader) { - simplecpp::Token *tok3 = tok1->next; + const simplecpp::Token *tok3 = tok1->next; if (!tok3) { throw std::runtime_error("missing __has_include closing angular bracket"); } @@ -2631,7 +2638,7 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI } } - for (simplecpp::Token *headerToken = tok1->next; headerToken != tok3; headerToken = headerToken->next) + for (const simplecpp::Token *headerToken = tok1->next; headerToken != tok3; headerToken = headerToken->next) header += headerToken->str(); } else { header = tok1->str().substr(1U, tok1->str().size() - 2U); diff --git a/simplecpp.h b/simplecpp.h index 4675d1f3..c6c3b515 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -426,6 +426,7 @@ namespace simplecpp { std::pair get(const std::string &sourcefile, const std::string &header, const DUI &dui, bool systemheader, std::vector &filenames, OutputList *outputList); void insert(FileData data) { + // NOLINTNEXTLINE(misc-const-correctness) - FP FileData *const newdata = new FileData(std::move(data)); mData.emplace_back(newdata); From dcc0380ae268f223c193434b6a1df6939305abc1 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:39:32 +0200 Subject: [PATCH 319/381] Fix #546 fuzzing crash in simplecpp::preprocess() (#553) --- simplecpp.cpp | 8 +++++--- test.cpp | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 320836be..c5760145 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3602,9 +3602,11 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL header = tok->str().substr(1U, tok->str().size() - 2U); closingAngularBracket = true; } - std::ifstream f; - const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); - expr.push_back(new Token(header2.empty() ? "0" : "1", tok->location)); + if (tok) { + std::ifstream f; + const std::string header2 = openHeader(f,dui,sourcefile,header,systemheader); + expr.push_back(new Token(header2.empty() ? "0" : "1", tok->location)); + } } if (par) tok = tok ? tok->next : nullptr; diff --git a/test.cpp b/test.cpp index b9869b5b..f42b2189 100644 --- a/test.cpp +++ b/test.cpp @@ -3259,6 +3259,12 @@ static void fuzz_crash() "foo(f##oo(intp))\n"; (void)preprocess(code, simplecpp::DUI()); // do not crash } + { // #546 + const char code[] = "#if __has_include<\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); // do not crash + ASSERT_EQUALS("file0,1,syntax_error,failed to evaluate #if condition\n", toString(outputList)); + } } // memory leaks detected by LSAN/valgrind From dcc4fddb4942dfd309897a8e867c9b61b170f79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 1 Oct 2025 17:28:29 +0200 Subject: [PATCH 320/381] main.cpp: error out when file/path provided by `-I` or `-include=`, or the input does not exist (#435) --- integration_test.py | 148 ++++++++++++++++++++++++++++++++++++++++++++ main.cpp | 75 +++++++++++++++------- 2 files changed, 202 insertions(+), 21 deletions(-) diff --git a/integration_test.py b/integration_test.py index a59ae338..e5497db3 100644 --- a/integration_test.py +++ b/integration_test.py @@ -298,3 +298,151 @@ def test_pragma_once_matching(record_property, tmpdir): assert stdout.count("ONCE") == 1 assert stderr == "" + + +def test_input_multiple(record_property, tmpdir): + test_file = os.path.join(tmpdir, "test.c") + with open(test_file, 'w'): + pass + + test_file_1 = os.path.join(tmpdir, "test1.c") + with open(test_file_1, 'w'): + pass + + args = [ + 'test.c', + 'test1.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: multiple filenames specified\n" == stdout + + +def test_input_missing(record_property, tmpdir): + args = [ + 'missing.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not open file 'missing.c'\n" == stdout + + +def test_input_dir(record_property, tmpdir): + test_dir = os.path.join(tmpdir, "test") + os.mkdir(test_dir) + + args = [ + 'test' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not open file 'test'\n" == stdout + + +def test_incpath_missing(record_property, tmpdir): + test_file = os.path.join(tmpdir, "test.c") + with open(test_file, 'w'): + pass + + test_dir = os.path.join(tmpdir, "test") + os.mkdir(test_dir) + + args = [ + '-Itest', + '-Imissing', + 'test.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not find include path 'missing'\n" == stdout + + +def test_incpath_file(record_property, tmpdir): + test_file = os.path.join(tmpdir, "test.c") + with open(test_file, 'w'): + pass + + inc_dir = os.path.join(tmpdir, "inc") + os.mkdir(inc_dir) + + inc_file = os.path.join(tmpdir, "inc.h") + with open(test_file, 'w'): + pass + + args = [ + '-Iinc', + '-Iinc.h', + 'test.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not find include path 'inc.h'\n" == stdout + + +def test_incfile_missing(record_property, tmpdir): + test_file = os.path.join(tmpdir, "test.c") + with open(test_file, 'w'): + pass + + inc_file = os.path.join(tmpdir, "inc.h") + with open(inc_file, 'w'): + pass + + args = [ + '-include=inc.h', + '-include=missing.h', + 'test.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not open include 'missing.h'\n" == stdout + + +def test_incpath_dir(record_property, tmpdir): + test_file = os.path.join(tmpdir, "test.c") + with open(test_file, 'w'): + pass + + inc_file = os.path.join(tmpdir, "inc.h") + with open(inc_file, 'w'): + pass + + inc_dir = os.path.join(tmpdir, "inc") + os.mkdir(inc_dir) + + args = [ + '-include=inc.h', + '-include=inc', + 'test.c' + ] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert '' == stderr + assert "error: could not open include 'inc'\n" == stdout diff --git a/main.cpp b/main.cpp index cbae6ac8..0196d6e4 100644 --- a/main.cpp +++ b/main.cpp @@ -9,10 +9,20 @@ #include #include #include +#include #include #include #include +static bool isDir(const std::string& path) +{ + struct stat file_stat; + if (stat(path.c_str(), &file_stat) == -1) + return false; + + return (file_stat.st_mode & S_IFMT) == S_IFDIR; +} + int main(int argc, char **argv) { bool error = false; @@ -23,6 +33,7 @@ int main(int argc, char **argv) // Settings.. simplecpp::DUI dui; + dui.removeComments = true; bool quiet = false; bool error_only = false; for (int i = 1; i < argc; i++) { @@ -114,18 +125,18 @@ int main(int argc, char **argv) } } else if (filename) { std::cout << "error: multiple filenames specified" << std::endl; - std::exit(1); + return 1; } else { filename = arg; } } if (error) - std::exit(1); + return 1; if (quiet && error_only) { std::cout << "error: -e cannot be used in conjunction with -q" << std::endl; - std::exit(1); + return 1; } if (!filename) { @@ -141,32 +152,54 @@ int main(int argc, char **argv) std::cout << " -e Output errors only." << std::endl; std::cout << " -f Fail when errors were encountered (exitcode 1)." << std::endl; std::cout << " -l Print lines numbers." << std::endl; - std::exit(0); + return 0; } - dui.removeComments = true; + // TODO: move this logic into simplecpp + bool inp_missing = false; + + for (const std::string& inc : dui.includes) { + std::ifstream f(inc); + if (!f.is_open() || isDir(inc)) { + inp_missing = true; + std::cout << "error: could not open include '" << inc << "'" << std::endl; + } + } + + for (const std::string& inc : dui.includePaths) { + if (!isDir(inc)) { + inp_missing = true; + std::cout << "error: could not find include path '" << inc << "'" << std::endl; + } + } + + std::ifstream f(filename); + if (!f.is_open() || isDir(filename)) { + inp_missing = true; + std::cout << "error: could not open file '" << filename << "'" << std::endl; + } + + if (inp_missing) + return 1; // Perform preprocessing simplecpp::OutputList outputList; std::vector files; - simplecpp::TokenList *rawtokens; - if (use_istream) { - std::ifstream f(filename); - if (!f.is_open()) { - std::cout << "error: could not open file '" << filename << "'" << std::endl; - std::exit(1); + simplecpp::TokenList outputTokens(files); + { + simplecpp::TokenList *rawtokens; + if (use_istream) { + rawtokens = new simplecpp::TokenList(f, files,filename,&outputList); + } else { + f.close(); + rawtokens = new simplecpp::TokenList(filename,files,&outputList); } - rawtokens = new simplecpp::TokenList(f, files,filename,&outputList); - } else { - rawtokens = new simplecpp::TokenList(filename,files,&outputList); + rawtokens->removeComments(); + simplecpp::FileDataCache filedata; + simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList); + simplecpp::cleanup(filedata); + delete rawtokens; } - rawtokens->removeComments(); - simplecpp::TokenList outputTokens(files); - simplecpp::FileDataCache filedata; - simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList); - simplecpp::cleanup(filedata); - delete rawtokens; - rawtokens = nullptr; // Output if (!quiet) { From 009ceca046d6cda5c75bce64a15d396e0312f785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 1 Oct 2025 17:28:43 +0200 Subject: [PATCH 321/381] enabled and fixed `modernize-use-override` clang-tidy warnings (#552) --- .clang-tidy | 1 - simplecpp.cpp | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 806a8608..c38c46f7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -36,7 +36,6 @@ Checks: > -modernize-use-equals-delete, -modernize-use-default-member-init, -modernize-use-nodiscard, - -modernize-use-override, -modernize-use-trailing-return-type, -modernize-use-using, -readability-avoid-nested-conditional-operator, diff --git a/simplecpp.cpp b/simplecpp.cpp index c5760145..2cf1c6c0 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -358,16 +358,16 @@ class StdIStream : public simplecpp::TokenList::Stream { init(); } - virtual int get() override { + int get() override { return istr.get(); } - virtual int peek() override { + int peek() override { return istr.peek(); } - virtual void unget() override { + void unget() override { istr.unget(); } - virtual bool good() override { + bool good() override { return istr.good(); } @@ -386,20 +386,20 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { init(); } - virtual int get() override { + int get() override { if (pos >= size) return lastStatus = EOF; return str[pos++]; } - virtual int peek() override { + int peek() override { if (pos >= size) return lastStatus = EOF; return str[pos]; } - virtual void unget() override { + void unget() override { --pos; } - virtual bool good() override { + bool good() override { return lastStatus != EOF; } @@ -429,20 +429,20 @@ class FileStream : public simplecpp::TokenList::Stream { file = nullptr; } - virtual int get() override { + int get() override { lastStatus = lastCh = fgetc(file); return lastCh; } - virtual int peek() override { + int peek() override { // keep lastCh intact const int ch = fgetc(file); unget_internal(ch); return ch; } - virtual void unget() override { + void unget() override { unget_internal(lastCh); } - virtual bool good() override { + bool good() override { return lastStatus != EOF; } From 5b736f252fe983c7ac7d122dccdf0aeb3b00ec64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 1 Oct 2025 17:37:28 +0200 Subject: [PATCH 322/381] improved and exposed `isAbsolutePath()` (#538) --- simplecpp.cpp | 29 +++++++++++++++++------------ simplecpp.h | 3 +++ test.cpp | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 2cf1c6c0..0029aac0 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2438,21 +2438,26 @@ namespace simplecpp { return windowsPath; } #endif -} + bool isAbsolutePath(const std::string &path) + { #ifdef SIMPLECPP_WINDOWS -static bool isAbsolutePath(const std::string &path) -{ - if (path.length() >= 3 && path[0] > 0 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) - return true; - return path.length() > 1U && (path[0] == '/' || path[0] == '\\'); -} + // C:\\path\\file + // C:/path/file + if (path.length() >= 3 && std::isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/')) + return true; + + // \\host\path\file + // //host/path/file + if (path.length() >= 2 && (path[0] == '\\' || path[0] == '/') && (path[1] == '\\' || path[1] == '/')) + return true; + + return false; #else -static bool isAbsolutePath(const std::string &path) -{ - return path.length() > 1U && path[0] == '/'; -} + return !path.empty() && path[0] == '/'; #endif + } +} namespace simplecpp { /** @@ -3013,7 +3018,7 @@ static std::string openHeaderDirect(std::ifstream &f, const std::string &path) static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader) { - if (isAbsolutePath(header)) + if (simplecpp::isAbsolutePath(header)) return openHeaderDirect(f, simplecpp::simplifyPath(header)); // prefer first to search the header relatively to source file if found, when not a system header diff --git a/simplecpp.h b/simplecpp.h index c6c3b515..b461de47 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -556,6 +556,9 @@ namespace simplecpp { /** Returns the __cplusplus value for a given standard */ SIMPLECPP_LIB std::string getCppStdString(const std::string &std); SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); + + /** Checks if given path is absolute */ + SIMPLECPP_LIB bool isAbsolutePath(const std::string &path); } #undef SIMPLECPP_TOKENLIST_ALLOW_PTR diff --git a/test.cpp b/test.cpp index f42b2189..d42d6570 100644 --- a/test.cpp +++ b/test.cpp @@ -45,7 +45,7 @@ static int assertEquals(const std::string &expected, const std::string &actual, if (expected != actual) { numberOfFailedAssertions++; std::cerr << "------ assertion failed ---------" << std::endl; - std::cerr << "line " << line << std::endl; + std::cerr << "line test.cpp:" << line << std::endl; std::cerr << "expected:" << pprint(expected) << std::endl; std::cerr << "actual:" << pprint(actual) << std::endl; } @@ -3246,6 +3246,39 @@ static void safe_api() #endif } +static void isAbsolutePath() { +#ifdef _WIN32 + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("C:\\foo\\bar")); + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("C:/foo/bar")); + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("\\\\foo\\bar")); + + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("foo\\bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("foo/bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("foo.cpp")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("C:foo.cpp")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("C:foo\\bar.cpp")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("bar.cpp")); + //ASSERT_EQUALS(true, simplecpp::isAbsolutePath("\\")); // TODO + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("0:\\foo\\bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("0:/foo/bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("\\foo\\bar")); + //ASSERT_EQUALS(false, simplecpp::isAbsolutePath("\\\\")); // TODO + //ASSERT_EQUALS(false, simplecpp::isAbsolutePath("//")); // TODO + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("/foo/bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("/")); +#else + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("/foo/bar")); + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("/")); + ASSERT_EQUALS(true, simplecpp::isAbsolutePath("//host/foo/bar")); + + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("foo/bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("foo.cpp")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("C:\\foo\\bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("C:/foo/bar")); + ASSERT_EQUALS(false, simplecpp::isAbsolutePath("\\\\foo\\bar")); +#endif +} + // crashes detected by fuzzer static void fuzz_crash() { @@ -3531,6 +3564,8 @@ int main(int argc, char **argv) TEST_CASE(safe_api); + TEST_CASE(isAbsolutePath); + TEST_CASE(fuzz_crash); TEST_CASE(leak); From a5fdea987afa4cf7cc30e9abd37b5b93c6bd6eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 2 Oct 2025 12:11:08 +0200 Subject: [PATCH 323/381] selfcheck.sh: also run with system includes made available (#438) --- .github/workflows/CI-unixish.yml | 4 +- Makefile | 2 +- selfcheck.sh | 106 ++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 07ee9731..9778f68a 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -44,10 +44,10 @@ jobs: sudo apt-get install valgrind - name: Install missing software on ubuntu (clang++) - if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' + if: contains(matrix.os, 'ubuntu') && matrix.compiler == 'clang++' run: | sudo apt-get update - sudo apt-get install libc++-18-dev + sudo apt-get install libc++-dev # coreutils contains "nproc" - name: Install missing software on macos diff --git a/Makefile b/Makefile index 7489ec83..b6bc26da 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ test: testrunner simplecpp python3 -m pytest integration_test.py -vv selfcheck: simplecpp - ./selfcheck.sh + CXX=$(CXX) ./selfcheck.sh simplecpp: main.o simplecpp.o $(CXX) $(LDFLAGS) main.o simplecpp.o -o simplecpp diff --git a/selfcheck.sh b/selfcheck.sh index a43ef8c5..cc77981d 100755 --- a/selfcheck.sh +++ b/selfcheck.sh @@ -1,11 +1,111 @@ -#!/bin/sh +#!/bin/bash output=$(./simplecpp simplecpp.cpp -e -f 2>&1) ec=$? errors=$(echo "$output" | grep -v 'Header not found: <') if [ $ec -ne 0 ]; then - # only fail if got errors which do not refer to missing system includes + # only fail if we got errors which do not refer to missing system includes if [ ! -z "$errors" ]; then exit $ec fi -fi \ No newline at end of file +fi + +if [ -z "$CXX" ]; then + exit 0 +fi + +cxx_type=$($CXX --version | head -1 | cut -d' ' -f1) +if [ "$cxx_type" = "Ubuntu" ] || [ "$cxx_type" = "Debian" ]; then + cxx_type=$($CXX --version | head -1 | cut -d' ' -f2) +fi + +# TODO: generate defines from compiler +if [ "$cxx_type" = "g++" ]; then + defs= + defs="$defs -D__GNUC__" + defs="$defs -D__STDC__" + defs="$defs -D__x86_64__" + defs="$defs -D__STDC_HOSTED__" + defs="$defs -D__CHAR_BIT__=8" + defs="$defs -D__has_builtin(x)=(1)" + defs="$defs -D__has_cpp_attribute(x)=(1)" + defs="$defs -D__has_attribute(x)=(1)" + + inc= + while read line + do + inc="$inc -I$line" + done <<< "$($CXX -x c++ -v -c -S - 2>&1 < /dev/null | grep -e'^ [/A-Z]' | grep -v /cc1plus)" +elif [ "$cxx_type" = "clang" ]; then + # libstdc++ + defs= + defs="$defs -D__x86_64__" + defs="$defs -D__STDC_HOSTED__" + defs="$defs -D__CHAR_BIT__=8" + defs="$defs -D__has_builtin(x)=(1)" + defs="$defs -D__has_cpp_attribute(x)=(1)" + defs="$defs -D__has_feature(x)=(1)" + defs="$defs -D__has_include_next(x)=(0)" + defs="$defs -D__has_attribute(x)=(0)" + defs="$defs -D__building_module(x)=(0)" + + inc= + while read line + do + inc="$inc -I$line" + done <<< "$($CXX -x c++ -v -c -S - 2>&1 < /dev/null | grep -e'^ [/A-Z]')" + + # TODO: enable + # libc++ + #defs= + #defs="$defs -D__x86_64__" + #defs="$defs -D__linux__" + #defs="$defs -D__SIZEOF_SIZE_T__=8" + #defs="$defs -D__has_include_next(x)=(0)" + #defs="$defs -D__has_builtin(x)=(1)" + #defs="$defs -D__has_feature(x)=(1)" + + #inc= + #while read line + #do + # inc="$inc -I$line" + #done <<< "$($CXX -x c++ -stdlib=libc++ -v -c -S - 2>&1 < /dev/null | grep -e'^ [/A-Z]')" +elif [ "$cxx_type" = "Apple" ]; then + defs= + defs="$defs -D__BYTE_ORDER__" + defs="$defs -D__APPLE__" + defs="$defs -D__GNUC__=15" + defs="$defs -D__x86_64__" + defs="$defs -D__SIZEOF_SIZE_T__=8" + defs="$defs -D__LITTLE_ENDIAN__" + defs="$defs -D__has_feature(x)=(0)" + defs="$defs -D__has_extension(x)=(1)" + defs="$defs -D__has_attribute(x)=(0)" + defs="$defs -D__has_cpp_attribute(x)=(0)" + defs="$defs -D__has_include_next(x)=(0)" + defs="$defs -D__has_builtin(x)=(1)" + defs="$defs -D__is_target_os(x)=(0)" + defs="$defs -D__is_target_arch(x)=(0)" + defs="$defs -D__is_target_vendor(x)=(0)" + defs="$defs -D__is_target_environment(x)=(0)" + defs="$defs -D__is_target_variant_os(x)=(0)" + defs="$defs -D__is_target_variant_environment(x)=(0)" + + inc= + while read line + do + inc="$inc -I$line" + # TODO: pass the framework path as such when possible + done <<< "$($CXX -x c++ -v -c -S - 2>&1 < /dev/null | grep -e'^ [/A-Z]' | sed 's/ (framework directory)//g')" + echo $inc +else + echo "unknown compiler '$cxx_type'" + exit 1 +fi + +# run with -std=gnuc++* so __has_include(...) is available +./simplecpp simplecpp.cpp -e -f -std=gnu++11 $defs $inc +ec=$? +if [ $ec -ne 0 ]; then + exit $ec +fi From 0334a803e4cc2d6fa938c4fa6e58b2d1dde3ef4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 2 Oct 2025 12:11:21 +0200 Subject: [PATCH 324/381] enabled and fixed `modernize-use-using` clang-tidy warnings (#557) --- .clang-tidy | 1 - simplecpp.h | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index c38c46f7..5bbc4850 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -37,7 +37,6 @@ Checks: > -modernize-use-default-member-init, -modernize-use-nodiscard, -modernize-use-trailing-return-type, - -modernize-use-using, -readability-avoid-nested-conditional-operator, -readability-braces-around-statements, -readability-function-cognitive-complexity, diff --git a/simplecpp.h b/simplecpp.h index b461de47..78609af3 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -68,7 +68,7 @@ namespace simplecpp { /** C++ code standard */ enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; - typedef std::string TokenString; + using TokenString = std::string; class Macro; class FileDataCache; @@ -221,7 +221,7 @@ namespace simplecpp { std::string msg; }; - typedef std::list OutputList; + using OutputList = std::list; /** List of tokens. */ class SIMPLECPP_LIB TokenList { @@ -439,10 +439,10 @@ namespace simplecpp { mData.clear(); } - typedef std::vector> container_type; - typedef container_type::iterator iterator; - typedef container_type::const_iterator const_iterator; - typedef container_type::size_type size_type; + using container_type = std::vector>; + using iterator = container_type::iterator; + using const_iterator = container_type::const_iterator; + using size_type = container_type::size_type; size_type size() const { return mData.size(); From b070df6f4ca9f600779b45b056fab3e3ac2c82c2 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:47:34 +0200 Subject: [PATCH 325/381] Fix #470 Crash in simplecpp::Macro::expand() (#555) --- simplecpp.cpp | 8 +++++++- test.cpp | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 0029aac0..ee58f430 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2019,7 +2019,13 @@ namespace simplecpp { tok = tok->next; if (tok == endToken2) { - output.push_back(new Token(*tok->previous)); + if (tok) { + output.push_back(new Token(*tok->previous)); + } + else { + output.push_back(new Token(*nameTokInst)); + output.back()->setstr("\"\""); + } break; } if (tok->op == '#') { diff --git a/test.cpp b/test.cpp index d42d6570..d5e5d1f8 100644 --- a/test.cpp +++ b/test.cpp @@ -1041,6 +1041,16 @@ static void define_va_opt_7() toString(outputList)); } +static void define_va_opt_8() +{ + const char code[] = "#define f(...) #__VA_OPT__(x)\n" + "const char* v1 = f();"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("\nconst char * v1 = \"\" ;", preprocess(code, &outputList)); + ASSERT_EQUALS("", toString(outputList)); +} + static void define_ifdef() { const char code[] = "#define A(X) X\n" @@ -3383,6 +3393,7 @@ int main(int argc, char **argv) TEST_CASE(define_va_opt_5); TEST_CASE(define_va_opt_6); TEST_CASE(define_va_opt_7); + TEST_CASE(define_va_opt_8); TEST_CASE(pragma_backslash); // multiline pragma directive From d1db554870f4f5ef320622801e0f693628aef417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 3 Oct 2025 11:06:04 +0200 Subject: [PATCH 326/381] enabled and fixed `modernize-use-emplace` clang-tidy warnings (#558) --- .clang-tidy | 1 - main.cpp | 6 +++--- simplecpp.cpp | 4 ++-- test.cpp | 38 +++++++++++++++++++------------------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 5bbc4850..b88a3d36 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -31,7 +31,6 @@ Checks: > -modernize-pass-by-value, -modernize-return-braced-init-list, -modernize-use-auto, - -modernize-use-emplace, -modernize-use-equals-default, -modernize-use-equals-delete, -modernize-use-default-member-init, diff --git a/main.cpp b/main.cpp index 0196d6e4..fe8ea292 100644 --- a/main.cpp +++ b/main.cpp @@ -50,7 +50,7 @@ int main(int argc, char **argv) error = true; break; } - dui.defines.push_back(value); + dui.defines.emplace_back(value); break; } case 'U': { // undefine symbol @@ -72,7 +72,7 @@ int main(int argc, char **argv) error = true; break; } - dui.includePaths.push_back(value); + dui.includePaths.emplace_back(value); break; } case 'i': @@ -84,7 +84,7 @@ int main(int argc, char **argv) error = true; break; } - dui.includes.push_back(std::move(value)); + dui.includes.emplace_back(std::move(value)); } else if (std::strncmp(arg, "-is",3)==0) { found = true; use_istream = true; diff --git a/simplecpp.cpp b/simplecpp.cpp index ee58f430..d37c0aa7 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1734,7 +1734,7 @@ namespace simplecpp { argtok->next && argtok->next->op == ')') { variadic = true; if (!argtok->previous->name) - args.push_back("__VA_ARGS__"); + args.emplace_back("__VA_ARGS__"); argtok = argtok->next; // goto ')' break; } @@ -3653,7 +3653,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL E += (E.empty() ? "" : " ") + tok->str(); const long long result = evaluate(expr, dui, sizeOfType); conditionIsTrue = (result != 0); - ifCond->push_back(IfCond(rawtok->location, E, result)); + ifCond->emplace_back(rawtok->location, E, result); } else { const long long result = evaluate(expr, dui, sizeOfType); conditionIsTrue = (result != 0); diff --git a/test.cpp b/test.cpp index d5e5d1f8..26e3b95b 100644 --- a/test.cpp +++ b/test.cpp @@ -1727,7 +1727,7 @@ static void strict_ansi_4() "#endif"; simplecpp::DUI dui; dui.std = "gnu99"; - dui.defines.push_back("__STRICT_ANSI__"); + dui.defines.emplace_back("__STRICT_ANSI__"); ASSERT_EQUALS("\nA", preprocess(code, dui)); } @@ -1774,7 +1774,7 @@ static void ifA() ASSERT_EQUALS("", preprocess(code)); simplecpp::DUI dui; - dui.defines.push_back("A=1"); + dui.defines.emplace_back("A=1"); ASSERT_EQUALS("\nX", preprocess(code, dui)); } @@ -1793,7 +1793,7 @@ static void ifDefined() "#endif"; simplecpp::DUI dui; ASSERT_EQUALS("", preprocess(code, dui)); - dui.defines.push_back("A=1"); + dui.defines.emplace_back("A=1"); ASSERT_EQUALS("\nX", preprocess(code, dui)); } @@ -1804,7 +1804,7 @@ static void ifDefinedNoPar() "#endif"; simplecpp::DUI dui; ASSERT_EQUALS("", preprocess(code, dui)); - dui.defines.push_back("A=1"); + dui.defines.emplace_back("A=1"); ASSERT_EQUALS("\nX", preprocess(code, dui)); } @@ -1816,7 +1816,7 @@ static void ifDefinedNested() "#endif"; simplecpp::DUI dui; ASSERT_EQUALS("", preprocess(code, dui)); - dui.defines.push_back("FOO=1"); + dui.defines.emplace_back("FOO=1"); ASSERT_EQUALS("\n\nX", preprocess(code, dui)); } @@ -1828,7 +1828,7 @@ static void ifDefinedNestedNoPar() "#endif"; simplecpp::DUI dui; ASSERT_EQUALS("", preprocess(code, dui)); - dui.defines.push_back("FOO=1"); + dui.defines.emplace_back("FOO=1"); ASSERT_EQUALS("\n\nX", preprocess(code, dui)); } @@ -1881,10 +1881,10 @@ static void ifLogical() simplecpp::DUI dui; ASSERT_EQUALS("", preprocess(code, dui)); dui.defines.clear(); - dui.defines.push_back("A=1"); + dui.defines.emplace_back("A=1"); ASSERT_EQUALS("\nX", preprocess(code, dui)); dui.defines.clear(); - dui.defines.push_back("B=1"); + dui.defines.emplace_back("B=1"); ASSERT_EQUALS("\nX", preprocess(code, dui)); } @@ -2079,7 +2079,7 @@ static void missingHeader2() simplecpp::TokenList tokens2(files); const simplecpp::TokenList rawtokens = makeTokenList(code,files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); } @@ -2111,7 +2111,7 @@ static void nestedInclude() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("file0,1,include_nested_too_deeply,#include nested too deeply\n", toString(outputList)); @@ -2129,7 +2129,7 @@ static void systemInclude() simplecpp::OutputList outputList; simplecpp::TokenList tokens2(files); simplecpp::DUI dui; - dui.includePaths.push_back("include"); + dui.includePaths.emplace_back("include"); simplecpp::preprocess(tokens2, rawtokens, files, cache, dui, &outputList); ASSERT_EQUALS("", toString(outputList)); @@ -2355,7 +2355,7 @@ static void include3() // #16 - crash when expanding macro from header simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n1234", out.stringify()); @@ -2382,8 +2382,8 @@ static void include4() // #27 - -include simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); - dui.includes.push_back("27.h"); + dui.includePaths.emplace_back("."); + dui.includes.emplace_back("27.h"); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("123", out.stringify()); @@ -2409,7 +2409,7 @@ static void include5() // #3 - handle #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); @@ -2455,7 +2455,7 @@ static void include7() // #include MACRO simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"3.h\"\n123", out.stringify()); @@ -2493,7 +2493,7 @@ static void include9() simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify()); @@ -2675,7 +2675,7 @@ static void stringify1() simplecpp::TokenList out(files); simplecpp::DUI dui; - dui.includePaths.push_back("."); + dui.includePaths.emplace_back("."); simplecpp::preprocess(out, rawtokens_c, files, cache, dui); ASSERT_EQUALS("\n#line 1 \"A.h\"\n1\n2\n#line 1 \"A.h\"\n1\n2", out.stringify()); @@ -2778,7 +2778,7 @@ static void userdef() { const char code[] = "#ifdef A\n123\n#endif\n"; simplecpp::DUI dui; - dui.defines.push_back("A=1"); + dui.defines.emplace_back("A=1"); ASSERT_EQUALS("\n123", preprocess(code, dui)); } From 41389249d6bb2a39d3e06de469a97aef7cdb9961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 Oct 2025 17:09:01 +0200 Subject: [PATCH 327/381] enabled and fixed `performance-enum-size` clang-tidy warnings (#559) --- .clang-tidy | 1 - simplecpp.cpp | 3 ++- simplecpp.h | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index b88a3d36..725db88e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -49,7 +49,6 @@ Checks: > -readability-use-concise-preprocessor-directives, -readability-uppercase-literal-suffix, -performance-avoid-endl, - -performance-enum-size, -performance-inefficient-string-concatenation, -performance-no-automatic-move, -performance-noexcept-move-constructor diff --git a/simplecpp.cpp b/simplecpp.cpp index d37c0aa7..ea33fd14 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -24,6 +24,7 @@ #include #include #include // IWYU pragma: keep +#include #include #include #include @@ -3366,7 +3367,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL // True => code in current #if block should be kept // ElseIsTrue => code in current #if block should be dropped. the code in the #else should be kept. // AlwaysFalse => drop all code in #if and #else - enum IfState { True, ElseIsTrue, AlwaysFalse }; + enum IfState : std::uint8_t { True, ElseIsTrue, AlwaysFalse }; std::stack ifstates; std::stack iftokens; ifstates.push(True); diff --git a/simplecpp.h b/simplecpp.h index 78609af3..da859cba 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -7,6 +7,7 @@ #define simplecppH #include +#include #include #include #include @@ -39,9 +40,7 @@ # define SIMPLECPP_LIB #endif -#ifdef _WIN32 -# include -#else +#ifndef _WIN32 # include #endif @@ -63,10 +62,10 @@ namespace simplecpp { /** C code standard */ - enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23, C2Y }; + enum cstd_t : std::int8_t { CUnknown=-1, C89, C99, C11, C17, C23, C2Y }; /** C++ code standard */ - enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; + enum cppstd_t : std::int8_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; using TokenString = std::string; class Macro; @@ -204,7 +203,7 @@ namespace simplecpp { /** Output from preprocessor */ struct SIMPLECPP_LIB Output { explicit Output(const std::vector &files) : type(ERROR), location(files) {} - enum Type { + enum Type : std::uint8_t { ERROR, /* #error */ WARNING, /* #warning */ MISSING_HEADER, From fba490989318d105ecf3a094943331499a89af74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 Oct 2025 17:46:14 +0200 Subject: [PATCH 328/381] fixed #562 - avoid potential redefinition of Windows-specific macros (#563) --- simplecpp.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ea33fd14..9f7de67d 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -7,8 +7,12 @@ # ifndef _WIN32_WINNT # define _WIN32_WINNT 0x0602 # endif -# define NOMINMAX -# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # undef ERROR #endif From 31164c2eb28cc117aad8dbc6e24afd0433884d92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 7 Oct 2025 19:24:22 +0200 Subject: [PATCH 329/381] selfcheck.sh: added options `VALGRIND_TOOL` and `SIMPLECPP_PATH` / CI-unixish.yml: use `selfcheck.sh` to run valgrind (#560) --- .github/workflows/CI-unixish.yml | 14 +++++++++----- selfcheck.sh | 26 ++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 9778f68a..b4089ee1 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -100,9 +100,10 @@ jobs: if: matrix.os == 'ubuntu-24.04' run: | make clean - make -j$(nproc) + make -j$(nproc) CXXOPTS="-O1" valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner - valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./simplecpp simplecpp.cpp -e + # TODO: run Python tests with valgrind + VALGRIND_TOOL=memcheck ./selfcheck.sh - name: Run with libstdc++ debug mode if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++' @@ -146,10 +147,13 @@ jobs: tar xvf 1.5.1.tar.gz make clean make -j$(nproc) CXXOPTS="-O2 -g3" - valgrind --tool=callgrind ./simplecpp -e simplecpp-1.5.1/simplecpp.cpp 2>callgrind.log || (cat callgrind.log && false) + VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind.log || (cat callgrind.log && false) cat callgrind.log - callgrind_annotate --auto=no > callgrind.annotated.log - head -50 callgrind.annotated.log + for f in callgrind.out.*; + do + callgrind_annotate --auto=no $f > $f.annotated.log + head -50 $f.annotated.log + done - uses: actions/upload-artifact@v4 if: matrix.os == 'ubuntu-24.04' diff --git a/selfcheck.sh b/selfcheck.sh index cc77981d..e708023a 100755 --- a/selfcheck.sh +++ b/selfcheck.sh @@ -1,7 +1,28 @@ #!/bin/bash -output=$(./simplecpp simplecpp.cpp -e -f 2>&1) +if [ -z "$SIMPLECPP_PATH" ]; then + SIMPLECPP_PATH=. +fi + +if [ -n "$VALGRIND_TOOL" ]; then + if [ "$VALGRIND_TOOL" = "memcheck" ]; then + VALGRIND_OPTS="--error-limit=yes --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42" + elif [ "$VALGRIND_TOOL" = "callgrind" ]; then + VALGRIND_OPTS="--tool=callgrind" + else + echo "unsupported valgrind tool '$VALGRIND_TOOL'" + exit 1 + fi + VALGRIND_CMD="valgrind --tool=$VALGRIND_TOOL --log-fd=9 $VALGRIND_OPTS" + VALGRIND_REDIRECT="valgrind_$VALGRIND_TOOL.log" +else + VALGRIND_CMD= + VALGRIND_REDIRECT="/dev/null" +fi + +output=$($VALGRIND_CMD ./simplecpp "$SIMPLECPP_PATH/simplecpp.cpp" -e -f 2>&1 9> "$VALGRIND_REDIRECT") ec=$? +cat "$VALGRIND_REDIRECT" errors=$(echo "$output" | grep -v 'Header not found: <') if [ $ec -ne 0 ]; then # only fail if we got errors which do not refer to missing system includes @@ -104,8 +125,9 @@ else fi # run with -std=gnuc++* so __has_include(...) is available -./simplecpp simplecpp.cpp -e -f -std=gnu++11 $defs $inc +$VALGRIND_CMD ./simplecpp "$SIMPLECPP_PATH/simplecpp.cpp" -e -f -std=gnu++11 $defs $inc 9> "$VALGRIND_REDIRECT" ec=$? +cat "$VALGRIND_REDIRECT" if [ $ec -ne 0 ]; then exit $ec fi From ccde80d6982d39f1f4ae6968828750f175b8912d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 8 Oct 2025 20:28:46 +0200 Subject: [PATCH 330/381] enabled and fixed `modernize-use-equals-default` clang-tidy warnings (#561) --- .clang-tidy | 1 - simplecpp.cpp | 2 +- simplecpp.h | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 725db88e..c93066ab 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -31,7 +31,6 @@ Checks: > -modernize-pass-by-value, -modernize-return-braced-init-list, -modernize-use-auto, - -modernize-use-equals-default, -modernize-use-equals-delete, -modernize-use-default-member-init, -modernize-use-nodiscard, diff --git a/simplecpp.cpp b/simplecpp.cpp index 9f7de67d..a8e197b6 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -245,7 +245,7 @@ void simplecpp::Token::printOut() const // cppcheck-suppress noConstructor - we call init() in the inherited to initialize the private members class simplecpp::TokenList::Stream { public: - virtual ~Stream() {} + virtual ~Stream() = default; virtual int get() = 0; virtual int peek() = 0; diff --git a/simplecpp.h b/simplecpp.h index da859cba..a23e4cb4 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -78,7 +78,7 @@ namespace simplecpp { public: explicit Location(const std::vector &f) : files(f), fileIndex(0), line(1U), col(0U) {} - Location(const Location &loc) : files(loc.files), fileIndex(loc.fileIndex), line(loc.line), col(loc.col) {} + Location(const Location &loc) = default; Location &operator=(const Location &other) { if (this != &other) { From a74ed72db655ea71be0177dd9944fb6d3fc02e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 9 Oct 2025 18:04:13 +0200 Subject: [PATCH 331/381] enabled and fixed `modernize-use-default-member-init` clang-tidy warnings (#564) --- .clang-tidy | 1 - simplecpp.cpp | 16 +++++++--------- simplecpp.h | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index c93066ab..b7c39f2b 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -32,7 +32,6 @@ Checks: > -modernize-return-braced-init-list, -modernize-use-auto, -modernize-use-equals-delete, - -modernize-use-default-member-init, -modernize-use-nodiscard, -modernize-use-trailing-return-type, -readability-avoid-nested-conditional-operator, diff --git a/simplecpp.cpp b/simplecpp.cpp index a8e197b6..d5b51080 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -386,8 +386,7 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { StdCharBufStream(const unsigned char* str, std::size_t size) : str(str) , size(size) - , pos(0) - , lastStatus(0) { + { init(); } @@ -411,8 +410,8 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { private: const unsigned char *str; const std::size_t size; - std::size_t pos; - int lastStatus; + std::size_t pos{}; + int lastStatus{}; }; class FileStream : public simplecpp::TokenList::Stream { @@ -420,8 +419,7 @@ class FileStream : public simplecpp::TokenList::Stream { // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members explicit FileStream(const std::string &filename, std::vector &files) : file(fopen(filename.c_str(), "rb")) - , lastCh(0) - , lastStatus(0) { + { if (!file) { files.push_back(filename); throw simplecpp::Output(files, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); @@ -465,8 +463,8 @@ class FileStream : public simplecpp::TokenList::Stream { FileStream &operator=(const FileStream&); FILE *file; - int lastCh; - int lastStatus; + int lastCh{}; + int lastStatus{}; }; simplecpp::TokenList::TokenList(std::vector &filenames) : frontToken(nullptr), backToken(nullptr), files(filenames) {} @@ -1487,7 +1485,7 @@ namespace simplecpp { class Macro { public: - explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), optExpandValue(nullptr), optNoExpandValue(nullptr), valueDefinedInCode_(false) {} + explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), valueDefinedInCode_(false) {} Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previousSkipComments(), tok)) diff --git a/simplecpp.h b/simplecpp.h index a23e4cb4..cbea068b 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -76,7 +76,7 @@ namespace simplecpp { */ class SIMPLECPP_LIB Location { public: - explicit Location(const std::vector &f) : files(f), fileIndex(0), line(1U), col(0U) {} + explicit Location(const std::vector &f) : files(f) {} Location(const Location &loc) = default; @@ -109,9 +109,9 @@ namespace simplecpp { } const std::vector &files; - unsigned int fileIndex; - unsigned int line; - unsigned int col; + unsigned int fileIndex{}; + unsigned int line{1}; + unsigned int col{}; private: static const std::string emptyFileName; }; @@ -123,12 +123,12 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc, bool wsahead = false) : - whitespaceahead(wsahead), location(loc), previous(nullptr), next(nullptr), nextcond(nullptr), string(s) { + whitespaceahead(wsahead), location(loc), string(s) { flags(); } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), previous(nullptr), next(nullptr), nextcond(nullptr), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} const TokenString& str() const { return string; @@ -153,9 +153,9 @@ namespace simplecpp { bool number; bool whitespaceahead; Location location; - Token *previous; - Token *next; - mutable const Token *nextcond; + Token *previous{}; + Token *next{}; + mutable const Token *nextcond{}; const Token *previousSkipComments() const { const Token *tok = this->previous; @@ -393,14 +393,14 @@ namespace simplecpp { * On the command line these are configured by -D, -U, -I, --include, -std */ struct SIMPLECPP_LIB DUI { - DUI() : clearIncludeCache(false), removeComments(false) {} + DUI() = default; std::list defines; std::set undefined; std::list includePaths; std::list includes; std::string std; - bool clearIncludeCache; - bool removeComments; /** remove comment tokens from included files */ + bool clearIncludeCache{}; + bool removeComments{}; /** remove comment tokens from included files */ }; struct SIMPLECPP_LIB FileData { From cb9a9a21cc1f9c1c14a5065fe98fcdf42d7b7319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 12 Oct 2025 14:21:28 +0200 Subject: [PATCH 332/381] enabled and fixed `modernize-use-equals-delete` clang-tidy warnings (#565) --- .clang-tidy | 1 - simplecpp.cpp | 6 +++--- simplecpp.h | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index b7c39f2b..182f1a55 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -31,7 +31,6 @@ Checks: > -modernize-pass-by-value, -modernize-return-braced-init-list, -modernize-use-auto, - -modernize-use-equals-delete, -modernize-use-nodiscard, -modernize-use-trailing-return-type, -readability-avoid-nested-conditional-operator, diff --git a/simplecpp.cpp b/simplecpp.cpp index d5b51080..7774fd29 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -427,6 +427,9 @@ class FileStream : public simplecpp::TokenList::Stream { init(); } + FileStream(const FileStream&) = delete; + FileStream &operator=(const FileStream&) = delete; + ~FileStream() override { fclose(file); file = nullptr; @@ -459,9 +462,6 @@ class FileStream : public simplecpp::TokenList::Stream { ungetc(ch, file); } - FileStream(const FileStream&); - FileStream &operator=(const FileStream&); - FILE *file; int lastCh{}; int lastStatus{}; diff --git a/simplecpp.h b/simplecpp.h index cbea068b..7e556657 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -130,6 +130,8 @@ namespace simplecpp { Token(const Token &tok) : macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + Token &operator=(const Token &tok) = delete; + const TokenString& str() const { return string; } @@ -195,9 +197,6 @@ namespace simplecpp { TokenString string; std::set mExpandedFrom; - - // Not implemented - prevent assignment - Token &operator=(const Token &tok); }; /** Output from preprocessor */ From 318a37b553b4ccafeb6c84db1086c4f8a0756ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 18 Oct 2025 16:18:49 +0200 Subject: [PATCH 333/381] optimized handling of some preprocessor directives (#502) --- simplecpp.cpp | 108 ++++++++++++++++++++++++-------------------------- simplecpp.h | 3 +- 2 files changed, 53 insertions(+), 58 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7774fd29..8f1879ef 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -697,33 +697,55 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (oldLastToken != cback()) { oldLastToken = cback(); - if (!isLastLinePreprocessor()) + const Token * const llTok = isLastLinePreprocessor(); + if (!llTok) continue; - const std::string lastline(lastLine()); - if (lastline == "# file %str%") { - const Token *strtok = cback(); - while (strtok->comment) - strtok = strtok->previous; - loc.push(location); - location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U)); - location.line = 1U; - } else if (lastline == "# line %num%") { - const Token *numtok = cback(); - while (numtok->comment) - numtok = numtok->previous; - lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location); - } else if (lastline == "# %num% %str%" || lastline == "# line %num% %str%") { - const Token *strtok = cback(); - while (strtok->comment) - strtok = strtok->previous; - const Token *numtok = strtok->previous; - while (numtok->comment) - numtok = numtok->previous; - lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")), - std::atol(numtok->str().c_str()), &location); + const Token * const llNextToken = llTok->next; + if (!llTok->next) + continue; + if (llNextToken->next) { + // #file "file.c" + if (llNextToken->str() == "file" && + llNextToken->next->str()[0] == '\"') + { + const Token *strtok = cback(); + while (strtok->comment) + strtok = strtok->previous; + loc.push(location); + location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U)); + location.line = 1U; + } + // #3 "file.c" + // #line 3 "file.c" + else if ((llNextToken->number && + llNextToken->next->str()[0] == '\"') || + (llNextToken->str() == "line" && + llNextToken->next->number && + llNextToken->next->next && + llNextToken->next->next->str()[0] == '\"')) + { + const Token *strtok = cback(); + while (strtok->comment) + strtok = strtok->previous; + const Token *numtok = strtok->previous; + while (numtok->comment) + numtok = numtok->previous; + lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")), + std::atol(numtok->str().c_str()), &location); + } + // #line 3 + else if (llNextToken->str() == "line" && + llNextToken->next->number) + { + const Token *numtok = cback(); + while (numtok->comment) + numtok = numtok->previous; + lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location); + } } // #endfile - else if (lastline == "# endfile" && !loc.empty()) { + else if (llNextToken->str() == "endfile" && !loc.empty()) + { location = loc.top(); loc.pop(); } @@ -740,8 +762,8 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, TokenString currentToken; if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#') { - const Token* const llTok = lastLineTok(); - if (llTok && llTok->op == '#' && llTok->next && (llTok->next->str() == "error" || llTok->next->str() == "warning")) { + const Token* const ppTok = cback()->previous; + if (ppTok->next && (ppTok->next->str() == "error" || ppTok->next->str() == "warning")) { char prev = ' '; while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) { currentToken += ch; @@ -1418,34 +1440,6 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca return ret; } -std::string simplecpp::TokenList::lastLine(int maxsize) const -{ - std::string ret; - int count = 0; - for (const Token *tok = cback(); ; tok = tok->previous) { - if (!sameline(tok, cback())) { - break; - } - if (tok->comment) - continue; - if (++count > maxsize) - return ""; - if (!ret.empty()) - ret += ' '; - // add tokens in reverse for performance reasons - if (tok->str()[0] == '\"') - ret += "%rts%"; // %str% - else if (tok->number) - ret += "%mun%"; // %num% - else { - ret += tok->str(); - std::reverse(ret.end() - tok->str().length(), ret.end()); - } - } - std::reverse(ret.begin(), ret.end()); - return ret; -} - const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const { const Token* prevTok = nullptr; @@ -1462,10 +1456,12 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const return prevTok; } -bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const +const simplecpp::Token* simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const { const Token * const prevTok = lastLineTok(maxsize); - return prevTok && prevTok->op == '#'; + if (prevTok && prevTok->op == '#') + return prevTok; + return nullptr; } unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) diff --git a/simplecpp.h b/simplecpp.h index 7e556657..e164fa90 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -359,9 +359,8 @@ namespace simplecpp { std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); - std::string lastLine(int maxsize=1000) const; const Token* lastLineTok(int maxsize=1000) const; - bool isLastLinePreprocessor(int maxsize=1000) const; + const Token* isLastLinePreprocessor(int maxsize=1000) const; unsigned int fileIndex(const std::string &filename); From a766d223ca5a9e38cbc4844ef94813535663ca15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 18 Oct 2025 16:19:04 +0200 Subject: [PATCH 334/381] enabled and cleaned up some compiler warning flags (#551) --- CMakeLists.txt | 36 ++++++++++++++++++++---------------- Makefile | 3 ++- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index efdc0d96..f13fb3fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,49 +19,50 @@ function(add_compile_options_safe FLAG) endif() endfunction() -if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-pedantic) + add_compile_options(-Wall) add_compile_options(-Wextra) add_compile_options(-Wcast-qual) # Cast for removing type qualifiers add_compile_options(-Wfloat-equal) # Floating values used in equality comparisons add_compile_options(-Wmissing-declarations) # If a global function is defined without a previous declaration add_compile_options(-Wmissing-format-attribute) # - add_compile_options(-Wno-long-long) add_compile_options(-Wpacked) # add_compile_options(-Wredundant-decls) # if anything is declared more than once in the same scope add_compile_options(-Wundef) - add_compile_options(-Wno-missing-braces) - add_compile_options(-Wno-sign-compare) - add_compile_options(-Wno-multichar) add_compile_options(-Woverloaded-virtual) # when a function declaration hides virtual functions from a base class add_compile_options(-Wsuggest-attribute=noreturn) add_compile_options_safe(-Wuseless-cast) -elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + + # we are not interested in these + set_source_files_properties(test.cpp PROPERTIES COMPILE_FLAGS -Wno-multichar) +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") add_compile_definitions(_CRT_SECURE_NO_WARNINGS) - # TODO: bump warning level - #add_compile_options(/W4) # Warning Level - # TODO: enable warning + + add_compile_options(/W4) # Warning Level + + add_compile_options(/wd4127) # warning C4127: conditional expression is constant + add_compile_options(/wd4244) # warning C4244: 'x': conversion from 'int' to 'char', possible loss of data add_compile_options(/wd4267) # warning C4267: '...': conversion from 'size_t' to 'unsigned int', possible loss of data + add_compile_options(/wd4706) # warning C4706: assignment within conditional expression elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Weverything) + # no need for c++98 compatibility add_compile_options(-Wno-c++98-compat-pedantic) - # these are not really fixable + + # these are not really fixable until newer standards add_compile_options(-Wno-exit-time-destructors) add_compile_options(-Wno-global-constructors) add_compile_options(-Wno-weak-vtables) add_compile_options_safe(-Wno-unsafe-buffer-usage) add_compile_options_safe(-Wno-nrvo) - # we are not interested in these - add_compile_options(-Wno-multichar) - add_compile_options(-Wno-four-char-constants) - # ignore C++11-specific warning - add_compile_options(-Wno-suggest-override) - add_compile_options(-Wno-suggest-destructor-override) + # contradicts -Wcovered-switch-default add_compile_options(-Wno-switch-default) + # TODO: fix these? add_compile_options(-Wno-padded) add_compile_options(-Wno-sign-conversion) @@ -69,6 +70,9 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-shorten-64-to-32) add_compile_options(-Wno-shadow-field-in-constructor) + # we are not interested in these + set_source_files_properties(test.cpp PROPERTIES COMPILE_FLAGS "-Wno-multichar -Wno-four-char-constants") + if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 14 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 14) # TODO: verify this regression still exists in clang-15 if (CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") diff --git a/Makefile b/Makefile index b6bc26da..b7b54597 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: testrunner simplecpp CPPFLAGS ?= -CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wredundant-decls -Wundef -Wno-multichar -Wold-style-cast -std=c++11 -g $(CXXOPTS) +CXXFLAGS = -Wall -Wextra -pedantic -Wcast-qual -Wfloat-equal -Wmissing-declarations -Wmissing-format-attribute -Wpacked -Wredundant-decls -Wundef -Woverloaded-virtual -std=c++11 -g $(CXXOPTS) LDFLAGS = -g $(LDOPTS) # Define test source dir macro for compilation (preprocessor flags) @@ -9,6 +9,7 @@ TEST_CPPFLAGS = -DSIMPLECPP_TEST_SOURCE_DIR=\"$(CURDIR)\" # Only test.o gets the define test.o: CPPFLAGS += $(TEST_CPPFLAGS) +test.o: CXXFLAGS += -Wno-multichar %.o: %.cpp simplecpp.h $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $< From 8e85885e62ff43ed51338c1c1e8a4dbb690b96a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 20 Oct 2025 17:21:01 +0200 Subject: [PATCH 335/381] refs #424 - removed filelist parameter from `simplecpp::Output` constructors / cleanups (#569) --- simplecpp.cpp | 220 +++++++++++++++++++++++++++----------------------- simplecpp.h | 3 +- 2 files changed, 122 insertions(+), 101 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 8f1879ef..d56a6ce8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -422,7 +422,7 @@ class FileStream : public simplecpp::TokenList::Stream { { if (!file) { files.push_back(filename); - throw simplecpp::Output(files, simplecpp::Output::FILE_NOT_FOUND, "File is missing: " + filename); + throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, simplecpp::Location(files), "File is missing: " + filename); } init(); } @@ -615,14 +615,15 @@ static std::string escapeString(const std::string &str) return ostr.str(); } -static void portabilityBackslash(simplecpp::OutputList *outputList, const std::vector &files, const simplecpp::Location &location) +static void portabilityBackslash(simplecpp::OutputList *outputList, const simplecpp::Location &location) { if (!outputList) return; - simplecpp::Output err(files); - err.type = simplecpp::Output::PORTABILITY_BACKSLASH; - err.location = location; - err.msg = "Combination 'backslash space newline' is not portable."; + simplecpp::Output err = { + simplecpp::Output::PORTABILITY_BACKSLASH, + location, + "Combination 'backslash space newline' is not portable." + }; outputList->push_back(std::move(err)); } @@ -670,13 +671,12 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (ch >= 0x80) { if (outputList) { - simplecpp::Output err(files); - err.type = simplecpp::Output::UNHANDLED_CHAR_ERROR; - err.location = location; - std::ostringstream s; - s << static_cast(ch); - err.msg = "The code contains unhandled character(s) (character code=" + s.str() + "). Neither unicode nor extended ascii is supported."; - outputList->push_back(err); + simplecpp::Output err = { + simplecpp::Output::UNHANDLED_CHAR_ERROR, + location, + "The code contains unhandled character(s) (character code=" + std::to_string(static_cast(ch)) + "). Neither unicode nor extended ascii is supported." + }; + outputList->push_back(std::move(err)); } clear(); return; @@ -685,7 +685,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (ch == '\n') { if (cback() && cback()->op == '\\') { if (location.col > cback()->location.col + 1U) - portabilityBackslash(outputList, files, cback()->location); + portabilityBackslash(outputList, cback()->location); ++multiline; deleteToken(back()); } else { @@ -812,7 +812,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const TokenString check_portability = currentToken + tmp; const std::string::size_type pos = check_portability.find_last_not_of(" \t"); if (pos < check_portability.size() - 1U && check_portability[pos] == '\\') - portabilityBackslash(outputList, files, location); + portabilityBackslash(outputList, location); ++multiline; tmp_ch = stream.readChar(); currentToken += '\n'; @@ -872,11 +872,12 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } if (!stream.good() || ch == '\n') { if (outputList) { - Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = location; - err.msg = "Invalid newline in raw string delimiter."; - outputList->push_back(err); + Output err = { + Output::SYNTAX_ERROR, + location, + "Invalid newline in raw string delimiter." + }; + outputList->push_back(std::move(err)); } return; } @@ -885,10 +886,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { - Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = location; - err.msg = "Raw string missing terminating delimiter."; + Output err = { + Output::SYNTAX_ERROR, + location, + "Raw string missing terminating delimiter." + }; outputList->push_back(std::move(err)); } return; @@ -1428,10 +1430,11 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca if (!stream.good() || ch != end) { clear(); if (outputList) { - Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = location; - err.msg = std::string("No pair for character (") + start + "). Can't process file. File is either invalid or unicode, which is currently not supported."; + Output err = { + Output::SYNTAX_ERROR, + location, + std::string("No pair for character (") + start + "). Can't process file. File is either invalid or unicode, which is currently not supported." + }; outputList->push_back(std::move(err)); } return ""; @@ -3160,10 +3163,11 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (filedata == nullptr) { if (outputList) { - simplecpp::Output err(filenames); - err.type = simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND; - err.location = Location(filenames); - err.msg = "Can not open include file '" + filename + "' that is explicitly included."; + simplecpp::Output err = { + simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND, + Location(filenames), + "Can not open include file '" + filename + "' that is explicitly included." + }; outputList->push_back(std::move(err)); } continue; @@ -3231,12 +3235,13 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token simplecpp::TokenList value(files); try { *tok1 = it->second.expand(value, tok, macros, files); - } catch (simplecpp::Macro::Error &err) { + } catch (const simplecpp::Macro::Error &err) { if (outputList) { - simplecpp::Output out(files); - out.type = simplecpp::Output::SYNTAX_ERROR; - out.location = err.location; - out.msg = "failed to expand \'" + tok->str() + "\', " + err.what; + simplecpp::Output out = { + simplecpp::Output::SYNTAX_ERROR, + err.location, + "failed to expand \'" + tok->str() + "\', " + err.what + }; outputList->push_back(std::move(out)); } return false; @@ -3348,10 +3353,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const cppstd_t cpp_std = simplecpp::getCppStd(dui.std); if (cpp_std == CPPUnknown) { if (outputList) { - simplecpp::Output err(files); - err.type = Output::DUI_ERROR; - err.msg = "unknown standard specified: '" + dui.std + "'"; - outputList->push_back(err); + simplecpp::Output err = { + Output::DUI_ERROR, + Location(files), + "unknown standard specified: '" + dui.std + "'" + }; + outputList->push_back(std::move(err)); } output.clear(); return; @@ -3403,11 +3410,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (ifstates.size() <= 1U && (rawtok->str() == ELIF || rawtok->str() == ELSE || rawtok->str() == ENDIF)) { if (outputList) { - simplecpp::Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = rawtok->location; - err.msg = "#" + rawtok->str() + " without #if"; - outputList->push_back(err); + simplecpp::Output err = { + Output::SYNTAX_ERROR, + rawtok->location, + "#" + rawtok->str() + " without #if" + }; + outputList->push_back(std::move(err)); } output.clear(); return; @@ -3415,15 +3423,19 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (ifstates.top() == True && (rawtok->str() == ERROR || rawtok->str() == WARNING)) { if (outputList) { - simplecpp::Output err(rawtok->location.files); - err.type = rawtok->str() == ERROR ? Output::ERROR : Output::WARNING; - err.location = rawtok->location; + std::string msg; for (const Token *tok = rawtok->next; tok && sameline(rawtok,tok); tok = tok->next) { - if (!err.msg.empty() && isNameChar(tok->str()[0])) - err.msg += ' '; - err.msg += tok->str(); + if (!msg.empty() && isNameChar(tok->str()[0])) + msg += ' '; + msg += tok->str(); } - err.msg = '#' + rawtok->str() + ' ' + err.msg; + msg = '#' + rawtok->str() + ' ' + msg; + simplecpp::Output err = { + rawtok->str() == ERROR ? Output::ERROR : Output::WARNING, + rawtok->location, + std::move(msg) + }; + outputList->push_back(std::move(err)); } if (rawtok->str() == ERROR) { @@ -3446,21 +3458,23 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } catch (const std::runtime_error &) { if (outputList) { - simplecpp::Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = rawtok->location; - err.msg = "Failed to parse #define"; - outputList->push_back(err); + simplecpp::Output err = { + Output::SYNTAX_ERROR, + rawtok->location, + "Failed to parse #define" + }; + outputList->push_back(std::move(err)); } output.clear(); return; - } catch (simplecpp::Macro::Error &err) { + } catch (const simplecpp::Macro::Error &err) { if (outputList) { - simplecpp::Output out(files); - out.type = simplecpp::Output::SYNTAX_ERROR; - out.location = err.location; - out.msg = "Failed to parse #define, " + err.what; - outputList->push_back(out); + simplecpp::Output out = { + simplecpp::Output::SYNTAX_ERROR, + err.location, + "Failed to parse #define, " + err.what + }; + outputList->push_back(std::move(out)); } output.clear(); return; @@ -3496,11 +3510,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (inc2.empty() || inc2.cfront()->str().size() <= 2U) { if (outputList) { - simplecpp::Output err(files); - err.type = Output::SYNTAX_ERROR; - err.location = rawtok->location; - err.msg = "No header in #include"; - outputList->push_back(err); + simplecpp::Output err = { + Output::SYNTAX_ERROR, + rawtok->location, + "No header in #include" + }; + outputList->push_back(std::move(err)); } output.clear(); return; @@ -3513,19 +3528,21 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const FileData *const filedata = cache.get(rawtok->location.file(), header, dui, systemheader, files, outputList).first; if (filedata == nullptr) { if (outputList) { - simplecpp::Output out(files); - out.type = Output::MISSING_HEADER; - out.location = rawtok->location; - out.msg = "Header not found: " + inctok->str(); - outputList->push_back(out); + simplecpp::Output out = { + simplecpp::Output::MISSING_HEADER, + rawtok->location, + "Header not found: " + inctok->str() + }; + outputList->push_back(std::move(out)); } } else if (includetokenstack.size() >= 400) { if (outputList) { - simplecpp::Output out(files); - out.type = Output::INCLUDE_NESTED_TOO_DEEPLY; - out.location = rawtok->location; - out.msg = "#include nested too deeply"; - outputList->push_back(out); + simplecpp::Output out = { + simplecpp::Output::INCLUDE_NESTED_TOO_DEEPLY, + rawtok->location, + "#include nested too deeply" + }; + outputList->push_back(std::move(out)); } } else if (pragmaOnce.find(filedata->filename) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); @@ -3535,11 +3552,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { if (!sameline(rawtok,rawtok->next)) { if (outputList) { - simplecpp::Output out(files); - out.type = Output::SYNTAX_ERROR; - out.location = rawtok->location; - out.msg = "Syntax error in #" + rawtok->str(); - outputList->push_back(out); + simplecpp::Output out = { + simplecpp::Output::SYNTAX_ERROR, + rawtok->location, + "Syntax error in #" + rawtok->str() + }; + outputList->push_back(std::move(out)); } output.clear(); return; @@ -3580,10 +3598,11 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { if (outputList) { - Output out(rawtok->location.files); - out.type = Output::SYNTAX_ERROR; - out.location = rawtok->location; - out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; + Output out = { + Output::SYNTAX_ERROR, + rawtok->location, + "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" + }; outputList->push_back(std::move(out)); } output.clear(); @@ -3622,11 +3641,12 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')') || (!closingAngularBracket)) { if (outputList) { - Output out(rawtok->location.files); - out.type = Output::SYNTAX_ERROR; - out.location = rawtok->location; - out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; - outputList->push_back(out); + Output out = { + Output::SYNTAX_ERROR, + rawtok->location, + "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" + }; + outputList->push_back(std::move(out)); } output.clear(); return; @@ -3659,13 +3679,15 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } catch (const std::exception &e) { if (outputList) { - Output out(rawtok->location.files); - out.type = Output::SYNTAX_ERROR; - out.location = rawtok->location; - out.msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; + std::string msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; if (e.what() && *e.what()) - out.msg += std::string(", ") + e.what(); - outputList->push_back(out); + msg += std::string(", ") + e.what(); + Output out = { + Output::SYNTAX_ERROR, + rawtok->location, + std::move(msg) + }; + outputList->push_back(std::move(out)); } output.clear(); return; diff --git a/simplecpp.h b/simplecpp.h index e164fa90..b1b25fad 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -201,7 +201,6 @@ namespace simplecpp { /** Output from preprocessor */ struct SIMPLECPP_LIB Output { - explicit Output(const std::vector &files) : type(ERROR), location(files) {} enum Type : std::uint8_t { ERROR, /* #error */ WARNING, /* #warning */ @@ -214,7 +213,7 @@ namespace simplecpp { FILE_NOT_FOUND, DUI_ERROR } type; - explicit Output(const std::vector& files, Type type, const std::string& msg) : type(type), location(files), msg(msg) {} + Output(Type type, const Location& loc, std::string msg) : type(type), location(loc), msg(std::move(msg)) {} Location location; std::string msg; }; From bd154957a525c71ad29362f22aecc67d96682b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 28 Oct 2025 12:08:34 +0100 Subject: [PATCH 336/381] enabled and fixed `readability-simplify-boolean-expr` clang-tidy warnings (#568) --- .clang-tidy | 1 - simplecpp.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 182f1a55..8dc64681 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -42,7 +42,6 @@ Checks: > -readability-isolate-declaration, -readability-magic-numbers, -readability-redundant-inline-specifier, - -readability-simplify-boolean-expr, -readability-use-concise-preprocessor-directives, -readability-uppercase-literal-suffix, -performance-avoid-endl, diff --git a/simplecpp.cpp b/simplecpp.cpp index d56a6ce8..1e18d2ef 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -882,7 +882,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, return; } const std::string endOfRawString(')' + delim + currentToken); - while (stream.good() && !(endsWith(currentToken, endOfRawString) && currentToken.size() > 1)) + while (stream.good() && (!endsWith(currentToken, endOfRawString) || currentToken.size() <= 1)) currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { @@ -2052,7 +2052,7 @@ namespace simplecpp { } const Token *recursiveExpandToken(TokenList &output, TokenList &temp, const Location &loc, const Token *tok, const MacroMap ¯os, const std::set &expandedmacros, const std::vector ¶metertokens) const { - if (!(temp.cback() && temp.cback()->name && tok->next && tok->next->op == '(')) { + if (!temp.cback() || !temp.cback()->name || !tok->next || tok->next->op != '(') { output.takeTokens(temp); return tok->next; } From e78af461426b3cae40b1cc79a4a5e540b057b613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 28 Oct 2025 20:36:01 +0100 Subject: [PATCH 337/381] cleaned up includes based on `include-what-you-use` (#576) --- simplecpp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index b1b25fad..ee52ad43 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #if __cplusplus >= 202002L # include @@ -41,7 +42,7 @@ #endif #ifndef _WIN32 -# include +# include #endif #if defined(_MSC_VER) @@ -69,7 +70,6 @@ namespace simplecpp { using TokenString = std::string; class Macro; - class FileDataCache; /** * Location in source code From 4aa40060b86e3103825a00b95a4c383e34a148a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 28 Oct 2025 20:36:15 +0100 Subject: [PATCH 338/381] enabled and fixed `modernize-use-auto` clang-tidy warnings (#574) --- .clang-tidy | 1 - simplecpp.cpp | 24 ++++++++++++------------ simplecpp.h | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 8dc64681..b63cb3d9 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -30,7 +30,6 @@ Checks: > -modernize-loop-convert, -modernize-pass-by-value, -modernize-return-braced-init-list, - -modernize-use-auto, -modernize-use-nodiscard, -modernize-use-trailing-return-type, -readability-avoid-nested-conditional-operator, diff --git a/simplecpp.cpp b/simplecpp.cpp index 1e18d2ef..e6186be2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -253,12 +253,12 @@ class simplecpp::TokenList::Stream { virtual bool good() = 0; unsigned char readChar() { - unsigned char ch = static_cast(get()); + auto ch = static_cast(get()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the // character is non-ASCII character then replace it with 0xff if (isUtf16) { - const unsigned char ch2 = static_cast(get()); + const auto ch2 = static_cast(get()); const int ch16 = makeUtf16Char(ch, ch2); ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); } @@ -281,13 +281,13 @@ class simplecpp::TokenList::Stream { } unsigned char peekChar() { - unsigned char ch = static_cast(peek()); + auto ch = static_cast(peek()); // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the // character is non-ASCII character then replace it with 0xff if (isUtf16) { (void)get(); - const unsigned char ch2 = static_cast(peek()); + const auto ch2 = static_cast(peek()); unget(); const int ch16 = makeUtf16Char(ch, ch2); ch = static_cast(((ch16 >= 0x80) ? 0xff : ch16)); @@ -1705,7 +1705,7 @@ namespace simplecpp { private: /** Create new token where Token::macro is set for replaced tokens */ Token *newMacroToken(const TokenString &str, const Location &loc, bool replaced, const Token *expandedFromToken=nullptr) const { - Token *tok = new Token(str,loc); + auto *tok = new Token(str,loc); if (replaced) tok->macro = nameTokDef->str(); if (expandedFromToken) @@ -2365,11 +2365,11 @@ namespace simplecpp { static bool isReplaced(const std::set &expandedmacros) { // return true if size > 1 - std::set::const_iterator it = expandedmacros.begin(); - if (it == expandedmacros.end()) + auto it = expandedmacros.cbegin(); + if (it == expandedmacros.cend()) return false; ++it; - return (it != expandedmacros.end()); + return (it != expandedmacros.cend()); } /** name token in definition */ @@ -3060,7 +3060,7 @@ std::pair simplecpp::FileDataCache::tryload(FileDat return {id_it->second, false}; } - FileData *const data = new FileData {path, TokenList(path, filenames, outputList)}; + auto *const data = new FileData {path, TokenList(path, filenames, outputList)}; if (dui.removeComments) data->tokens.removeComments(); @@ -3154,7 +3154,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, std::list filelist; // -include files - for (std::list::const_iterator it = dui.includes.begin(); it != dui.includes.end(); ++it) { + for (auto it = dui.includes.cbegin(); it != dui.includes.cend(); ++it) { const std::string &filename = *it; const auto loadResult = cache.get("", filename, dui, false, filenames, outputList); @@ -3316,7 +3316,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool hasInclude = isCpp17OrLater(dui) || isGnu(dui); MacroMap macros; bool strictAnsiDefined = false; - for (std::list::const_iterator it = dui.defines.begin(); it != dui.defines.end(); ++it) { + for (auto it = dui.defines.cbegin(); it != dui.defines.cend(); ++it) { const std::string ¯ostr = *it; const std::string::size_type eq = macrostr.find('='); const std::string::size_type par = macrostr.find('('); @@ -3382,7 +3382,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::set pragmaOnce; includetokenstack.push(rawtokens.cfront()); - for (std::list::const_iterator it = dui.includes.begin(); it != dui.includes.end(); ++it) { + for (auto it = dui.includes.cbegin(); it != dui.includes.cend(); ++it) { const FileData *const filedata = cache.get("", *it, dui, false, files, outputList).first; if (filedata != nullptr && filedata->tokens.cfront() != nullptr) includetokenstack.push(filedata->tokens.cfront()); diff --git a/simplecpp.h b/simplecpp.h index ee52ad43..22f3c19f 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -423,7 +423,7 @@ namespace simplecpp { void insert(FileData data) { // NOLINTNEXTLINE(misc-const-correctness) - FP - FileData *const newdata = new FileData(std::move(data)); + auto *const newdata = new FileData(std::move(data)); mData.emplace_back(newdata); mNameMap.emplace(newdata->filename, newdata); From a8fcf9bb5eb26dfcd1a830f0ac0502d46cd42604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 28 Oct 2025 20:36:27 +0100 Subject: [PATCH 339/381] moved static array into `simplifyName()` and simplified it (#575) --- simplecpp.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index e6186be2..4e892e03 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2666,12 +2666,11 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI } } -static const char * const altopData[] = {"and","or","bitand","bitor","compl","not","not_eq","xor"}; -static const std::set altop(&altopData[0], &altopData[8]); static void simplifyName(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->name) { + static const std::set altop = {"and","or","bitand","bitor","compl","not","not_eq","xor"}; if (altop.find(tok->str()) != altop.end()) { bool alt; if (tok->str() == "not" || tok->str() == "compl") { From 6ddb8c6a37a304069527053972269eb7081bee32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 28 Oct 2025 20:37:05 +0100 Subject: [PATCH 340/381] improved testing of `#line` handling (#505) --- simplecpp.cpp | 1 + test.cpp | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 4e892e03..7d306fd7 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -715,6 +715,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U)); location.line = 1U; } + // TODO: add support for "# 3" // #3 "file.c" // #line 3 "file.c" else if ((llNextToken->number && diff --git a/test.cpp b/test.cpp index 26e3b95b..5f09c17f 100644 --- a/test.cpp +++ b/test.cpp @@ -2061,6 +2061,77 @@ static void location5() "int x ;", preprocess(code)); } +static void location6() +{ + const char code[] = + "#line 3\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "\n" + "3 \"\"", + preprocess(code)); +} + +static void location7() +{ + const char code[] = + "#line 3 \"file.c\"\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "#line 3 \"file.c\"\n" + "3 \"file.c\"", + preprocess(code)); +} + +static void location8() +{ + const char code[] = + "# 3\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "2 \"\"", // TODO: should say 3 + preprocess(code)); +} + +static void location9() +{ + const char code[] = + "# 3 \"file.c\"\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "#line 3 \"file.c\"\n" + "3 \"file.c\"", + preprocess(code)); +} + +static void location10() +{ + const char code[] = + "#line 3\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "\n" // TODO: should this have the #line marker? + "3 \"\"", + preprocess(code)); +} + +static void location11() +{ + const char code[] = + "#line 3 \"file.c\"\n" + "__LINE__ __FILE__\n" + "#line 33 \"file2.c\"\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "#line 3 \"file.c\"\n" + "3 \"file.c\"\n" + "#line 33 \"file2.c\"\n" + "33 \"file2.c\"", + preprocess(code)); +} + +// TODO: test #file/#endfile + static void missingHeader1() { const char code[] = "#include \"notexist.h\"\n"; @@ -3489,6 +3560,12 @@ int main(int argc, char **argv) TEST_CASE(location3); TEST_CASE(location4); TEST_CASE(location5); + TEST_CASE(location6); + TEST_CASE(location7); + TEST_CASE(location8); + TEST_CASE(location9); + TEST_CASE(location10); + TEST_CASE(location11); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2); From d381ec26441476e5b6618b1cdc0bb118395547a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 1 Nov 2025 14:23:29 +0100 Subject: [PATCH 341/381] CI-unixish.yml: added profile-guided optimization (PGO) to callgrind step (#580) --- .github/workflows/CI-unixish.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index b4089ee1..d6800a82 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -43,11 +43,12 @@ jobs: sudo apt-get update sudo apt-get install valgrind + # llvm contains llvm-profdata - name: Install missing software on ubuntu (clang++) if: contains(matrix.os, 'ubuntu') && matrix.compiler == 'clang++' run: | sudo apt-get update - sudo apt-get install libc++-dev + sudo apt-get install libc++-dev llvm # coreutils contains "nproc" - name: Install missing software on macos @@ -145,15 +146,34 @@ jobs: run: | wget https://github.com/danmar/simplecpp/archive/refs/tags/1.5.1.tar.gz tar xvf 1.5.1.tar.gz + rm -f 1.5.1.tar.gz + make clean - make -j$(nproc) CXXOPTS="-O2 -g3" + make -j$(nproc) CXXOPTS="-O2 -g3" simplecpp VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind.log || (cat callgrind.log && false) cat callgrind.log + + # PGO - start + make clean + make -j$(nproc) CXXOPTS="-O2 -g3 -fprofile-generate" LDOPTS="-fprofile-generate" simplecpp + SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >/dev/null + + if compgen -G "default_*.profraw" > /dev/null; then + llvm-profdata merge -output=default.profdata default_*.profraw + fi + + make clean + make -j$(nproc) CXXOPTS="-O2 -g3 -fprofile-use" LDOPTS="-fprofile-use" simplecpp + VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind_pgo.log || (cat callgrind_pgo.log && false) + cat callgrind_pgo.log + # PGO - end + for f in callgrind.out.*; do callgrind_annotate --auto=no $f > $f.annotated.log head -50 $f.annotated.log done + rm -rf simplecpp-1.5.1 - uses: actions/upload-artifact@v4 if: matrix.os == 'ubuntu-24.04' From 7a81c1ab935a28b38e21d0338654ee256577d3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Wed, 5 Nov 2025 11:49:40 +0100 Subject: [PATCH 342/381] Fix #581 (Failure when header is included twice with different macros defined) (#582) --- integration_test.py | 35 +++++++++++++++++++++++++++++++++++ simplecpp.cpp | 11 +++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/integration_test.py b/integration_test.py index e5497db3..068da776 100644 --- a/integration_test.py +++ b/integration_test.py @@ -446,3 +446,38 @@ def test_incpath_dir(record_property, tmpdir): assert '' == stderr assert "error: could not open include 'inc'\n" == stdout + + +def test_include_header_twice(tmpdir): + """ Issue #581 - Failure when header is included twice with different + macros defined""" + + header_file = tmpdir / 'test.h' + with open(header_file, 'wt') as f: + f.write(f""" + #if defined AAA + #elif defined BBB + # undef BBB + #endif + + #ifdef BBB + # error BBB is defined + #endif + """) + + test_file = os.path.join(tmpdir, 'test.c') + with open(test_file, 'wt') as f: + f.write(f""" + # define Y + # include "test.h" + + # define BBB + # include "test.h" + """) + + args = [test_file] + + _, stdout, stderr = simplecpp(args, cwd=tmpdir) + + assert stderr == '' + diff --git a/simplecpp.cpp b/simplecpp.cpp index 7d306fd7..ef891eb1 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3701,12 +3701,11 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL else ifstates.push(conditionIsTrue ? True : ElseIsTrue); iftokens.push(rawtok); - } else if (ifstates.top() == True) { - ifstates.top() = AlwaysFalse; - iftokens.top()->nextcond = rawtok; - iftokens.top() = rawtok; - } else if (ifstates.top() == ElseIsTrue && conditionIsTrue) { - ifstates.top() = True; + } else { + if (ifstates.top() == True) + ifstates.top() = AlwaysFalse; + else if (ifstates.top() == ElseIsTrue && conditionIsTrue) + ifstates.top() = True; iftokens.top()->nextcond = rawtok; iftokens.top() = rawtok; } From 5cd15b3eaf7389347b018131e8f6e2bf45590b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 7 Nov 2025 15:06:52 +0100 Subject: [PATCH 343/381] test.cpp: do not unconditionally remove comments (#506) --- test.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test.cpp b/test.cpp index 5f09c17f..69e08781 100644 --- a/test.cpp +++ b/test.cpp @@ -107,7 +107,8 @@ static std::string preprocess(const char code[], const simplecpp::DUI &dui, simp std::vector files; simplecpp::FileDataCache cache; simplecpp::TokenList tokens = makeTokenList(code,files); - tokens.removeComments(); + if (dui.removeComments) + tokens.removeComments(); simplecpp::TokenList tokens2(files); simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList); simplecpp::cleanup(cache); @@ -437,33 +438,36 @@ static void comment() static void comment_multiline() { + simplecpp::DUI dui; + dui.removeComments = true; + const char code[] = "#define ABC {// \\\n" "}\n" "void f() ABC\n"; - ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code)); + ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code, dui)); const char code1[] = "#define ABC {// \\\r\n" "}\n" "void f() ABC\n"; - ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code1)); + ASSERT_EQUALS("\n\nvoid f ( ) {", preprocess(code1, dui)); const char code2[] = "#define A 1// \\\r" "\r" "2\r" "A\r"; - ASSERT_EQUALS("\n\n2\n1", preprocess(code2)); + ASSERT_EQUALS("\n\n2\n1", preprocess(code2, dui)); const char code3[] = "void f() {// \\ \n}\n"; - ASSERT_EQUALS("void f ( ) {", preprocess(code3)); + ASSERT_EQUALS("void f ( ) {", preprocess(code3, dui)); const char code4[] = "void f() {// \\\\\\\t\t\n}\n"; - ASSERT_EQUALS("void f ( ) {", preprocess(code4)); + ASSERT_EQUALS("void f ( ) {", preprocess(code4, dui)); const char code5[] = "void f() {// \\\\\\a\n}\n"; - ASSERT_EQUALS("void f ( ) {\n}", preprocess(code5)); + ASSERT_EQUALS("void f ( ) {\n}", preprocess(code5, dui)); const char code6[] = "void f() {// \\\n\n\n}\n"; - ASSERT_EQUALS("void f ( ) {\n\n\n}", preprocess(code6)); + ASSERT_EQUALS("void f ( ) {\n\n\n}", preprocess(code6, dui)); // #471 ensure there is newline in comment so that line-splicing can be detected by tools ASSERT_EQUALS("// abc\ndef", readfile("// abc\\\ndef")); @@ -570,9 +574,12 @@ static void define6() static void define7() { + simplecpp::DUI dui; + dui.removeComments = true; + const char code[] = "#define A(X) X+1\n" "A(1 /*23*/)"; - ASSERT_EQUALS("\n1 + 1", preprocess(code)); + ASSERT_EQUALS("\n1 + 1", preprocess(code, dui)); } static void define8() // 6.10.3.10 @@ -1591,6 +1598,7 @@ static void has_include_2() " #endif\n" "#endif"; simplecpp::DUI dui; + dui.removeComments = true; // TODO: remove this dui.includePaths.push_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++14"; @@ -2167,7 +2175,9 @@ static void missingHeader4() { const char code[] = "#/**/include <>\n"; simplecpp::OutputList outputList; - ASSERT_EQUALS("", preprocess(code, &outputList)); + simplecpp::DUI dui; + dui.removeComments = true; // TODO: remove this + ASSERT_EQUALS("", preprocess(code, dui, &outputList)); ASSERT_EQUALS("file0,1,syntax_error,No header in #include\n", toString(outputList)); } From 7ac711b48903a658f15e3a28b5b23765f0e49f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 29 Nov 2025 17:56:19 +0100 Subject: [PATCH 344/381] fixed #583 - fall back to `GetFileInformationByHandle()` when `GetFileInformationByHandleEx()` fails in `simplecpp::FileDataCache::getFileId()` (#594) Co-authored-by: willenbr24 --- simplecpp.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index ef891eb1..9ce846d9 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3126,7 +3126,21 @@ bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) if (hFile == INVALID_HANDLE_VALUE) return false; - const BOOL ret = GetFileInformationByHandleEx(hFile, FileIdInfo, &id.fileIdInfo, sizeof(id.fileIdInfo)); + BOOL ret = GetFileInformationByHandleEx(hFile, FileIdInfo, &id.fileIdInfo, sizeof(id.fileIdInfo)); + if (!ret) { + const DWORD err = GetLastError(); + if (err == ERROR_INVALID_PARAMETER || // encountered when using a non-NTFS filesystem e.g. exFAT + err == ERROR_NOT_SUPPORTED) // encountered on Windows Server Core (used as a Docker container) + { + BY_HANDLE_FILE_INFORMATION fileInfo; + ret = GetFileInformationByHandle(hFile, &fileInfo); + if (ret) { + id.fileIdInfo.VolumeSerialNumber = static_cast(fileInfo.dwVolumeSerialNumber); + id.fileIdInfo.FileId.IdentifierHi = static_cast(fileInfo.nFileIndexHigh); + id.fileIdInfo.FileId.IdentifierLo = static_cast(fileInfo.nFileIndexLow); + } + } + } CloseHandle(hFile); From 72ecd546caa0f1d0737ea999d7b989eaa5e44aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 29 Nov 2025 18:47:07 +0100 Subject: [PATCH 345/381] avoid some unchecked pointer dereferences (#593) --- simplecpp.cpp | 24 ++++++++++++------------ simplecpp.h | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9ce846d9..8e10ca54 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -984,7 +984,7 @@ void simplecpp::TokenList::constFold() constFoldComparison(tok); constFoldBitwise(tok); constFoldLogicalOp(tok); - constFoldQuestionOp(&tok); + constFoldQuestionOp(tok); // If there is no '(' we are done with the constant folding if (tok->op != '(') @@ -1354,11 +1354,11 @@ void simplecpp::TokenList::constFoldLogicalOp(Token *tok) } } -void simplecpp::TokenList::constFoldQuestionOp(Token **tok1) +void simplecpp::TokenList::constFoldQuestionOp(Token *&tok1) { bool gotoTok1 = false; // NOLINTNEXTLINE(misc-const-correctness) - technically correct but used to access non-const data - for (Token *tok = *tok1; tok && tok->op != ')'; tok = gotoTok1 ? *tok1 : tok->next) { + for (Token *tok = tok1; tok && tok->op != ')'; tok = gotoTok1 ? tok1 : tok->next) { gotoTok1 = false; if (tok->str() != "?") continue; @@ -1373,8 +1373,8 @@ void simplecpp::TokenList::constFoldQuestionOp(Token **tok1) Token * const falseTok = trueTok->next->next; if (!falseTok) throw std::runtime_error("invalid expression"); - if (condTok == *tok1) - *tok1 = (condTok->str() != "0" ? trueTok : falseTok); + if (condTok == tok1) + tok1 = (condTok->str() != "0" ? trueTok : falseTok); deleteToken(condTok->next); // ? deleteToken(trueTok->next); // : deleteToken(condTok->str() == "0" ? trueTok : falseTok); @@ -3241,14 +3241,14 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, return cache; } -static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token **tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) +static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token *&tok1, simplecpp::MacroMap ¯os, std::vector &files, simplecpp::OutputList *outputList) { - const simplecpp::Token * const tok = *tok1; + const simplecpp::Token * const tok = tok1; const simplecpp::MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end(); if (it != macros.end()) { simplecpp::TokenList value(files); try { - *tok1 = it->second.expand(value, tok, macros, files); + tok1 = it->second.expand(value, tok, macros, files); } catch (const simplecpp::Macro::Error &err) { if (outputList) { simplecpp::Output out = { @@ -3264,7 +3264,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token } else { if (!tok->comment) output.push_back(new simplecpp::Token(*tok)); - *tok1 = tok->next; + tok1 = tok->next; } return true; } @@ -3502,7 +3502,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL TokenList inc2(files); if (!inc1.empty() && inc1.cfront()->name) { const Token *inctok = inc1.cfront(); - if (!preprocessToken(inc2, &inctok, macros, files, outputList)) { + if (!preprocessToken(inc2, inctok, macros, files, outputList)) { output.clear(); return; } @@ -3671,7 +3671,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); const Token *tmp = tok; - if (!preprocessToken(expr, &tmp, macros, files, outputList)) { + if (!preprocessToken(expr, tmp, macros, files, outputList)) { output.clear(); return; } @@ -3769,7 +3769,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const Location loc(rawtok->location); TokenList tokens(files); - if (!preprocessToken(tokens, &rawtok, macros, files, outputList)) { + if (!preprocessToken(tokens, rawtok, macros, files, outputList)) { output.clear(); return; } diff --git a/simplecpp.h b/simplecpp.h index 22f3c19f..15187063 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -353,7 +353,7 @@ namespace simplecpp { void constFoldComparison(Token *tok); void constFoldBitwise(Token *tok); void constFoldLogicalOp(Token *tok); - void constFoldQuestionOp(Token **tok1); + void constFoldQuestionOp(Token *&tok1); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); From d764cb250ee3310d79d6ac1ff723ce08fdf0648d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 7 Dec 2025 23:24:43 +0100 Subject: [PATCH 346/381] refs #424 / fixed #589 - do not store reference to filelist in `simplecpp::Location` / added `simplecpp::TokenList::file()` / cleanups (#573) --- integration_test.py | 22 +++++++++++++++++++ main.cpp | 2 +- simplecpp.cpp | 53 +++++++++++++++++++++++---------------------- simplecpp.h | 33 ++++++++++------------------ test.cpp | 2 +- 5 files changed, 63 insertions(+), 49 deletions(-) diff --git a/integration_test.py b/integration_test.py index 068da776..3ca2fd02 100644 --- a/integration_test.py +++ b/integration_test.py @@ -481,3 +481,25 @@ def test_include_header_twice(tmpdir): assert stderr == '' + +def test_define(record_property, tmpdir): # #589 + test_file = os.path.join(tmpdir, "test.cpp") + with open(test_file, 'w') as f: + f.write( +"""#define PREFIX_WITH_MACRO(test_name) Macro##test_name + +TEST_P(PREFIX_WITH_MACRO(NamingTest), n) {} +""") + + args = [ + '-DTEST_P(A,B)=void __ ## A ## _ ## B ( )', + 'test.cpp' + ] + + exitcode, stdout, stderr = simplecpp(args, cwd=tmpdir) + record_property("stdout", stdout) + record_property("stderr", stderr) + + assert exitcode == 0 + assert stderr == "test.cpp:1: syntax error: failed to expand 'TEST_P', Invalid ## usage when expanding 'TEST_P': Unexpected token ')'\n" + assert stdout == '\n' \ No newline at end of file diff --git a/main.cpp b/main.cpp index fe8ea292..62ccc022 100644 --- a/main.cpp +++ b/main.cpp @@ -207,7 +207,7 @@ int main(int argc, char **argv) std::cout << outputTokens.stringify(linenrs) << std::endl; for (const simplecpp::Output &output : outputList) { - std::cerr << output.location.file() << ':' << output.location.line << ": "; + std::cerr << outputTokens.file(output.location) << ':' << output.location.line << ": "; switch (output.type) { case simplecpp::Output::ERROR: std::cerr << "#error: "; diff --git a/simplecpp.cpp b/simplecpp.cpp index 8e10ca54..be775a90 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -182,8 +182,6 @@ static std::string replaceAll(std::string s, const std::string& from, const std: return s; } -const std::string simplecpp::Location::emptyFileName; - void simplecpp::Location::adjust(const std::string &str) { if (strpbrk(str.c_str(), "\r\n") == nullptr) { @@ -422,7 +420,7 @@ class FileStream : public simplecpp::TokenList::Stream { { if (!file) { files.push_back(filename); - throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, simplecpp::Location(files), "File is missing: " + filename); + throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, {}, "File is missing: " + filename); } init(); } @@ -564,11 +562,11 @@ void simplecpp::TokenList::dump(bool linenrs) const std::string simplecpp::TokenList::stringify(bool linenrs) const { std::ostringstream ret; - Location loc(files); + Location loc; bool filechg = true; for (const Token *tok = cfront(); tok; tok = tok->next) { if (tok->location.line < loc.line || tok->location.fileIndex != loc.fileIndex) { - ret << "\n#line " << tok->location.line << " \"" << tok->location.file() << "\"\n"; + ret << "\n#line " << tok->location.line << " \"" << file(tok->location) << "\"\n"; loc = tok->location; filechg = true; } @@ -633,16 +631,16 @@ static bool isStringLiteralPrefix(const std::string &str) str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; } -void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int line, Location *location) +void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int line, Location &location) { - if (fileIndex != location->fileIndex || line >= location->line) { - location->fileIndex = fileIndex; - location->line = line; + if (fileIndex != location.fileIndex || line >= location.line) { + location.fileIndex = fileIndex; + location.line = line; return; } - if (line + 2 >= location->line) { - location->line = line; + if (line + 2 >= location.line) { + location.line = line; while (cback()->op != '#') deleteToken(back()); deleteToken(back()); @@ -660,10 +658,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const Token *oldLastToken = nullptr; - Location location(files); - location.fileIndex = fileIndex(filename); - location.line = 1U; - location.col = 1U; + Location location(fileIndex(filename), 1, 1); while (stream.good()) { unsigned char ch = stream.readChar(); if (!stream.good()) @@ -732,7 +727,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, while (numtok->comment) numtok = numtok->previous; lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")), - std::atol(numtok->str().c_str()), &location); + std::atol(numtok->str().c_str()), location); } // #line 3 else if (llNextToken->str() == "line" && @@ -741,7 +736,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const Token *numtok = cback(); while (numtok->comment) numtok = numtok->previous; - lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location); + lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), location); } } // #endfile @@ -1478,6 +1473,12 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) return files.size() - 1U; } +const std::string& simplecpp::TokenList::file(const Location& loc) const +{ + static const std::string s_emptyFileName; + return loc.fileIndex < files.size() ? files[loc.fileIndex] : s_emptyFileName; +} + namespace simplecpp { class Macro; @@ -1885,7 +1886,7 @@ namespace simplecpp { usageList.push_back(loc); if (nameTokInst->str() == "__FILE__") { - output.push_back(new Token('\"'+loc.file()+'\"', loc)); + output.push_back(new Token('\"'+output.file(loc)+'\"', loc)); return nameTokInst->next; } if (nameTokInst->str() == "__LINE__") { @@ -2637,7 +2638,7 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI } } - const std::string &sourcefile = tok->location.file(); + const std::string &sourcefile = expr.file(tok->location); const bool systemheader = (tok1 && tok1->op == '<'); std::string header; if (systemheader) { @@ -3179,7 +3180,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (outputList) { simplecpp::Output err = { simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND, - Location(filenames), + {}, "Can not open include file '" + filename + "' that is explicitly included." }; outputList->push_back(std::move(err)); @@ -3212,7 +3213,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (!rawtok || rawtok->str() != INCLUDE) continue; - const std::string &sourcefile = rawtok->location.file(); + const std::string &sourcefile = rawtokens.file(rawtok->location); const Token * const htok = rawtok->nextSkipComments(); if (!sameline(rawtok, htok)) @@ -3369,7 +3370,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (outputList) { simplecpp::Output err = { Output::DUI_ERROR, - Location(files), + {}, "unknown standard specified: '" + dui.std + "'" }; outputList->push_back(std::move(err)); @@ -3539,7 +3540,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool systemheader = (inctok->str()[0] == '<'); const std::string header(inctok->str().substr(1U, inctok->str().size() - 2U)); - const FileData *const filedata = cache.get(rawtok->location.file(), header, dui, systemheader, files, outputList).first; + const FileData *const filedata = cache.get(rawtokens.file(rawtok->location), header, dui, systemheader, files, outputList).first; if (filedata == nullptr) { if (outputList) { simplecpp::Output out = { @@ -3632,7 +3633,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tok->next; bool closingAngularBracket = false; if (tok) { - const std::string &sourcefile = rawtok->location.file(); + const std::string &sourcefile = rawtokens.file(rawtok->location); const bool systemheader = (tok && tok->op == '<'); std::string header; @@ -3740,7 +3741,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.erase(tok->str()); } } else if (ifstates.top() == True && rawtok->str() == PRAGMA && rawtok->next && rawtok->next->str() == ONCE && sameline(rawtok,rawtok->next)) { - pragmaOnce.insert(rawtok->location.file()); + pragmaOnce.insert(rawtokens.file(rawtok->location)); } if (ifstates.top() != True && rawtok->nextcond) rawtok = rawtok->nextcond->previous; @@ -3796,7 +3797,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const std::list& temp = maybeUsedMacros[macro.name()]; usage.insert(usage.end(), temp.begin(), temp.end()); for (std::list::const_iterator usageIt = usage.begin(); usageIt != usage.end(); ++usageIt) { - MacroUsage mu(usageIt->files, macro.valueDefinedInCode()); + MacroUsage mu(macro.valueDefinedInCode()); mu.macroName = macro.name(); mu.macroLocation = macro.defineLocation(); mu.useLocation = *usageIt; diff --git a/simplecpp.h b/simplecpp.h index 15187063..42a8ed80 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -74,20 +74,16 @@ namespace simplecpp { /** * Location in source code */ - class SIMPLECPP_LIB Location { - public: - explicit Location(const std::vector &f) : files(f) {} + struct SIMPLECPP_LIB Location { + Location() = default; + Location(unsigned int fileIndex, unsigned int line, unsigned int col) + : fileIndex(fileIndex) + , line(line) + , col(col) + {} Location(const Location &loc) = default; - - Location &operator=(const Location &other) { - if (this != &other) { - fileIndex = other.fileIndex; - line = other.line; - col = other.col; - } - return *this; - } + Location &operator=(const Location &other) = default; /** increment this location by string */ void adjust(const std::string &str); @@ -104,16 +100,9 @@ namespace simplecpp { return fileIndex == other.fileIndex && line == other.line; } - const std::string& file() const { - return fileIndex < files.size() ? files[fileIndex] : emptyFileName; - } - - const std::vector &files; unsigned int fileIndex{}; unsigned int line{1}; unsigned int col{}; - private: - static const std::string emptyFileName; }; /** @@ -341,6 +330,8 @@ namespace simplecpp { return files; } + const std::string& file(const Location& loc) const; + private: TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList, int unused); @@ -356,7 +347,7 @@ namespace simplecpp { void constFoldQuestionOp(Token *&tok1); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); - void lineDirective(unsigned int fileIndex, unsigned int line, Location *location); + void lineDirective(unsigned int fileIndex, unsigned int line, Location &location); const Token* lastLineTok(int maxsize=1000) const; const Token* isLastLinePreprocessor(int maxsize=1000) const; @@ -370,7 +361,7 @@ namespace simplecpp { /** Tracking how macros are used */ struct SIMPLECPP_LIB MacroUsage { - explicit MacroUsage(const std::vector &f, bool macroValueKnown_) : macroLocation(f), useLocation(f), macroValueKnown(macroValueKnown_) {} + explicit MacroUsage(bool macroValueKnown_) : macroValueKnown(macroValueKnown_) {} std::string macroName; Location macroLocation; Location useLocation; diff --git a/test.cpp b/test.cpp index 69e08781..e46e408d 100644 --- a/test.cpp +++ b/test.cpp @@ -3206,7 +3206,7 @@ static void stdValid() static void assertToken(const std::string& s, bool name, bool number, bool comment, char op, int line) { const std::vector f; - const simplecpp::Location l(f); + const simplecpp::Location l; const simplecpp::Token t(s, l); assertEquals(name, t.name, line); assertEquals(number, t.number, line); From f4226e008514685ab49c7d85068ba2b486d434f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 10 Dec 2025 21:39:30 +0100 Subject: [PATCH 347/381] CI-unixish.yml: removed `macos-13` / added `macos-15-intel` and `macos-26` (#602) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index d6800a82..c675d565 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14, macos-15] + os: [ubuntu-22.04, ubuntu-24.04, macos-14, macos-15, macos-15-intel, macos-26] compiler: [clang++] include: - os: ubuntu-22.04 From c7b9f5ab351a40c52624b742455e04d81abd7f89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Dec 2025 20:04:52 +0100 Subject: [PATCH 348/381] updated `actions/setup-python` to `v6` (#604) --- .github/workflows/CI-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index ccf208fa..9418d31d 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -33,7 +33,7 @@ jobs: uses: microsoft/setup-msbuild@v2 - name: Set up Python 3.13 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.13' check-latest: true From e177522d05957085339d7bb958552fbc32a0ea8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Dec 2025 20:05:05 +0100 Subject: [PATCH 349/381] updated `actions/checkout` to `v6` (#603) --- .github/workflows/CI-unixish.yml | 2 +- .github/workflows/CI-windows.yml | 2 +- .github/workflows/clang-tidy.yml | 2 +- .github/workflows/format.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index c675d565..4a4cfcdc 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -25,7 +25,7 @@ jobs: CXX: ${{ matrix.compiler }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 9418d31d..06286525 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -25,7 +25,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 4b52d7bc..9109be9e 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 4d5657f8..e9f1361d 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -28,7 +28,7 @@ jobs: UNCRUSTIFY_INSTALL_DIR: ${{ github.workspace }}/runformat-uncrustify steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: persist-credentials: false From a45afce7ca12b28f0451f12b6c67eb653be6eb60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 12 Dec 2025 20:05:19 +0100 Subject: [PATCH 350/381] added `@throws` to documentation / adjusted some catch handlers (#600) --- simplecpp.cpp | 66 +++++++++++++++++++++++++-------------------------- simplecpp.h | 34 ++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index be775a90..581c9b10 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -414,6 +413,9 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { class FileStream : public simplecpp::TokenList::Stream { public: + /** + * @throws simplecpp::Output thrown if file is not found + */ // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members explicit FileStream(const std::string &filename, std::vector &files) : file(fopen(filename.c_str(), "rb")) @@ -487,7 +489,7 @@ simplecpp::TokenList::TokenList(const std::string &filename, std::vectorpush_back(e); } } @@ -1488,6 +1490,9 @@ namespace simplecpp { public: explicit Macro(std::vector &f) : nameTokDef(nullptr), valueToken(nullptr), endToken(nullptr), files(f), tokenListDefine(f), variadic(false), variadicOpt(false), valueDefinedInCode_(false) {} + /** + * @throws std::runtime_error thrown on bad macro syntax + */ Macro(const Token *tok, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(true) { if (sameline(tok->previousSkipComments(), tok)) throw std::runtime_error("bad macro syntax"); @@ -1504,6 +1509,9 @@ namespace simplecpp { throw std::runtime_error("bad macro syntax"); } + /** + * @throws std::runtime_error thrown on bad macro syntax + */ Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); StdCharBufStream stream(reinterpret_cast(def.data()), def.size()); @@ -1552,7 +1560,9 @@ namespace simplecpp { * @param macros list of macros * @param inputFiles the input files * @return token after macro - * @throw Can throw wrongNumberOfParameters or invalidHashHash + * @throws Error thrown on missing or invalid preprocessor directives + * @throws wrongNumberOfParameters thrown on invalid number of parameters + * @throws invalidHashHash thrown on invalid ## usage */ const Token * expand(TokenList & output, const Token * rawtok, @@ -2544,7 +2554,9 @@ namespace simplecpp { } } -/** Evaluate sizeof(type) */ +/** Evaluate sizeof(type) + * @throws std::runtime_error thrown on missing arguments or invalid expression + */ static void simplifySizeof(simplecpp::TokenList &expr, const std::map &sizeOfType) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { @@ -2612,6 +2624,10 @@ static std::string dirPath(const std::string& path, bool withTrailingSlash=true) } static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const std::string &sourcefile, const std::string &header, bool systemheader); + +/** Evaluate __has_include(include) + * @throws std::runtime_error thrown on missing arguments or invalid expression + */ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI &dui) { if (!isCpp17OrLater(dui) && !isGnu(dui)) @@ -2668,6 +2684,9 @@ static void simplifyHasInclude(simplecpp::TokenList &expr, const simplecpp::DUI } } +/** Evaluate name + * @throws std::runtime_error thrown on undefined function-like macro + */ static void simplifyName(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { @@ -2696,7 +2715,7 @@ static void simplifyName(simplecpp::TokenList &expr) * unsigned long long value, updating pos to point to the first * unused element of s. * Returns ULLONG_MAX if the result is not representable and - * throws if the above requirements were not possible to satisfy. + * @throws std::runtime_error thrown if the above requirements were not possible to satisfy. */ static unsigned long long stringToULLbounded( const std::string& s, @@ -2716,34 +2735,6 @@ static unsigned long long stringToULLbounded( return value; } -/* Converts character literal (including prefix, but not ud-suffix) - * to long long value. - * - * Assumes ASCII-compatible single-byte encoded str for narrow literals - * and UTF-8 otherwise. - * - * For target assumes - * - execution character set encoding matching str - * - UTF-32 execution wide-character set encoding - * - requirements for __STDC_UTF_16__, __STDC_UTF_32__ and __STDC_ISO_10646__ satisfied - * - char16_t is 16bit wide - * - char32_t is 32bit wide - * - wchar_t is 32bit wide and unsigned - * - matching char signedness to host - * - matching sizeof(int) to host - * - * For host assumes - * - ASCII-compatible execution character set - * - * For host and target assumes - * - CHAR_BIT == 8 - * - two's complement - * - * Implements multi-character narrow literals according to GCC's behavior, - * except multi code unit universal character names are not supported. - * Multi-character wide literals are not supported. - * Limited support of universal character names for non-UTF-8 execution character set encodings. - */ long long simplecpp::characterLiteralToLL(const std::string& str) { // default is wide/utf32 @@ -2937,6 +2928,9 @@ long long simplecpp::characterLiteralToLL(const std::string& str) return multivalue; } +/** + * @throws std::runtime_error thrown on invalid literal + */ static void simplifyNumbers(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { @@ -2959,6 +2953,10 @@ static void simplifyComments(simplecpp::TokenList &expr) } } +/** + * @throws std::runtime_error thrown on invalid literals, missing sizeof arguments or invalid expressions, + * missing __has_include() arguments or expressions, undefined function-like macros, invalid number literals + */ static long long evaluate(simplecpp::TokenList &expr, const simplecpp::DUI &dui, const std::map &sizeOfType) { simplifyComments(expr); @@ -3692,7 +3690,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const long long result = evaluate(expr, dui, sizeOfType); conditionIsTrue = (result != 0); } - } catch (const std::exception &e) { + } catch (const std::runtime_error &e) { if (outputList) { std::string msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; if (e.what() && *e.what()) diff --git a/simplecpp.h b/simplecpp.h index 42a8ed80..6f7c8c66 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -338,12 +338,18 @@ namespace simplecpp { void combineOperators(); void constFoldUnaryNotPosNeg(Token *tok); + /** + * @throws std::overflow_error thrown on overflow or division by zero + */ void constFoldMulDivRem(Token *tok); void constFoldAddSub(Token *tok); void constFoldShift(Token *tok); void constFoldComparison(Token *tok); void constFoldBitwise(Token *tok); void constFoldLogicalOp(Token *tok); + /** + * @throws std::runtime_error thrown on invalid expressions + */ void constFoldQuestionOp(Token *&tok1); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); @@ -501,6 +507,34 @@ namespace simplecpp { id_map_type mIdMap; }; + /** Converts character literal (including prefix, but not ud-suffix) to long long value. + * + * Assumes ASCII-compatible single-byte encoded str for narrow literals + * and UTF-8 otherwise. + * + * For target assumes + * - execution character set encoding matching str + * - UTF-32 execution wide-character set encoding + * - requirements for __STDC_UTF_16__, __STDC_UTF_32__ and __STDC_ISO_10646__ satisfied + * - char16_t is 16bit wide + * - char32_t is 32bit wide + * - wchar_t is 32bit wide and unsigned + * - matching char signedness to host + * - matching sizeof(int) to host + * + * For host assumes + * - ASCII-compatible execution character set + * + * For host and target assumes + * - CHAR_BIT == 8 + * - two's complement + * + * Implements multi-character narrow literals according to GCC's behavior, + * except multi code unit universal character names are not supported. + * Multi-character wide literals are not supported. + * Limited support of universal character names for non-UTF-8 execution character set encodings. + * @throws std::runtime_error thrown on invalid literal + */ SIMPLECPP_LIB long long characterLiteralToLL(const std::string& str); SIMPLECPP_LIB FileDataCache load(const TokenList &rawtokens, std::vector &filenames, const DUI &dui, OutputList *outputList = nullptr, FileDataCache cache = {}); From 43f68be586cfb8e20583a25f62047eaa38ee9a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 14 Dec 2025 17:15:27 +0100 Subject: [PATCH 351/381] CI-unixish.yml: also run sanitizers on latest macOS (#607) --- .github/workflows/CI-unixish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 4a4cfcdc..d7ea894a 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -119,7 +119,7 @@ jobs: make -j$(nproc) test selfcheck CXXOPTS="-Werror -stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" - name: Run AddressSanitizer - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-26' run: | make clean make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=address" LDOPTS="-fsanitize=address" @@ -127,7 +127,7 @@ jobs: ASAN_OPTIONS: detect_stack_use_after_return=1 - name: Run UndefinedBehaviorSanitizer - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-26' run: | make clean make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDOPTS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" From 077bed15bd38605d64ed9aee8642e6507e72df46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 15 Dec 2025 12:17:23 +0100 Subject: [PATCH 352/381] build all code with all available C++ standards in CI (#606) --- .github/workflows/CI-unixish.yml | 19 +++++++++++++++---- .github/workflows/CI-windows.yml | 18 +++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index d7ea894a..6fd8a429 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -71,15 +71,26 @@ jobs: run: | make -j$(nproc) selfcheck - - name: make testrunner (c++17) + - name: make (c++14) run: | make clean - make -j$(nproc) testrunner CXXOPTS="-std=c++17" + make -j$(nproc) CXXOPTS="-Werror -std=c++14" - - name: make testrunner (c++20) + - name: make (c++17) run: | make clean - make -j$(nproc) testrunner CXXOPTS="-std=c++20" + make -j$(nproc) CXXOPTS="-Werror -std=c++17" + + - name: make (c++20) + run: | + make clean + make -j$(nproc) CXXOPTS="-Werror -std=c++20" + + - name: make (c++23) + run: | + make clean + # ubuntu-22.04 and macos-14 do not support c++23 yet + make -j$(nproc) CXXOPTS="-Werror -std=c++2b" - name: Run CMake run: | diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 06286525..e78c1f7d 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -63,4 +63,20 @@ jobs: run: | set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe python -m pytest integration_test.py -vv || exit /b !errorlevel! - + + - name: Run CMake (c++17) + run: | + cmake -S . -B build.cxx17 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=20 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! + + - name: Build (c++17) + run: | + msbuild -m build.cxx17\simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! + + - name: Run CMake (c++20) + run: | + cmake -S . -B build.cxx20 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=20 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! + + - name: Build (c++20) + run: | + msbuild -m build.cxx20\simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! + From 00a131e5c988b35765e882cacc8b31e2d988d558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 16 Dec 2025 10:34:48 +0100 Subject: [PATCH 353/381] improved `simplecpp::TokenList` constructors (#599) --- simplecpp.h | 51 +++++++++++++++++++++++++---- test.cpp | 94 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 16 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 6f7c8c66..9a847d14 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -69,6 +69,46 @@ namespace simplecpp { enum cppstd_t : std::int8_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; using TokenString = std::string; + +#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) + using View = std::string_view; +#else + struct View + { + // cppcheck-suppress noExplicitConstructor + View(const char* data) + : mData(data) + , mSize(strlen(data)) + {} + + // only provide when std::span is not available so using untyped initilization won't use View +#if !defined(__cpp_lib_span) + View(const char* data, std::size_t size) + : mData(data) + , mSize(size) + {} + + // cppcheck-suppress noExplicitConstructor + View(const std::string& str) + : mData(str.data()) + , mSize(str.size()) + {} +#endif // !defined(__cpp_lib_span) + + const char* data() const { + return mData; + } + + std::size_t size() const { + return mSize; + } + + private: + const char* mData; + std::size_t mSize; + }; +#endif // defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) + class Macro; /** @@ -217,7 +257,6 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); -#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR /** generates a token list from the given buffer */ template TokenList(const char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) @@ -228,7 +267,7 @@ namespace simplecpp { TokenList(const unsigned char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(data, size-1, filenames, filename, outputList, 0) {} - +#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR /** generates a token list from the given buffer */ TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(data, size, filenames, filename, outputList, 0) @@ -237,13 +276,11 @@ namespace simplecpp { TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(reinterpret_cast(data), size, filenames, filename, outputList, 0) {} -#endif -#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) +#endif // SIMPLECPP_TOKENLIST_ALLOW_PTR /** generates a token list from the given buffer */ - TokenList(std::string_view data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) + TokenList(View data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(reinterpret_cast(data.data()), data.size(), filenames, filename, outputList, 0) {} -#endif #ifdef __cpp_lib_span /** generates a token list from the given buffer */ TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) @@ -254,7 +291,7 @@ namespace simplecpp { TokenList(std::span data, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(data.data(), data.size(), filenames, filename, outputList, 0) {} -#endif +#endif // __cpp_lib_span /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); diff --git a/test.cpp b/test.cpp index e46e408d..300245cd 100644 --- a/test.cpp +++ b/test.cpp @@ -3299,20 +3299,97 @@ static void preprocess_files() } } -static void safe_api() +static void tokenlist_api() { - // this test is to make sure the safe APIs are compiling -#if defined(__cpp_lib_string_view) || defined(__cpp_lib_span) std::vector filenames; -# if defined(__cpp_lib_string_view) +# if !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) + // sized array + size + { + char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(input,sizeof(input),filenames,""); + } + { + const char input[] = "code"; + simplecpp::TokenList(input,sizeof(input),filenames,""); + } + { + unsigned char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(input,sizeof(input),filenames,""); + } + { + const unsigned char input[] = "code"; + simplecpp::TokenList(input,sizeof(input),filenames,""); + } +#endif // !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) + // pointer via View + { + const char * const input = "code"; + simplecpp::TokenList({input},filenames,""); + } + // sized array via View + { + char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(simplecpp::View{input},filenames,""); + } + { + const char input[] = "code"; + simplecpp::TokenList(simplecpp::View{input},filenames,""); + } + // sized array + size via View/std::span + { + char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList({input,sizeof(input)},filenames,""); + } + { + const char input[] = "code"; + simplecpp::TokenList({input,sizeof(input)},filenames,""); + } + // sized array + { + char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(input,filenames,""); + } + { + const char input[] = "code"; + simplecpp::TokenList(input,filenames,""); + } + { + unsigned char input[] = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(input,filenames,""); + } + { + const unsigned char input[] = "code"; + simplecpp::TokenList(input,filenames,""); + } + // std::string via View/std::span (implicit) + { + std::string input = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList(input,filenames,""); + } + { + const std::string input = "code"; + simplecpp::TokenList(input,filenames,""); + } + // std::string via View/std::span (explicit) + { + std::string input = "code"; // NOLINT(misc-const-correctness) + simplecpp::TokenList({input},filenames,""); + } + { + const std::string input = "code"; + simplecpp::TokenList({input},filenames,""); + } + + // this test is to make sure the safe APIs are compiling +#ifdef __cpp_lib_string_view { const char input[] = "code"; const std::string_view sv = input; // std::string_view can be implicitly converted into a std::span simplecpp::TokenList(sv,filenames,""); } -# endif -# ifdef __cpp_lib_span +#endif // __cpp_lib_string_view +#ifdef __cpp_lib_span { char input[] = "code"; const std::span sp = input; @@ -3333,8 +3410,7 @@ static void safe_api() const std::span sp = input; simplecpp::TokenList(sp,filenames,""); } -# endif -#endif +#endif // __cpp_lib_span } static void isAbsolutePath() { @@ -3660,7 +3736,7 @@ int main(int argc, char **argv) TEST_CASE(preprocess_files); - TEST_CASE(safe_api); + TEST_CASE(tokenlist_api); TEST_CASE(isAbsolutePath); From cb62a62042986f5bd6da281d924980e429d37242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 17 Dec 2025 18:33:59 +0100 Subject: [PATCH 354/381] improved test coverage of `simplecpp::characterLiteralToLL()` (#601) --- test.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test.cpp b/test.cpp index 300245cd..2b2badd6 100644 --- a/test.cpp +++ b/test.cpp @@ -321,6 +321,8 @@ static void characterLiteral() ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\343\201\202'"), std::runtime_error, "code point too large"); ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8'\360\223\200\200'"), std::runtime_error, "code point too large"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'\\U11111111"), std::runtime_error, "code point too large"); + ASSERT_EQUALS('\x89', simplecpp::characterLiteralToLL("'\x89'")); ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\x89'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); @@ -372,6 +374,33 @@ static void characterLiteral() ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xed\xa0\x80'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U'\xed\xbf\xbf'"), std::runtime_error, "assumed UTF-8 encoded source, but sequence is invalid"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL(""), std::runtime_error, "expected a character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("LU"), std::runtime_error, "expected a character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL(";\n"), std::runtime_error, "expected a character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8U"), std::runtime_error, "expected a character literal"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'\n\n"), std::runtime_error, "raw single quotes and newlines not allowed in character literals"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("''&"), std::runtime_error, "raw single quotes and newlines not allowed in character literals"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("L'\fff"), std::runtime_error, "multiple characters only supported in narrow character literals"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'\\\n"), std::runtime_error, "unexpected end of character literal"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'"), std::runtime_error, "missing closing quote in character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u'"), std::runtime_error, "missing closing quote in character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("L'"), std::runtime_error, "missing closing quote in character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'a"), std::runtime_error, "missing closing quote in character literal"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("''"), std::runtime_error, "empty character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("L''"), std::runtime_error, "empty character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("U''"), std::runtime_error, "empty character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u''"), std::runtime_error, "empty character literal"); + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u8''"), std::runtime_error, "empty character literal"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("'\\555"), std::runtime_error, "numeric escape sequence too large"); + + ASSERT_THROW_EQUALS(simplecpp::characterLiteralToLL("u'Ó"), std::runtime_error, "assumed UTF-8 encoded source, but character literal ends unexpectedly"); } static void combineOperators_floatliteral() From 9eea499a2412eb47b5338bb5071a57bfa9fab861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 17 Dec 2025 21:36:42 +0100 Subject: [PATCH 355/381] CI-windows.yml: fixed standard in C++17 build (#609) --- .github/workflows/CI-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index e78c1f7d..521bc24c 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -66,7 +66,7 @@ jobs: - name: Run CMake (c++17) run: | - cmake -S . -B build.cxx17 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=20 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! + cmake -S . -B build.cxx17 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=17 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! - name: Build (c++17) run: | From 10cfb949b62819fe37f731255818faf4fd2ddd54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 17 Dec 2025 21:37:49 +0100 Subject: [PATCH 356/381] added MinGW workflow (#475) Co-authored-by: glankk --- .github/workflows/CI-mingw.yml | 138 +++++++++++++++++++++++++++++++++ CMakeLists.txt | 12 ++- selfcheck.sh | 15 +++- simplecpp.cpp | 2 +- 4 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/CI-mingw.yml diff --git a/.github/workflows/CI-mingw.yml b/.github/workflows/CI-mingw.yml new file mode 100644 index 00000000..a3bac5cf --- /dev/null +++ b/.github/workflows/CI-mingw.yml @@ -0,0 +1,138 @@ +name: CI-mingw + +on: [push, pull_request] + +permissions: + contents: read + +defaults: + run: + shell: msys2 {0} + +jobs: + build: + + strategy: + matrix: + compiler: [g++, clang++] + # TODO: add MSYS after #556 is fixed + msystem: [MINGW32, MINGW64, CLANG64] + include: + #- msystem: MSYS + # pkg-prefix: '' + - msystem: MINGW32 + pkg-prefix: 'mingw-w64-i686-' + - msystem: MINGW64 + pkg-prefix: 'mingw-w64-x86_64-' + - msystem: CLANG64 + pkg-prefix: 'mingw-w64-clang-x86_64-' + - compiler: g++ + compiler-pkg: gcc + - compiler: clang++ + compiler-pkg: clang + exclude: + - msystem: CLANG64 + compiler: g++ + fail-fast: false + + runs-on: windows-2025 + + env: + CXX: ${{ matrix.compiler }} + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up MSYS2 + uses: msys2/setup-msys2@v2 + with: + release: false # use pre-installed + msystem: ${{ matrix.msystem }} + # TODO: install mingw-w64-x86_64-make and use mingw32.make instead - currently fails with "Windows Subsystem for Linux has no installed distributions." + # TODO: also run tests with non-prefixed Python? + install: >- + make + ${{ matrix.pkg-prefix }}cmake + ${{ matrix.pkg-prefix }}python + ${{ matrix.pkg-prefix }}python-pytest + + - name: install compiler + run: | + pacman -S --noconfirm ${{ matrix.pkg-prefix }}${{ matrix.compiler-pkg }} + ${CXX} -v + + - name: make simplecpp + run: | + make -j$(nproc) CXXOPTS="-Werror" + + # gcc *and* clang are required to run-tests.py + # install it at this point since it has gcc as dependency which might interfere with the build + - name: install compiler (clang) + if: matrix.compiler == 'g++' + run: | + pacman -S --noconfirm clang + + - name: install compiler (gcc) + if: matrix.compiler == 'clang++' + run: | + pacman -S --noconfirm gcc + + - name: make test + run: | + # TODO: run tests with Windows paths + make -j$(nproc) test + + - name: selfcheck + run: | + # TODO: run tests with Windows paths + make -j$(nproc) selfcheck + + - name: make (c++14) + run: | + make clean + make -j$(nproc) CXXOPTS="-Werror -std=c++14" + + - name: make (c++17) + run: | + make clean + make -j$(nproc) CXXOPTS="-Werror -std=c++17" + + - name: make (c++20) + run: | + make clean + make -j$(nproc) CXXOPTS="-Werror -std=c++20" + + - name: make (c++23) + run: | + make clean + make -j$(nproc) CXXOPTS="-Werror -std=c++23" + + - name: Run CMake + run: | + cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On + + - name: CMake simplecpp + run: | + cmake --build cmake.output --target simplecpp -- -j $(nproc) + + - name: CMake testrunner + run: | + cmake --build cmake.output --target testrunner -- -j $(nproc) + + - name: Run testrunner + run: | + ./cmake.output/testrunner + + - name: Run with libstdc++ debug mode + if: matrix.compiler == 'g++' + run: | + make clean + make -j$(nproc) test selfcheck CXXOPTS="-Werror -g3 -D_GLIBCXX_DEBUG" + + - name: Run with libc++ hardening mode + if: matrix.compiler == 'clang++' && matrix.msystem == 'CLANG64' + run: | + make clean + make -j$(nproc) test selfcheck CXXOPTS="-Werror -stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" diff --git a/CMakeLists.txt b/CMakeLists.txt index f13fb3fb..0a90efae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,9 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Woverloaded-virtual) # when a function declaration hides virtual functions from a base class add_compile_options(-Wsuggest-attribute=noreturn) - add_compile_options_safe(-Wuseless-cast) + if (NOT MINGW) + add_compile_options_safe(-Wuseless-cast) + endif() # we are not interested in these set_source_files_properties(test.cpp PROPERTIES COMPILE_FLAGS -Wno-multichar) @@ -62,6 +64,14 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # contradicts -Wcovered-switch-default add_compile_options(-Wno-switch-default) + if (MINGW) + add_compile_options(-Wno-reserved-macro-identifier) + add_compile_options(-Wno-unused-macros) + endif() + + # these are experimental warnings which might produce false positives + add_compile_options_safe(-Wno-thread-safety-negative) + add_compile_options_safe(-Wno-thread-safety-beta) # TODO: fix these? add_compile_options(-Wno-padded) diff --git a/selfcheck.sh b/selfcheck.sh index e708023a..b2129cc9 100755 --- a/selfcheck.sh +++ b/selfcheck.sh @@ -41,16 +41,20 @@ if [ "$cxx_type" = "Ubuntu" ] || [ "$cxx_type" = "Debian" ]; then fi # TODO: generate defines from compiler -if [ "$cxx_type" = "g++" ]; then +if [ "$cxx_type" = "g++" ] || [ "$cxx_type" = "g++.exe" ]; then defs= defs="$defs -D__GNUC__" defs="$defs -D__STDC__" defs="$defs -D__x86_64__" defs="$defs -D__STDC_HOSTED__" defs="$defs -D__CHAR_BIT__=8" + if [ "${MSYSTEM}" = "MINGW32" ] || [ "${MSYSTEM}" = "MINGW64" ]; then + defs="$defs -D_WIN32" + fi defs="$defs -D__has_builtin(x)=(1)" defs="$defs -D__has_cpp_attribute(x)=(1)" defs="$defs -D__has_attribute(x)=(1)" + defs="$defs -Ddefined(x)=(0)" inc= while read line @@ -63,12 +67,19 @@ elif [ "$cxx_type" = "clang" ]; then defs="$defs -D__x86_64__" defs="$defs -D__STDC_HOSTED__" defs="$defs -D__CHAR_BIT__=8" + defs="$defs -D__BYTE_ORDER__=1234" + defs="$defs -D__SIZEOF_SIZE_T__=8" + if [ "${MSYSTEM}" = "MINGW32" ] || [ "${MSYSTEM}" = "MINGW64" ] || [ "${MSYSTEM}" = "CLANG64" ]; then + defs="$defs -D_WIN32" + fi defs="$defs -D__has_builtin(x)=(1)" defs="$defs -D__has_cpp_attribute(x)=(1)" defs="$defs -D__has_feature(x)=(1)" - defs="$defs -D__has_include_next(x)=(0)" + defs="$defs -D__has_include_next(x)=(1)" defs="$defs -D__has_attribute(x)=(0)" defs="$defs -D__building_module(x)=(0)" + defs="$defs -D__has_extension(x)=(1)" + defs="$defs -Ddefined(x)=(0)" inc= while read line diff --git a/simplecpp.cpp b/simplecpp.cpp index 581c9b10..b1525d6d 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3120,7 +3120,7 @@ std::pair simplecpp::FileDataCache::get(const std:: bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) { #ifdef _WIN32 - HANDLE hFile = CreateFileA(path.c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + HANDLE hFile = CreateFileA(path.c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (hFile == INVALID_HANDLE_VALUE) return false; From d7a259d2f80240c1749d97407a64fdeb6ab9705b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 31 Dec 2025 15:31:58 +0100 Subject: [PATCH 357/381] do not default `simplecpp::Location::line` to `1` (#597) --- simplecpp.cpp | 1 + simplecpp.h | 2 +- test.cpp | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index b1525d6d..e2845dd2 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -565,6 +565,7 @@ std::string simplecpp::TokenList::stringify(bool linenrs) const { std::ostringstream ret; Location loc; + loc.line = 1; bool filechg = true; for (const Token *tok = cfront(); tok; tok = tok->next) { if (tok->location.line < loc.line || tok->location.fileIndex != loc.fileIndex) { diff --git a/simplecpp.h b/simplecpp.h index 9a847d14..d131ded4 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -141,7 +141,7 @@ namespace simplecpp { } unsigned int fileIndex{}; - unsigned int line{1}; + unsigned int line{}; unsigned int col{}; }; diff --git a/test.cpp b/test.cpp index 2b2badd6..a17ffbb9 100644 --- a/test.cpp +++ b/test.cpp @@ -2761,7 +2761,7 @@ static void readfile_file_not_found() simplecpp::OutputList outputList; std::vector files; (void)simplecpp::TokenList("NotAFile", files, &outputList); - ASSERT_EQUALS("file0,1,file_not_found,File is missing: NotAFile\n", toString(outputList)); + ASSERT_EQUALS("file0,0,file_not_found,File is missing: NotAFile\n", toString(outputList)); } static void stringify1() From 10f505214f70b03a7e9b34e293b7324e3a996415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 2 Jan 2026 17:42:49 +0100 Subject: [PATCH 358/381] enabled and fixed `modernize-return-braced-init-list` clang-tidy warnings (#610) --- .clang-tidy | 1 - simplecpp.cpp | 10 +++++----- test.cpp | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index b63cb3d9..d16cc35c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -29,7 +29,6 @@ Checks: > -modernize-avoid-c-arrays, -modernize-loop-convert, -modernize-pass-by-value, - -modernize-return-braced-init-list, -modernize-use-nodiscard, -modernize-use-trailing-return-type, -readability-avoid-nested-conditional-operator, diff --git a/simplecpp.cpp b/simplecpp.cpp index e2845dd2..62dca039 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1700,19 +1700,19 @@ namespace simplecpp { : Error(loc, format(macroName, message)) {} static inline invalidHashHash unexpectedToken(const Location &loc, const std::string ¯oName, const Token *tokenA) { - return invalidHashHash(loc, macroName, "Unexpected token '"+ tokenA->str()+"'"); + return {loc, macroName, "Unexpected token '"+ tokenA->str()+"'"}; } static inline invalidHashHash cannotCombine(const Location &loc, const std::string ¯oName, const Token *tokenA, const Token *tokenB) { - return invalidHashHash(loc, macroName, "Combining '"+ tokenA->str()+ "' and '"+ tokenB->str() + "' yields an invalid token."); + return {loc, macroName, "Combining '"+ tokenA->str()+ "' and '"+ tokenB->str() + "' yields an invalid token."}; } static inline invalidHashHash unexpectedNewline(const Location &loc, const std::string ¯oName) { - return invalidHashHash(loc, macroName, "Unexpected newline"); + return {loc, macroName, "Unexpected newline"}; } static inline invalidHashHash universalCharacterUB(const Location &loc, const std::string ¯oName, const Token* tokenA, const std::string& strAB) { - return invalidHashHash(loc, macroName, "Combining '\\"+ tokenA->str()+ "' and '"+ strAB.substr(tokenA->str().size()) + "' yields universal character '\\" + strAB + "'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4."); + return {loc, macroName, "Combining '\\"+ tokenA->str()+ "' and '"+ strAB.substr(tokenA->str().size()) + "' yields universal character '\\" + strAB + "'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4."}; } }; private: @@ -1829,7 +1829,7 @@ namespace simplecpp { std::vector getMacroParameters(const Token *nameTokInst, bool calledInDefine) const { if (!nameTokInst->next || nameTokInst->next->op != '(' || !functionLike()) - return std::vector(); + return {}; std::vector parametertokens; parametertokens.push_back(nameTokInst->next); diff --git a/test.cpp b/test.cpp index a17ffbb9..78df4a0e 100644 --- a/test.cpp +++ b/test.cpp @@ -82,7 +82,7 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { std::istringstream istr(std::string(code, size)); - return simplecpp::TokenList(istr,filenames,filename,outputList); + return {istr,filenames,filename,outputList}; } static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) From 6d45cd7bf7f12878ed99a5dcf5ee410ef6492a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 14 Jan 2026 02:33:57 +0100 Subject: [PATCH 359/381] fixed #616 - report bad macro syntax via `OutputList` (#617) --- simplecpp.cpp | 17 +++++++++++++++-- test.cpp | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 62dca039..3a05ec84 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3341,8 +3341,21 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; const std::string lhs(macrostr.substr(0,eq)); const std::string rhs(eq==std::string::npos ? std::string("1") : macrostr.substr(eq+1)); - const Macro macro(lhs, rhs, dummy); - macros.insert(std::pair(macro.name(), macro)); + try { + const Macro macro(lhs, rhs, dummy); + macros.insert(std::pair(macro.name(), macro)); + } catch (const std::runtime_error& e) { + if (outputList) { + simplecpp::Output err = { + Output::DUI_ERROR, + {}, + e.what() + }; + outputList->push_back(std::move(err)); + } + output.clear(); + return; + } } const bool strictAnsiUndefined = dui.undefined.find("__STRICT_ANSI__") != dui.undefined.cend(); diff --git a/test.cpp b/test.cpp index 78df4a0e..9583468f 100644 --- a/test.cpp +++ b/test.cpp @@ -3442,6 +3442,18 @@ static void tokenlist_api() #endif // __cpp_lib_span } +static void bad_macro_syntax() // #616 +{ + simplecpp::DUI dui; + dui.defines.emplace_back("\""); + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("", dui, &outputList)); + ASSERT_EQUALS(1, outputList.size()); + ASSERT_EQUALS(simplecpp::Output::Type::DUI_ERROR, outputList.cbegin()->type); + ASSERT_EQUALS("bad macro syntax. macroname=\" value=1", outputList.cbegin()->msg); +} + static void isAbsolutePath() { #ifdef _WIN32 ASSERT_EQUALS(true, simplecpp::isAbsolutePath("C:\\foo\\bar")); @@ -3769,6 +3781,8 @@ int main(int argc, char **argv) TEST_CASE(isAbsolutePath); + TEST_CASE(bad_macro_syntax); + TEST_CASE(fuzz_crash); TEST_CASE(leak); From 15f833511270545440a8567ba44d8b4c9d9e6936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 14 Jan 2026 15:53:18 +0100 Subject: [PATCH 360/381] added more `@throws` to documentation (#618) --- simplecpp.cpp | 1 + simplecpp.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index 3a05ec84..94ee2fa7 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2957,6 +2957,7 @@ static void simplifyComments(simplecpp::TokenList &expr) /** * @throws std::runtime_error thrown on invalid literals, missing sizeof arguments or invalid expressions, * missing __has_include() arguments or expressions, undefined function-like macros, invalid number literals + * @throws std::overflow_error thrown on overflow or division by zero */ static long long evaluate(simplecpp::TokenList &expr, const simplecpp::DUI &dui, const std::map &sizeOfType) { diff --git a/simplecpp.h b/simplecpp.h index d131ded4..46a43bc4 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -311,6 +311,10 @@ namespace simplecpp { std::string stringify(bool linenrs = false) const; void readfile(Stream &stream, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** + * @throws std::overflow_error thrown on overflow or division by zero + * @throws std::runtime_error thrown on invalid expressions + */ void constFold(); void removeComments(); From 10c96815e402299da1d3ab1f753dda4c1b0b12f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 15 Jan 2026 16:58:22 +0100 Subject: [PATCH 361/381] use `emplace_back` (#619) --- simplecpp.cpp | 74 +++++++++++++++++++++++++-------------------------- test.cpp | 12 ++++----- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 94ee2fa7..1d14a06c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -421,7 +421,7 @@ class FileStream : public simplecpp::TokenList::Stream { : file(fopen(filename.c_str(), "rb")) { if (!file) { - files.push_back(filename); + files.emplace_back(filename); throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, {}, "File is missing: " + filename); } init(); @@ -490,7 +490,7 @@ simplecpp::TokenList::TokenList(const std::string &filename, std::vectorpush_back(e); + outputList->emplace_back(e); } } @@ -625,7 +625,7 @@ static void portabilityBackslash(simplecpp::OutputList *outputList, const simple location, "Combination 'backslash space newline' is not portable." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } static bool isStringLiteralPrefix(const std::string &str) @@ -674,7 +674,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, location, "The code contains unhandled character(s) (character code=" + std::to_string(static_cast(ch)) + "). Neither unicode nor extended ascii is supported." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } clear(); return; @@ -876,7 +876,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, location, "Invalid newline in raw string delimiter." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } return; } @@ -890,7 +890,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, location, "Raw string missing terminating delimiter." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } return; } @@ -1434,7 +1434,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca location, std::string("No pair for character (") + start + "). Can't process file. File is either invalid or unicode, which is currently not supported." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } return ""; } @@ -1472,7 +1472,7 @@ unsigned int simplecpp::TokenList::fileIndex(const std::string &filename) if (files[i] == filename) return i; } - files.push_back(filename); + files.emplace_back(filename); return files.size() - 1U; } @@ -1754,7 +1754,7 @@ namespace simplecpp { break; } if (argtok->op != ',') - args.push_back(argtok->str()); + args.emplace_back(argtok->str()); argtok = argtok->next; } if (!sameline(nametoken, argtok)) { @@ -1832,19 +1832,19 @@ namespace simplecpp { return {}; std::vector parametertokens; - parametertokens.push_back(nameTokInst->next); + parametertokens.emplace_back(nameTokInst->next); unsigned int par = 0U; for (const Token *tok = nameTokInst->next->next; calledInDefine ? sameline(tok, nameTokInst) : (tok != nullptr); tok = tok->next) { if (tok->op == '(') ++par; else if (tok->op == ')') { if (par == 0U) { - parametertokens.push_back(tok); + parametertokens.emplace_back(tok); break; } --par; } else if (par == 0U && tok->op == ',' && (!variadic || parametertokens.size() < args.size())) - parametertokens.push_back(tok); + parametertokens.emplace_back(tok); } return parametertokens; } @@ -1894,7 +1894,7 @@ namespace simplecpp { std::cout << " expand " << name() << " " << locstring(defineLocation()) << std::endl; #endif - usageList.push_back(loc); + usageList.emplace_back(loc); if (nameTokInst->str() == "__FILE__") { output.push_back(new Token('\"'+output.file(loc)+'\"', loc)); @@ -1954,11 +1954,11 @@ namespace simplecpp { for (const Token *tok = parametertokens1[0]; tok && par < parametertokens1.size(); tok = tok->next) { if (tok->str() == "__COUNTER__") { tokensparams.push_back(new Token(toString(counterMacro.usageList.size()), tok->location)); - counterMacro.usageList.push_back(tok->location); + counterMacro.usageList.emplace_back(tok->location); } else { tokensparams.push_back(new Token(*tok)); if (tok == parametertokens1[par]) { - parametertokens2.push_back(tokensparams.cback()); + parametertokens2.emplace_back(tokensparams.cback()); par++; } } @@ -3183,7 +3183,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, {}, "Can not open include file '" + filename + "' that is explicitly included." }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } continue; } @@ -3197,7 +3197,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (dui.removeComments) filedata->tokens.removeComments(); - filelist.push_back(filedata->tokens.front()); + filelist.emplace_back(filedata->tokens.front()); } for (const Token *rawtok = rawtokens.cfront(); rawtok || !filelist.empty(); rawtok = rawtok ? rawtok->next : nullptr) { @@ -3236,7 +3236,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (dui.removeComments) filedata->tokens.removeComments(); - filelist.push_back(filedata->tokens.front()); + filelist.emplace_back(filedata->tokens.front()); } return cache; @@ -3257,7 +3257,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token err.location, "failed to expand \'" + tok->str() + "\', " + err.what }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } return false; } @@ -3352,7 +3352,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL {}, e.what() }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } output.clear(); return; @@ -3386,7 +3386,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL {}, "unknown standard specified: '" + dui.std + "'" }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } output.clear(); return; @@ -3443,7 +3443,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "#" + rawtok->str() + " without #if" }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } output.clear(); return; @@ -3464,7 +3464,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::move(msg) }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } if (rawtok->str() == ERROR) { output.clear(); @@ -3491,7 +3491,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "Failed to parse #define" }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } output.clear(); return; @@ -3502,7 +3502,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL err.location, "Failed to parse #define, " + err.what }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } output.clear(); return; @@ -3543,7 +3543,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "No header in #include" }; - outputList->push_back(std::move(err)); + outputList->emplace_back(std::move(err)); } output.clear(); return; @@ -3561,7 +3561,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "Header not found: " + inctok->str() }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } } else if (includetokenstack.size() >= 400) { if (outputList) { @@ -3570,7 +3570,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "#include nested too deeply" }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } } else if (pragmaOnce.find(filedata->filename) == pragmaOnce.end()) { includetokenstack.push(gotoNextLine(rawtok)); @@ -3585,7 +3585,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "Syntax error in #" + rawtok->str() }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } output.clear(); return; @@ -3596,10 +3596,10 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL conditionIsTrue = false; else if (rawtok->str() == IFDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end() || (hasInclude && rawtok->next->str() == HAS_INCLUDE)); - maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location); } else if (rawtok->str() == IFNDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) == macros.end() && !(hasInclude && rawtok->next->str() == HAS_INCLUDE)); - maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location); } else { /*if (rawtok->str() == IF || rawtok->str() == ELIF)*/ TokenList expr(files); for (const Token *tok = rawtok->next; tok && tok->location.sameline(rawtok->location); tok = tok->next) { @@ -3613,7 +3613,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const bool par = (tok && tok->op == '('); if (par) tok = tok->next; - maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location); if (tok) { if (macros.find(tok->str()) != macros.end()) expr.push_back(new Token("1", tok->location)); @@ -3631,7 +3631,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } output.clear(); return; @@ -3674,7 +3674,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } output.clear(); return; @@ -3682,7 +3682,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } - maybeUsedMacros[rawtok->next->str()].push_back(rawtok->next->location); + maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location); const Token *tmp = tok; if (!preprocessToken(expr, tmp, macros, files, outputList)) { @@ -3715,7 +3715,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL rawtok->location, std::move(msg) }; - outputList->push_back(std::move(out)); + outputList->emplace_back(std::move(out)); } output.clear(); return; @@ -3814,7 +3814,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL mu.macroName = macro.name(); mu.macroLocation = macro.defineLocation(); mu.useLocation = *usageIt; - macroUsage->push_back(std::move(mu)); + macroUsage->emplace_back(std::move(mu)); } } } diff --git a/test.cpp b/test.cpp index 9583468f..ef7249e7 100644 --- a/test.cpp +++ b/test.cpp @@ -1607,7 +1607,7 @@ static void has_include_1() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.includePaths.push_back(testSourceDir); + dui.includePaths.emplace_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++14"; ASSERT_EQUALS("", preprocess(code, dui)); @@ -1628,7 +1628,7 @@ static void has_include_2() "#endif"; simplecpp::DUI dui; dui.removeComments = true; // TODO: remove this - dui.includePaths.push_back(testSourceDir); + dui.includePaths.emplace_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++14"; ASSERT_EQUALS("", preprocess(code, dui)); @@ -1657,7 +1657,7 @@ static void has_include_3() ASSERT_EQUALS("\n\n\n\nB", preprocess(code, dui)); // Unless -I is set (preferably, we should differentiate -I and -isystem...) - dui.includePaths.push_back(testSourceDir + "/testsuite"); + dui.includePaths.emplace_back(testSourceDir + "/testsuite"); dui.std = ""; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++14"; @@ -1678,7 +1678,7 @@ static void has_include_4() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.includePaths.push_back(testSourceDir); // we default to latest standard internally + dui.includePaths.emplace_back(testSourceDir); // we default to latest standard internally ASSERT_EQUALS("\n\nA", preprocess(code, dui)); dui.std = "c++14"; ASSERT_EQUALS("", preprocess(code, dui)); @@ -1699,7 +1699,7 @@ static void has_include_5() "#endif"; simplecpp::DUI dui; ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally - dui.includePaths.push_back(testSourceDir); + dui.includePaths.emplace_back(testSourceDir); dui.std = "c++14"; ASSERT_EQUALS("", preprocess(code, dui)); dui.std = "c++17"; @@ -1718,7 +1718,7 @@ static void has_include_6() " #endif\n" "#endif"; simplecpp::DUI dui; - dui.includePaths.push_back(testSourceDir); + dui.includePaths.emplace_back(testSourceDir); ASSERT_EQUALS("\n\nA", preprocess(code, dui)); // we default to latest standard internally dui.std = "c++99"; ASSERT_EQUALS("", preprocess(code, dui)); From 1453e4e738e9f4159165893c881380bba975c29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 19 Jan 2026 17:31:29 +0100 Subject: [PATCH 362/381] use initializer lists without assignment (#624) --- simplecpp.cpp | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1d14a06c..9371eaf6 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -620,7 +620,7 @@ static void portabilityBackslash(simplecpp::OutputList *outputList, const simple { if (!outputList) return; - simplecpp::Output err = { + simplecpp::Output err{ simplecpp::Output::PORTABILITY_BACKSLASH, location, "Combination 'backslash space newline' is not portable." @@ -669,7 +669,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (ch >= 0x80) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ simplecpp::Output::UNHANDLED_CHAR_ERROR, location, "The code contains unhandled character(s) (character code=" + std::to_string(static_cast(ch)) + "). Neither unicode nor extended ascii is supported." @@ -871,7 +871,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, } if (!stream.good() || ch == '\n') { if (outputList) { - Output err = { + Output err{ Output::SYNTAX_ERROR, location, "Invalid newline in raw string delimiter." @@ -885,7 +885,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, currentToken += stream.readChar(); if (!endsWith(currentToken, endOfRawString)) { if (outputList) { - Output err = { + Output err{ Output::SYNTAX_ERROR, location, "Raw string missing terminating delimiter." @@ -1429,7 +1429,7 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca if (!stream.good() || ch != end) { clear(); if (outputList) { - Output err = { + Output err{ Output::SYNTAX_ERROR, location, std::string("No pair for character (") + start + "). Can't process file. File is either invalid or unicode, which is currently not supported." @@ -2692,7 +2692,7 @@ static void simplifyName(simplecpp::TokenList &expr) { for (simplecpp::Token *tok = expr.front(); tok; tok = tok->next) { if (tok->name) { - static const std::set altop = {"and","or","bitand","bitor","compl","not","not_eq","xor"}; + static const std::set altop{"and","or","bitand","bitor","compl","not","not_eq","xor"}; if (altop.find(tok->str()) != altop.end()) { bool alt; if (tok->str() == "not" || tok->str() == "compl") { @@ -3178,7 +3178,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens, if (filedata == nullptr) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND, {}, "Can not open include file '" + filename + "' that is explicitly included." @@ -3252,7 +3252,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token tok1 = it->second.expand(value, tok, macros, files); } catch (const simplecpp::Macro::Error &err) { if (outputList) { - simplecpp::Output out = { + simplecpp::Output out{ simplecpp::Output::SYNTAX_ERROR, err.location, "failed to expand \'" + tok->str() + "\', " + err.what @@ -3347,7 +3347,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::pair(macro.name(), macro)); } catch (const std::runtime_error& e) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ Output::DUI_ERROR, {}, e.what() @@ -3366,7 +3366,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__FILE__", Macro("__FILE__", "__FILE__", dummy))); macros.insert(std::make_pair("__LINE__", Macro("__LINE__", "__LINE__", dummy))); macros.insert(std::make_pair("__COUNTER__", Macro("__COUNTER__", "__COUNTER__", dummy))); - struct tm ltime = {}; + struct tm ltime {}; getLocaltime(ltime); macros.insert(std::make_pair("__DATE__", Macro("__DATE__", getDateDefine(<ime), dummy))); macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), dummy))); @@ -3381,7 +3381,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const cppstd_t cpp_std = simplecpp::getCppStd(dui.std); if (cpp_std == CPPUnknown) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ Output::DUI_ERROR, {}, "unknown standard specified: '" + dui.std + "'" @@ -3438,7 +3438,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (ifstates.size() <= 1U && (rawtok->str() == ELIF || rawtok->str() == ELSE || rawtok->str() == ENDIF)) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ Output::SYNTAX_ERROR, rawtok->location, "#" + rawtok->str() + " without #if" @@ -3458,7 +3458,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL msg += tok->str(); } msg = '#' + rawtok->str() + ' ' + msg; - simplecpp::Output err = { + simplecpp::Output err{ rawtok->str() == ERROR ? Output::ERROR : Output::WARNING, rawtok->location, std::move(msg) @@ -3486,7 +3486,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } catch (const std::runtime_error &) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ Output::SYNTAX_ERROR, rawtok->location, "Failed to parse #define" @@ -3497,7 +3497,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL return; } catch (const simplecpp::Macro::Error &err) { if (outputList) { - simplecpp::Output out = { + simplecpp::Output out{ simplecpp::Output::SYNTAX_ERROR, err.location, "Failed to parse #define, " + err.what @@ -3538,7 +3538,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL if (inc2.empty() || inc2.cfront()->str().size() <= 2U) { if (outputList) { - simplecpp::Output err = { + simplecpp::Output err{ Output::SYNTAX_ERROR, rawtok->location, "No header in #include" @@ -3556,7 +3556,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL const FileData *const filedata = cache.get(rawtokens.file(rawtok->location), header, dui, systemheader, files, outputList).first; if (filedata == nullptr) { if (outputList) { - simplecpp::Output out = { + simplecpp::Output out{ simplecpp::Output::MISSING_HEADER, rawtok->location, "Header not found: " + inctok->str() @@ -3565,7 +3565,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } } else if (includetokenstack.size() >= 400) { if (outputList) { - simplecpp::Output out = { + simplecpp::Output out{ simplecpp::Output::INCLUDE_NESTED_TOO_DEEPLY, rawtok->location, "#include nested too deeply" @@ -3580,7 +3580,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } else if (rawtok->str() == IF || rawtok->str() == IFDEF || rawtok->str() == IFNDEF || rawtok->str() == ELIF) { if (!sameline(rawtok,rawtok->next)) { if (outputList) { - simplecpp::Output out = { + simplecpp::Output out{ simplecpp::Output::SYNTAX_ERROR, rawtok->location, "Syntax error in #" + rawtok->str() @@ -3626,7 +3626,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')')) { if (outputList) { - Output out = { + Output out{ Output::SYNTAX_ERROR, rawtok->location, "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" @@ -3669,7 +3669,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL tok = tok ? tok->next : nullptr; if (!tok || !sameline(rawtok,tok) || (par && tok->op != ')') || (!closingAngularBracket)) { if (outputList) { - Output out = { + Output out{ Output::SYNTAX_ERROR, rawtok->location, "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition" @@ -3710,7 +3710,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::string msg = "failed to evaluate " + std::string(rawtok->str() == IF ? "#if" : "#elif") + " condition"; if (e.what() && *e.what()) msg += std::string(", ") + e.what(); - Output out = { + Output out{ Output::SYNTAX_ERROR, rawtok->location, std::move(msg) From 814942b380d25437cbf69fd5b4addd1a49268309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Mon, 19 Jan 2026 17:31:43 +0100 Subject: [PATCH 363/381] test.cpp: improved testing of `__FILE__` (#623) --- test.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index ef7249e7..a14f588e 100644 --- a/test.cpp +++ b/test.cpp @@ -102,11 +102,11 @@ static std::string readfile(const char code[], std::size_t size, simplecpp::Outp return makeTokenList(code,size,files,std::string(),outputList).stringify(); } -static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList) +static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList, const std::string &file = std::string()) { std::vector files; simplecpp::FileDataCache cache; - simplecpp::TokenList tokens = makeTokenList(code,files); + simplecpp::TokenList tokens = makeTokenList(code,files, file); if (dui.removeComments) tokens.removeComments(); simplecpp::TokenList tokens2(files); @@ -120,6 +120,11 @@ static std::string preprocess(const char code[]) return preprocess(code, simplecpp::DUI(), nullptr); } +static std::string preprocess(const char code[], const std::string &file) +{ + return preprocess(code, simplecpp::DUI(), nullptr, file); +} + static std::string preprocess(const char code[], const simplecpp::DUI &dui) { return preprocess(code, dui, nullptr); @@ -193,7 +198,7 @@ static void backslash() static void builtin() { - ASSERT_EQUALS("\"\" 1 0", preprocess("__FILE__ __LINE__ __COUNTER__")); + ASSERT_EQUALS("\"test.c\" 1 0", preprocess("__FILE__ __LINE__ __COUNTER__", "test.c")); ASSERT_EQUALS("\n\n3", preprocess("\n\n__LINE__")); ASSERT_EQUALS("\n\n0", preprocess("\n\n__COUNTER__")); ASSERT_EQUALS("\n\n0 1", preprocess("\n\n__COUNTER__ __COUNTER__")); From 4c068867b2519ff03d65334230da690dd8355fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 21 Jan 2026 23:46:34 +0100 Subject: [PATCH 364/381] make it possible to explicitly disable the legacy `TokenList` constructors (#621) --- main.cpp | 1 + simplecpp.h | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/main.cpp b/main.cpp index 62ccc022..d67f94c2 100644 --- a/main.cpp +++ b/main.cpp @@ -3,6 +3,7 @@ * Copyright (C) 2016-2023 simplecpp team */ +#define SIMPLECPP_TOKENLIST_ALLOW_PTR 0 #include "simplecpp.h" #include diff --git a/simplecpp.h b/simplecpp.h index 46a43bc4..54f6a90c 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -57,7 +57,9 @@ #ifndef SIMPLECPP_TOKENLIST_ALLOW_PTR // still provide the legacy API in case we lack the performant wrappers # if !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span) -# define SIMPLECPP_TOKENLIST_ALLOW_PTR +# define SIMPLECPP_TOKENLIST_ALLOW_PTR 1 +# else +# define SIMPLECPP_TOKENLIST_ALLOW_PTR 0 # endif #endif @@ -267,7 +269,7 @@ namespace simplecpp { TokenList(const unsigned char (&data)[size], std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(data, size-1, filenames, filename, outputList, 0) {} -#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR +#if SIMPLECPP_TOKENLIST_ALLOW_PTR /** generates a token list from the given buffer */ TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr) : TokenList(data, size, filenames, filename, outputList, 0) From 863489a2ceb32f41776ea1c88a5e745ca4254d00 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:57:53 +0100 Subject: [PATCH 365/381] Fix #615 User-defined literal created from alternative `and` (#620) --- simplecpp.cpp | 13 +++++++++---- test.cpp | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 9371eaf6..6f929fa8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1005,6 +1005,14 @@ static bool isFloatSuffix(const simplecpp::Token *tok) return c == 'f' || c == 'l'; } +static const std::string AND("and"); +static const std::string BITAND("bitand"); +static const std::string BITOR("bitor"); +static bool isAlternativeAndBitandBitor(const simplecpp::Token* tok) +{ + return isAlternativeBinaryOp(tok, AND) || isAlternativeBinaryOp(tok, BITAND) || isAlternativeBinaryOp(tok, BITOR); +} + void simplecpp::TokenList::combineOperators() { std::stack executableScope; @@ -1040,7 +1048,7 @@ void simplecpp::TokenList::combineOperators() if (tok->previous && tok->previous->number && sameline(tok->previous, tok) && tok->previous->str().find_first_of("._") == std::string::npos) { tok->setstr(tok->previous->str() + '.'); deleteToken(tok->previous); - if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp")))) { + if (sameline(tok, tok->next) && (isFloatSuffix(tok->next) || (tok->next && tok->next->startsWithOneOf("AaBbCcDdEeFfPp") && !isAlternativeAndBitandBitor(tok->next)))) { tok->setstr(tok->str() + tok->next->str()); deleteToken(tok->next); } @@ -1285,8 +1293,6 @@ void simplecpp::TokenList::constFoldComparison(Token *tok) } } -static const std::string BITAND("bitand"); -static const std::string BITOR("bitor"); static const std::string XOR("xor"); void simplecpp::TokenList::constFoldBitwise(Token *tok) { @@ -1321,7 +1327,6 @@ void simplecpp::TokenList::constFoldBitwise(Token *tok) } } -static const std::string AND("and"); static const std::string OR("or"); void simplecpp::TokenList::constFoldLogicalOp(Token *tok) { diff --git a/test.cpp b/test.cpp index a14f588e..26a2fb49 100644 --- a/test.cpp +++ b/test.cpp @@ -433,6 +433,7 @@ static void combineOperators_floatliteral() ASSERT_EQUALS("1p + 3", preprocess("1p+3")); ASSERT_EQUALS("1.0_a . b", preprocess("1.0_a.b")); ASSERT_EQUALS("1_a . b", preprocess("1_a.b")); + ASSERT_EQUALS("bool x = d != 0. and b ;", preprocess("bool x = d != 0. and b;")); } static void combineOperators_increment() From d3cc78d0aa27a07d1189c9bc43d2bcac95ae5dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 28 Jan 2026 19:28:02 +0100 Subject: [PATCH 366/381] clang-tidy.yml: updated to Clang 22 (#514) --- .clang-tidy | 4 +- .github/workflows/clang-tidy.yml | 10 +- simplecpp.cpp | 263 +++++++++++++++++-------------- test.cpp | 3 +- 4 files changed, 152 insertions(+), 128 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index d16cc35c..e0b384bf 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -4,6 +4,7 @@ Checks: > -abseil-*, -altera-*, -android-*, + -boost-*, -cert-*, -clang-analyzer-*, -cppcoreguidelines-*, @@ -17,11 +18,12 @@ Checks: > -objc-*, -openmp-*, -zircon-*, - -boost-use-ranges, -bugprone-branch-clone, -bugprone-easily-swappable-parameters, -bugprone-narrowing-conversions, -bugprone-switch-missing-default-case, + -bugprone-throwing-static-initialization, + -bugprone-unchecked-string-to-number-conversion, -concurrency-mt-unsafe, -misc-no-recursion, -misc-non-private-member-variables-in-classes, diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 9109be9e..b2b32b7d 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -33,19 +33,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 21 - sudo apt-get install clang-tidy-21 + sudo ./llvm.sh 22 + sudo apt-get install clang-tidy-22 - name: Verify clang-tidy configuration run: | - clang-tidy-21 --verify-config + clang-tidy-22 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: - CXX: clang-21 + CXX: clang-22 - name: Clang-Tidy run: | - run-clang-tidy-21 -q -j $(nproc) -p=cmake.output + run-clang-tidy-22 -q -j $(nproc) -enable-check-profile -p=cmake.output diff --git a/simplecpp.cpp b/simplecpp.cpp index 6f929fa8..8263ddb3 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -351,121 +351,124 @@ class simplecpp::TokenList::Stream { bool isUtf16; }; -class StdIStream : public simplecpp::TokenList::Stream { -public: - // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - explicit StdIStream(std::istream &istr) - : istr(istr) { - assert(istr.good()); - init(); - } +namespace { + class StdIStream : public simplecpp::TokenList::Stream { + public: + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + explicit StdIStream(std::istream &istr) + : istr(istr) { + assert(istr.good()); + init(); + } - int get() override { - return istr.get(); - } - int peek() override { - return istr.peek(); - } - void unget() override { - istr.unget(); - } - bool good() override { - return istr.good(); - } + int get() override { + return istr.get(); + } + int peek() override { + return istr.peek(); + } + void unget() override { + istr.unget(); + } + bool good() override { + return istr.good(); + } -private: - std::istream &istr; -}; + private: + std::istream &istr; + }; -class StdCharBufStream : public simplecpp::TokenList::Stream { -public: - // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - StdCharBufStream(const unsigned char* str, std::size_t size) - : str(str) - , size(size) - { - init(); - } + class StdCharBufStream : public simplecpp::TokenList::Stream { + public: + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + StdCharBufStream(const unsigned char* str, std::size_t size) + : str(str) + , size(size) + { + init(); + } - int get() override { - if (pos >= size) - return lastStatus = EOF; - return str[pos++]; - } - int peek() override { - if (pos >= size) - return lastStatus = EOF; - return str[pos]; - } - void unget() override { - --pos; - } - bool good() override { - return lastStatus != EOF; - } + int get() override { + if (pos >= size) + return lastStatus = EOF; + return str[pos++]; + } + int peek() override { + if (pos >= size) + return lastStatus = EOF; + return str[pos]; + } + void unget() override { + --pos; + } + bool good() override { + return lastStatus != EOF; + } -private: - const unsigned char *str; - const std::size_t size; - std::size_t pos{}; - int lastStatus{}; -}; + private: + const unsigned char *str; + const std::size_t size; + std::size_t pos{}; + int lastStatus{}; + }; -class FileStream : public simplecpp::TokenList::Stream { -public: - /** - * @throws simplecpp::Output thrown if file is not found - */ - // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members - explicit FileStream(const std::string &filename, std::vector &files) - : file(fopen(filename.c_str(), "rb")) - { - if (!file) { - files.emplace_back(filename); - throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, {}, "File is missing: " + filename); + class FileStream : public simplecpp::TokenList::Stream { + public: + /** + * @throws simplecpp::Output thrown if file is not found + */ + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + explicit FileStream(const std::string &filename, std::vector &files) + : file(fopen(filename.c_str(), "rb")) + { + if (!file) { + files.emplace_back(filename); + throw simplecpp::Output(simplecpp::Output::FILE_NOT_FOUND, {}, "File is missing: " + filename); + } + init(); } - init(); - } - FileStream(const FileStream&) = delete; - FileStream &operator=(const FileStream&) = delete; + FileStream(const FileStream&) = delete; + FileStream &operator=(const FileStream&) = delete; - ~FileStream() override { - fclose(file); - file = nullptr; - } + ~FileStream() override { + fclose(file); + file = nullptr; + } - int get() override { - lastStatus = lastCh = fgetc(file); - return lastCh; - } - int peek() override { - // keep lastCh intact - const int ch = fgetc(file); - unget_internal(ch); - return ch; - } - void unget() override { - unget_internal(lastCh); - } - bool good() override { - return lastStatus != EOF; - } + int get() override { + lastStatus = lastCh = fgetc(file); + return lastCh; + } + int peek() override { + // keep lastCh intact + const int ch = fgetc(file); + unget_internal(ch); + return ch; + } + void unget() override { + unget_internal(lastCh); + } + bool good() override { + return lastStatus != EOF; + } -private: - void unget_internal(int ch) { - if (isUtf16) { - // TODO: use ungetc() as well - // UTF-16 has subsequent unget() calls - fseek(file, -1, SEEK_CUR); - } else - ungetc(ch, file); - } + private: + void unget_internal(int ch) { + if (isUtf16) { + // TODO: use ungetc() as well + // UTF-16 has subsequent unget() calls + fseek(file, -1, SEEK_CUR); + } else { + ungetc(ch, file); + } + } - FILE *file; - int lastCh{}; - int lastStatus{}; -}; + FILE *file; + int lastCh{}; + int lastStatus{}; + }; +} simplecpp::TokenList::TokenList(std::vector &filenames) : frontToken(nullptr), backToken(nullptr), files(filenames) {} @@ -1187,8 +1190,9 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) continue; long long result; - if (tok->op == '*') + if (tok->op == '*') { result = (stringToLL(tok->previous->str()) * stringToLL(tok->next->str())); + } else if (tok->op == '/' || tok->op == '%') { const long long rhs = stringToLL(tok->next->str()); if (rhs == 0) @@ -1200,8 +1204,9 @@ void simplecpp::TokenList::constFoldMulDivRem(Token *tok) result = (lhs / rhs); else result = (lhs % rhs); - } else + } else { continue; + } tok = tok->previous; tok->setstr(toString(result)); @@ -1422,8 +1427,9 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca ret.erase(ret.size()-1U); backslash = (next == '\r'); update_ch = false; - } else if (next == '\\') + } else if (next == '\\') { update_ch = !update_ch; + } ret += next; } while (next == '\\'); if (update_ch) @@ -1544,8 +1550,9 @@ namespace simplecpp { if (this != &other) { files = other.files; valueDefinedInCode_ = other.valueDefinedInCode_; - if (other.tokenListDefine.empty()) + if (other.tokenListDefine.empty()) { parseDefine(other.nameTokDef); + } else { tokenListDefine = other.tokenListDefine; parseDefine(tokenListDefine.cfront()); @@ -1614,15 +1621,17 @@ namespace simplecpp { if (par==0) break; --par; - } else if (macro2tok->op == ')') + } else if (macro2tok->op == ')') { ++par; + } macro2tok = macro2tok->previous; } if (macro2tok) { // macro2tok->op == '(' macro2tok = macro2tok->previous; expandedmacros.insert(name()); - } else if (rawtok->op == '(') + } else if (rawtok->op == '(') { macro2tok = output2.back(); + } if (!macro2tok || !macro2tok->name) break; if (output2.cfront() != output2.cback() && macro2tok->str() == this->name()) @@ -1642,8 +1651,9 @@ namespace simplecpp { const Token *rawtok2 = rawtok; for (; rawtok2; rawtok2 = rawtok2->next) { rawtokens2.push_back(new Token(rawtok2->str(), loc)); - if (rawtok2->op == '(') + if (rawtok2->op == '(') { ++par; + } else if (rawtok2->op == ')') { if (par <= 1U) break; @@ -1840,16 +1850,18 @@ namespace simplecpp { parametertokens.emplace_back(nameTokInst->next); unsigned int par = 0U; for (const Token *tok = nameTokInst->next->next; calledInDefine ? sameline(tok, nameTokInst) : (tok != nullptr); tok = tok->next) { - if (tok->op == '(') + if (tok->op == '(') { ++par; + } else if (tok->op == ')') { if (par == 0U) { parametertokens.emplace_back(tok); break; } --par; - } else if (par == 0U && tok->op == ',' && (!variadic || parametertokens.size() < args.size())) + } else if (par == 0U && tok->op == ',' && (!variadic || parametertokens.size() < args.size())) { parametertokens.emplace_back(tok); + } } return parametertokens; } @@ -1877,8 +1889,9 @@ namespace simplecpp { tokens.back()->macro = name(); } - if (tok->op == '(') + if (tok->op == '(') { ++par; + } else if (tok->op == ')') { --par; if (par == 0U) @@ -1951,8 +1964,9 @@ namespace simplecpp { const MacroMap::const_iterator m = macros.find("__COUNTER__"); - if (!counter || m == macros.end()) + if (!counter || m == macros.end()) { parametertokens2.swap(parametertokens1); + } else { const Macro &counterMacro = m->second; unsigned int par = 0; @@ -2141,8 +2155,9 @@ namespace simplecpp { TokenList tokens(files); tokens.push_back(new Token(*tok)); const Token * tok2 = nullptr; - if (tok->next->op == '(') + if (tok->next->op == '(') { tok2 = appendTokens(tokens, loc, tok->next, macros, expandedmacros, parametertokens); + } else if (expandArg(tokens, tok->next, loc, macros, expandedmacros, parametertokens)) { tokens.front()->location = loc; if (tokens.cfront()->next && tokens.cfront()->next->op == '(') @@ -2318,12 +2333,15 @@ namespace simplecpp { const bool varargs = variadic && !args.empty() && B->str() == args[args.size()-1U]; if (expandArg(tokensB, B, parametertokens)) { - if (tokensB.empty()) + if (tokensB.empty()) { strAB = A->str(); - else if (varargs && A->op == ',') + } + else if (varargs && A->op == ',') { strAB = ","; - else if (varargs && unexpectedA) + } + else if (varargs && unexpectedA) { throw invalidHashHash::unexpectedToken(tok->location, name(), A); + } else { strAB = A->str() + tokensB.cfront()->str(); tokensB.deleteToken(tokensB.front()); @@ -2342,8 +2360,9 @@ namespace simplecpp { throw invalidHashHash::universalCharacterUB(tok->location, name(), A, strAB); } - if (varargs && tokensB.empty() && tok->previous->str() == ",") + if (varargs && tokensB.empty() && tok->previous->str() == ",") { output.deleteToken(A); + } else if (strAB != "," && macros.find(strAB) == macros.end()) { A->setstr(strAB); for (Token *b = tokensB.front(); b; b = b->next) @@ -2761,8 +2780,9 @@ long long simplecpp::characterLiteralToLL(const std::string& str) pos = 3; } else if (str.size() >= 2 && (str[0] == 'L' || str[0] == 'U') && str[1] == '\'') { pos = 2; - } else + } else { throw std::runtime_error("expected a character literal"); + } unsigned long long multivalue = 0; @@ -3597,8 +3617,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } bool conditionIsTrue; - if (ifstates.top() == AlwaysFalse || (ifstates.top() == ElseIsTrue && rawtok->str() != ELIF)) + if (ifstates.top() == AlwaysFalse || (ifstates.top() == ElseIsTrue && rawtok->str() != ELIF)) { conditionIsTrue = false; + } else if (rawtok->str() == IFDEF) { conditionIsTrue = (macros.find(rawtok->next->str()) != macros.end() || (hasInclude && rawtok->next->str() == HAS_INCLUDE)); maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location); diff --git a/test.cpp b/test.cpp index 26a2fb49..fa94a266 100644 --- a/test.cpp +++ b/test.cpp @@ -67,8 +67,9 @@ static void assertThrowFailed(int line) static void testcase(const std::string &name, void (*f)(), int argc, char * const *argv) { - if (argc == 1) + if (argc == 1) { f(); + } else { for (int i = 1; i < argc; i++) { if (name == argv[i]) From 5e00b6083ae36b0e156b2eb6e8dc2e7fe225716f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 10 Feb 2026 14:39:09 +0100 Subject: [PATCH 367/381] removed workarounds for Visual Studio conflicts with Cppcheck functions with same names (#627) --- simplecpp.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 8263ddb3..7afc17ab 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -67,14 +67,12 @@ static bool isOct(const std::string &s) return s.size()>1 && (s[0]=='0') && (s[1] >= '0') && (s[1] < '8'); } -// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild -static bool isStringLiteral_(const std::string &s) +static bool isStringLiteral(const std::string &s) { return s.size() > 1 && (s[0]=='\"') && (*s.rbegin()=='\"'); } -// TODO: added an undercore since this conflicts with a function of the same name in utils.h from Cppcheck source when building Cppcheck with MSBuild -static bool isCharLiteral_(const std::string &s) +static bool isCharLiteral(const std::string &s) { // char literal patterns can include 'a', '\t', '\000', '\xff', 'abcd', and maybe '' // This only checks for the surrounding '' but doesn't parse the content. @@ -2295,7 +2293,7 @@ namespace simplecpp { throw invalidHashHash::unexpectedNewline(tok->location, name()); const bool canBeConcatenatedWithEqual = A->isOneOf("+-*/%&|^") || A->str() == "<<" || A->str() == ">>"; - const bool canBeConcatenatedStringOrChar = isStringLiteral_(A->str()) || isCharLiteral_(A->str()); + const bool canBeConcatenatedStringOrChar = isStringLiteral(A->str()) || isCharLiteral(A->str()); const bool unexpectedA = (!A->name && !A->number && !A->str().empty() && !canBeConcatenatedWithEqual && !canBeConcatenatedStringOrChar); const Token * const B = tok->next->next; From 8a30993ececc91d9540832dd70e943727dd0050e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Tue, 10 Mar 2026 10:36:31 +0100 Subject: [PATCH 368/381] reverted to Clang 21 for now (#630) --- .clang-tidy | 2 -- .github/workflows/clang-tidy.yml | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index e0b384bf..ba15bb0f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,8 +22,6 @@ Checks: > -bugprone-easily-swappable-parameters, -bugprone-narrowing-conversions, -bugprone-switch-missing-default-case, - -bugprone-throwing-static-initialization, - -bugprone-unchecked-string-to-number-conversion, -concurrency-mt-unsafe, -misc-no-recursion, -misc-non-private-member-variables-in-classes, diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index b2b32b7d..9109be9e 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -33,19 +33,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 22 - sudo apt-get install clang-tidy-22 + sudo ./llvm.sh 21 + sudo apt-get install clang-tidy-21 - name: Verify clang-tidy configuration run: | - clang-tidy-22 --verify-config + clang-tidy-21 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: - CXX: clang-22 + CXX: clang-21 - name: Clang-Tidy run: | - run-clang-tidy-22 -q -j $(nproc) -enable-check-profile -p=cmake.output + run-clang-tidy-21 -q -j $(nproc) -p=cmake.output From efb55b81fd6addfd23f608cce33f0eef02960752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 15 Mar 2026 18:45:54 +0100 Subject: [PATCH 369/381] clang-tidy.yml: updated to Clang 22 - again (#633) --- .clang-tidy | 2 ++ .github/workflows/clang-tidy.yml | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index ba15bb0f..e0b384bf 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,6 +22,8 @@ Checks: > -bugprone-easily-swappable-parameters, -bugprone-narrowing-conversions, -bugprone-switch-missing-default-case, + -bugprone-throwing-static-initialization, + -bugprone-unchecked-string-to-number-conversion, -concurrency-mt-unsafe, -misc-no-recursion, -misc-non-private-member-variables-in-classes, diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 9109be9e..b2b32b7d 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -33,19 +33,19 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 21 - sudo apt-get install clang-tidy-21 + sudo ./llvm.sh 22 + sudo apt-get install clang-tidy-22 - name: Verify clang-tidy configuration run: | - clang-tidy-21 --verify-config + clang-tidy-22 --verify-config - name: Prepare CMake run: | cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: - CXX: clang-21 + CXX: clang-22 - name: Clang-Tidy run: | - run-clang-tidy-21 -q -j $(nproc) -p=cmake.output + run-clang-tidy-22 -q -j $(nproc) -enable-check-profile -p=cmake.output From ae1f0fbdfc626548398e80676928d5bfd01e0688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 15 Mar 2026 20:06:17 +0100 Subject: [PATCH 370/381] added basic test coverage for `IfCond` and `MacroUsage` (#629) --- test.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index fa94a266..1dd59587 100644 --- a/test.cpp +++ b/test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -103,7 +104,7 @@ static std::string readfile(const char code[], std::size_t size, simplecpp::Outp return makeTokenList(code,size,files,std::string(),outputList).stringify(); } -static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList, const std::string &file = std::string()) +static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list *macroUsage = nullptr, std::list *ifCond = nullptr, const std::string &file = std::string()) { std::vector files; simplecpp::FileDataCache cache; @@ -111,7 +112,7 @@ static std::string preprocess(const char code[], const simplecpp::DUI &dui, simp if (dui.removeComments) tokens.removeComments(); simplecpp::TokenList tokens2(files); - simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList); + simplecpp::preprocess(tokens2, tokens, files, cache, dui, outputList, macroUsage, ifCond); simplecpp::cleanup(cache); return tokens2.stringify(); } @@ -123,7 +124,7 @@ static std::string preprocess(const char code[]) static std::string preprocess(const char code[], const std::string &file) { - return preprocess(code, simplecpp::DUI(), nullptr, file); + return preprocess(code, simplecpp::DUI(), nullptr, nullptr, nullptr, file); } static std::string preprocess(const char code[], const simplecpp::DUI &dui) @@ -136,6 +137,16 @@ static std::string preprocess(const char code[], simplecpp::OutputList *outputLi return preprocess(code, simplecpp::DUI(), outputList); } +static std::string preprocess(const char code[], std::list *ifCond) +{ + return preprocess(code, simplecpp::DUI(), nullptr, nullptr, ifCond); +} + +static std::string preprocess(const char code[], std::list *macroUsage) +{ + return preprocess(code, simplecpp::DUI(), nullptr, macroUsage); +} + static std::string toString(const simplecpp::OutputList &outputList) { std::ostringstream ostr; @@ -3461,6 +3472,70 @@ static void bad_macro_syntax() // #616 ASSERT_EQUALS("bad macro syntax. macroname=\" value=1", outputList.cbegin()->msg); } +static void ifCond() +{ + { + const char code[] = "int i;"; + std::list ifCond; + ASSERT_EQUALS("int i ;", preprocess(code, &ifCond)); + ASSERT_EQUALS(0, ifCond.size()); + } + { + const char code[] = "#if 0\n" + "# elif __GNUC__ == 1\n" + "# elif defined(__APPLE__)\n" + "#endif\n"; + std::list ifCond; + ASSERT_EQUALS("", preprocess(code, &ifCond)); + ASSERT_EQUALS(3, ifCond.size()); + auto it = ifCond.cbegin(); + ASSERT_EQUALS(0, it->location.fileIndex); + ASSERT_EQUALS(1, it->location.line); + ASSERT_EQUALS(2, it->location.col); + ASSERT_EQUALS("0", it->E); + ASSERT_EQUALS(0, it->result); + ++it; + ASSERT_EQUALS(0, it->location.fileIndex); + ASSERT_EQUALS(2, it->location.line); + ASSERT_EQUALS(3, it->location.col); + ASSERT_EQUALS("__GNUC__ == 1", it->E); + ASSERT_EQUALS(0, it->result); + ++it; + ASSERT_EQUALS(0, it->location.fileIndex); + ASSERT_EQUALS(3, it->location.line); + ASSERT_EQUALS(4, it->location.col); + ASSERT_EQUALS("0", it->E); + ASSERT_EQUALS(0, it->result); + } +} + +static void macroUsage() +{ + { + const char code[] = "int i;"; + std::list macroUsage; + ASSERT_EQUALS("int i ;", preprocess(code, ¯oUsage)); + ASSERT_EQUALS(0, macroUsage.size()); + } + { + const char code[] = "#define DEF_1\n" + "#ifdef DEF_1\n" + "#endif\n"; + std::list macroUsage; + ASSERT_EQUALS("", preprocess(code, ¯oUsage)); + ASSERT_EQUALS(1, macroUsage.size()); + auto it = macroUsage.cbegin(); + ASSERT_EQUALS("DEF_1", it->macroName); + ASSERT_EQUALS(0, it->macroLocation.fileIndex); + ASSERT_EQUALS(1, it->macroLocation.line); + ASSERT_EQUALS(9, it->macroLocation.col); + ASSERT_EQUALS(true, it->macroValueKnown); + ASSERT_EQUALS(0, it->useLocation.fileIndex); + ASSERT_EQUALS(2, it->useLocation.line); + ASSERT_EQUALS(8, it->useLocation.col); + } +} + static void isAbsolutePath() { #ifdef _WIN32 ASSERT_EQUALS(true, simplecpp::isAbsolutePath("C:\\foo\\bar")); @@ -3790,6 +3865,9 @@ int main(int argc, char **argv) TEST_CASE(bad_macro_syntax); + TEST_CASE(ifCond); + TEST_CASE(macroUsage); + TEST_CASE(fuzz_crash); TEST_CASE(leak); From f437d61a26ccf3a7163456d14a507db6b1de5e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 27 Mar 2026 18:16:06 +0100 Subject: [PATCH 371/381] fixed #632 - avoid unhandled `simplecpp::Macro::Error` in `simplecpp::preprocess` with `-D` (#631) --- simplecpp.cpp | 22 ++++++++++++++++++---- test.cpp | 14 ++++++++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 7afc17ab..b59f773c 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1739,6 +1739,9 @@ namespace simplecpp { return tok; } + /** + * @throws Error thrown in case of __VA_OPT__ issues + */ bool parseDefine(const Token *nametoken) { nameTokDef = nametoken; variadic = false; @@ -3379,6 +3382,17 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL } output.clear(); return; + } catch (const simplecpp::Macro::Error& e) { + if (outputList) { + simplecpp::Output err{ + Output::DUI_ERROR, + {}, + e.what + }; + outputList->emplace_back(std::move(err)); + } + output.clear(); + return; } } @@ -3507,14 +3521,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL else it->second = macro; } - } catch (const std::runtime_error &) { + } catch (const std::runtime_error &err) { if (outputList) { - simplecpp::Output err{ + simplecpp::Output out{ Output::SYNTAX_ERROR, rawtok->location, - "Failed to parse #define" + std::string("Failed to parse #define, ") + err.what() }; - outputList->emplace_back(std::move(err)); + outputList->emplace_back(std::move(out)); } output.clear(); return; diff --git a/test.cpp b/test.cpp index 1dd59587..b0bf0b4f 100644 --- a/test.cpp +++ b/test.cpp @@ -698,7 +698,7 @@ static void define_invalid_1() const char code[] = "#define A(\nB\n"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, bad macro syntax\n", toString(outputList)); } static void define_invalid_2() @@ -706,7 +706,7 @@ static void define_invalid_2() const char code[] = "#define\nhas#"; simplecpp::OutputList outputList; ASSERT_EQUALS("", preprocess(code, &outputList)); - ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define\n", toString(outputList)); + ASSERT_EQUALS("file0,1,syntax_error,Failed to parse #define, bad macro syntax\n", toString(outputList)); } static void define_define_1() @@ -1105,6 +1105,15 @@ static void define_va_opt_8() ASSERT_EQUALS("", toString(outputList)); } +static void define_va_opt_9() +{ + simplecpp::DUI dui; + dui.defines.emplace_back("f(...)=__VA_OPT__"); + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess("", dui, &outputList)); + ASSERT_EQUALS("file0,0,dui_error,In definition of 'f': Missing opening parenthesis for __VA_OPT__\n", toString(outputList)); +} + static void define_ifdef() { const char code[] = "#define A(X) X\n" @@ -3674,6 +3683,7 @@ int main(int argc, char **argv) TEST_CASE(define_va_opt_6); TEST_CASE(define_va_opt_7); TEST_CASE(define_va_opt_8); + TEST_CASE(define_va_opt_9); // #632 TEST_CASE(pragma_backslash); // multiline pragma directive From 87c13d6709762fa722beea544002a4e7a44761b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 9 Apr 2026 15:33:19 +0200 Subject: [PATCH 372/381] clang-tidy.yml: run clang-tidy with C++23 (#635) --- .github/workflows/clang-tidy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index b2b32b7d..333672d7 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -42,7 +42,7 @@ jobs: - name: Prepare CMake run: | - cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=23 -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON env: CXX: clang-22 From 6521caaab5024148138c4e0954d5fc966dd11cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 9 Apr 2026 16:22:13 +0200 Subject: [PATCH 373/381] CI-mingw.yml: exclude `clang++` on `MINGW32` as it is no longer available (#644) --- .github/workflows/CI-mingw.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/CI-mingw.yml b/.github/workflows/CI-mingw.yml index a3bac5cf..390de83d 100644 --- a/.github/workflows/CI-mingw.yml +++ b/.github/workflows/CI-mingw.yml @@ -33,6 +33,9 @@ jobs: exclude: - msystem: CLANG64 compiler: g++ + # the mingw-w64-i686-clang package is no longer available + - msystem: MINGW32 + compiler: clang++ fail-fast: false runs-on: windows-2025 From 7360858b281db99cd5a7e6da6a0558ca6aa8d9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Thu, 9 Apr 2026 17:45:28 +0200 Subject: [PATCH 374/381] CI-windows.yml: updated Python to 3.14 (#642) --- .github/workflows/CI-windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 521bc24c..cfced0d2 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -32,10 +32,10 @@ jobs: - name: Setup msbuild.exe uses: microsoft/setup-msbuild@v2 - - name: Set up Python 3.13 + - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.13' + python-version: '3.14' check-latest: true - name: Install missing Python packages From 1d0c78039150157f91972b854f464347a4b9df71 Mon Sep 17 00:00:00 2001 From: chrchr-github <78114321+chrchr-github@users.noreply.github.com> Date: Fri, 10 Apr 2026 10:25:08 +0200 Subject: [PATCH 375/381] Refs #638 Fix stylistic issues uncovered by cppcheck (#643) --- simplecpp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 54f6a90c..49aa600b 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -83,7 +83,7 @@ namespace simplecpp { , mSize(strlen(data)) {} - // only provide when std::span is not available so using untyped initilization won't use View + // only provide when std::span is not available so using untyped initialization won't use View #if !defined(__cpp_lib_span) View(const char* data, std::size_t size) : mData(data) @@ -376,7 +376,7 @@ namespace simplecpp { const std::string& file(const Location& loc) const; private: - TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList, int unused); + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList, int /*unused*/); void combineOperators(); From 7e7a525ab1afb4fd7c16f357950ed1d1e213ff19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 10 Apr 2026 14:07:33 +0200 Subject: [PATCH 376/381] CI-unixish.yml: added `ubuntu-22.04-arm`, `ubuntu-24.04-arm` and `macos-26-intel` (#640) --- .github/workflows/CI-unixish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 6fd8a429..1a4ce649 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04, macos-14, macos-15, macos-15-intel, macos-26] + os: [ubuntu-22.04, ubuntu-22.04-arm, ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel] compiler: [clang++] include: - os: ubuntu-22.04 From 4044f8fcdf175814668c17ea4a768a751ba1313c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 12 Apr 2026 09:04:34 +0200 Subject: [PATCH 377/381] main.cpp: fixed handling of options without value (#645) --- main.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/main.cpp b/main.cpp index d67f94c2..5c48f830 100644 --- a/main.cpp +++ b/main.cpp @@ -86,7 +86,7 @@ int main(int argc, char **argv) break; } dui.includes.emplace_back(std::move(value)); - } else if (std::strncmp(arg, "-is",3)==0) { + } else if (std::strcmp(arg, "-is")==0) { found = true; use_istream = true; } @@ -104,20 +104,28 @@ int main(int argc, char **argv) } break; case 'q': - found = true; - quiet = true; + if (std::strcmp(arg, "-q")==0) { + found = true; + quiet = true; + } break; case 'e': - found = true; - error_only = true; + if (std::strcmp(arg, "-e")==0) { + found = true; + error_only = true; + } break; case 'f': - found = true; - fail_on_error = true; + if (std::strcmp(arg, "-f")==0) { + found = true; + fail_on_error = true; + } break; case 'l': - linenrs = true; - found = true; + if (std::strcmp(arg, "-l")==0) { + linenrs = true; + found = true; + } break; } if (!found) { From ac5cbe3408ad76c2c845c66ccbc2680e93a5c80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 13 Apr 2026 10:49:46 +0200 Subject: [PATCH 378/381] Fix #590 (Incorrect expansion of functional macros) (#646) --- simplecpp.cpp | 2 ++ simplecpp.h | 3 +++ test.cpp | 11 +++++++++++ 3 files changed, 16 insertions(+) diff --git a/simplecpp.cpp b/simplecpp.cpp index b59f773c..a0e8caa8 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -2204,6 +2204,8 @@ namespace simplecpp { } output.push_back(newMacroToken(tok->str(), loc, true, tok)); + if (it != macros.end()) + output.back()->markExpandedFrom(&it->second); return tok->next; } diff --git a/simplecpp.h b/simplecpp.h index 49aa600b..b744e62d 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -213,6 +213,9 @@ namespace simplecpp { bool isExpandedFrom(const Macro* m) const { return mExpandedFrom.find(m) != mExpandedFrom.end(); } + void markExpandedFrom(const Macro* m) { + mExpandedFrom.insert(m); + } void printAll() const; void printOut() const; diff --git a/test.cpp b/test.cpp index b0bf0b4f..bc8cb4d5 100644 --- a/test.cpp +++ b/test.cpp @@ -946,6 +946,16 @@ static void define_define_23() // #403 crash (infinite recursion) ASSERT_EQUALS("\n\n\n\nYdieZ ( void ) ;", preprocess(code)); } +static void define_define_24() // #590 +{ + const char code[] = "#define B A\n" + "#define A x(B)\n" + "#define C(s) s\n" + "#define D(s) C(s)\n" + "D(A)\n"; + ASSERT_EQUALS("\n\n\n\nx ( A )", preprocess(code)); +} + static void define_va_args_1() { const char code[] = "#define A(fmt...) dostuff(fmt)\n" @@ -3671,6 +3681,7 @@ int main(int argc, char **argv) TEST_CASE(define_define_21); TEST_CASE(define_define_22); // #400 TEST_CASE(define_define_23); // #403 - crash, infinite recursion + TEST_CASE(define_define_24); // #590 TEST_CASE(define_va_args_1); TEST_CASE(define_va_args_2); TEST_CASE(define_va_args_3); From ea9d3cbb19207e21e2ce4708b4c7076e11a8fd85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20Gunne=20Lindstr=C3=B6m?= Date: Thu, 16 Apr 2026 13:51:08 +0200 Subject: [PATCH 379/381] Refs #638: Avoid shadowing arguments (#639) --- simplecpp.cpp | 6 +++--- simplecpp.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index a0e8caa8..a7ced05a 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -635,10 +635,10 @@ static bool isStringLiteralPrefix(const std::string &str) str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; } -void simplecpp::TokenList::lineDirective(unsigned int fileIndex, unsigned int line, Location &location) +void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location) { - if (fileIndex != location.fileIndex || line >= location.line) { - location.fileIndex = fileIndex; + if (fileIndex_ != location.fileIndex || line >= location.line) { + location.fileIndex = fileIndex_; location.line = line; return; } diff --git a/simplecpp.h b/simplecpp.h index b744e62d..f29166ff 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -174,9 +174,9 @@ namespace simplecpp { bool isOneOf(const char ops[]) const; bool startsWithOneOf(const char c[]) const; bool endsWithOneOf(const char c[]) const; - static bool isNumberLike(const std::string& str) { - return std::isdigit(static_cast(str[0])) || - (str.size() > 1U && (str[0] == '-' || str[0] == '+') && std::isdigit(static_cast(str[1]))); + static bool isNumberLike(const std::string& s) { + return std::isdigit(static_cast(s[0])) || + (s.size() > 1U && (s[0] == '-' || s[0] == '+') && std::isdigit(static_cast(s[1]))); } TokenString macro; @@ -399,7 +399,7 @@ namespace simplecpp { void constFoldQuestionOp(Token *&tok1); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); - void lineDirective(unsigned int fileIndex, unsigned int line, Location &location); + void lineDirective(unsigned int fileIndex_, unsigned int line, Location &location); const Token* lastLineTok(int maxsize=1000) const; const Token* isLastLinePreprocessor(int maxsize=1000) const; From 316d4ee8e489ceaf1b9926df24d37b784d3655a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 17 Apr 2026 00:14:30 +0200 Subject: [PATCH 380/381] test.cpp: also run tests with char buffer (#261) --- test.cpp | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test.cpp b/test.cpp index bc8cb4d5..b654e159 100644 --- a/test.cpp +++ b/test.cpp @@ -6,6 +6,7 @@ #include "simplecpp.h" #include +#include #include #include #include @@ -25,6 +26,15 @@ #define STRINGIZE(x) STRINGIZE_(x) static const std::string testSourceDir = SIMPLECPP_TEST_SOURCE_DIR; + +namespace { + enum class Input : std::uint8_t { + Stringstream, + CharBuffer + }; +} + +static Input USE_INPUT = Input::Stringstream; static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) @@ -41,11 +51,21 @@ static std::string pprint(const std::string &in) return ret; } +static const char* inputString(Input input) { + switch (input) { + case Input::Stringstream: + return "Stringstream"; + case Input::CharBuffer: + return "CharBuffer"; + } + return ""; // unreachable - needed for GCC and Visual Studio +} + static int assertEquals(const std::string &expected, const std::string &actual, int line) { if (expected != actual) { numberOfFailedAssertions++; - std::cerr << "------ assertion failed ---------" << std::endl; + std::cerr << "------ assertion failed (" << inputString(USE_INPUT) << ")---------" << std::endl; std::cerr << "line test.cpp:" << line << std::endl; std::cerr << "expected:" << pprint(expected) << std::endl; std::cerr << "actual:" << pprint(actual) << std::endl; @@ -83,8 +103,16 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(std::string(code, size)); - return {istr,filenames,filename,outputList}; + switch (USE_INPUT) { + case Input::Stringstream: { + std::istringstream istr(std::string(code, size)); + return {istr,filenames,filename,outputList}; + } + case Input::CharBuffer: + return {{code, size}, filenames, filename, outputList}; + } + + return simplecpp::TokenList{filenames}; // unreachable - needed for GCC and Visual Studio } static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) @@ -3619,8 +3647,10 @@ static void leak() } } -int main(int argc, char **argv) +static void runTests(int argc, char **argv, Input input) { + USE_INPUT = input; + TEST_CASE(backslash); TEST_CASE(builtin); @@ -3892,6 +3922,11 @@ int main(int argc, char **argv) TEST_CASE(fuzz_crash); TEST_CASE(leak); +} +int main(int argc, char **argv) +{ + runTests(argc, argv, Input::Stringstream); + runTests(argc, argv, Input::CharBuffer); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 470196af7c753d3aee222c1534c63e5da869335e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 17 Apr 2026 23:27:32 +0200 Subject: [PATCH 381/381] CI-unixish.yml: added some missing `g++` cases to matrix (#649) --- .github/workflows/CI-unixish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 1a4ce649..cdda088b 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -15,8 +15,12 @@ jobs: include: - os: ubuntu-22.04 compiler: g++ + - os: ubuntu-22.04-arm + compiler: g++ - os: ubuntu-24.04 compiler: g++ + - os: ubuntu-24.04-arm + compiler: g++ fail-fast: false runs-on: ${{ matrix.os }}