diff --git a/man/manual-premium.md b/man/manual-premium.md index 9175ef47c8a..07eeab86094 100644 --- a/man/manual-premium.md +++ b/man/manual-premium.md @@ -307,6 +307,17 @@ To ignore certain folders you can use `-i`. This will skip analysis of source fi cppcheck --project=compile_commands.json -ifoo +### `-isystem`, `-I`, `--sysroot` + +We have a [script](https://github.com/cppcheck-opensource/cppcheck/blob/main/tools/tweak-compile-commands.py) that tweaks compile_commands.json. + +You can use it to: + * use `--sysroot` flags in Cppcheck analysis + * use `-isystem` paths in Cppcheck analysis + * remove `-I` paths from the compile_commands.json + +See [script documentation](https://github.com/cppcheck-opensource/cppcheck/blob/main/tools/tweak-compile-commands.md). + ## Visual Studio You can run Cppcheck on individual project files (`*.vcxproj`) or on a whole solution (`*.sln`) or (`*.slnx`). @@ -1166,6 +1177,7 @@ To use a `.cfg` file shipped with Cppcheck, pass the `--library=` option. T | `lua.cfg` | | | | `mfc.cfg` | [MFC](https://learn.microsoft.com/en-us/cpp/mfc/mfc-desktop-applications) | | | `microsoft_atl.cfg` | [ATL](https://learn.microsoft.com/en-us/cpp/atl/active-template-library-atl-concepts) | | +| `microsoft_gsl.cfg` | [Microsoft.GSL](https://github.com/microsoft/gsl) | | | `microsoft_sal.cfg` | [SAL annotations](https://learn.microsoft.com/en-us/cpp/c-runtime-library/sal-annotations) | | | `microsoft_unittest.cfg` | [CppUnitTest](https://learn.microsoft.com/en-us/visualstudio/test/microsoft-visualstudio-testtools-cppunittestframework-api-reference) | | | `motif.cfg` | | | diff --git a/man/manual.md b/man/manual.md index 93d67e8f402..6227ce0abb0 100644 --- a/man/manual.md +++ b/man/manual.md @@ -308,6 +308,17 @@ To ignore certain folders you can use `-i`. This will skip analysis of source fi cppcheck --project=compile_commands.json -ifoo +### `-isystem`, `-I`, `--sysroot` + +We have a [script](https://github.com/cppcheck-opensource/cppcheck/blob/main/tools/tweak-compile-commands.py) that tweaks compile_commands.json. + +You can use it to: + * use `--sysroot` flags in Cppcheck analysis + * use `-isystem` paths in Cppcheck analysis + * remove `-I` paths from the compile_commands.json + +See [script documentation](https://github.com/cppcheck-opensource/cppcheck/blob/main/tools/tweak-compile-commands.md). + ## Visual Studio You can run Cppcheck on individual project files (`*.vcxproj`) or on a whole solution (`*.sln`) or (`*.slnx`). diff --git a/releasenotes.txt b/releasenotes.txt index 2b3f4865397..1652fe61215 100644 --- a/releasenotes.txt +++ b/releasenotes.txt @@ -20,7 +20,7 @@ Changed interface: - Infrastructure & dependencies: -- +- compile_commands.json - flexible handling of -isystem, --sysroot and -I flags through the script tweak-compile_commands.py. Other: - Added configuration file for Microsoft.GSL (Guideline Support Library). diff --git a/tools/readme.md b/tools/readme.md index 4f914b3cce6..3a2de757199 100644 --- a/tools/readme.md +++ b/tools/readme.md @@ -83,3 +83,9 @@ message. Script to compare the error IDs in the expected `testrunner` output (without executing it) with the `--errorlist` output. It will report missing test coverage for an ID and missing IDs in the `--errorlist` output. + +### * tools/tweak-compile-commands.py + +Script to tweak `-isystem`/`--sysroot`/`-I` options in a `compile_commands.json` file, for example to make +implicit `--sysroot`-relative `-isystem` paths explicit, convert `-isystem` to `-I`, or remove unwanted include +paths. See `tools/tweak-compile-commands.md` for details. diff --git a/tools/tweak-compile-commands.md b/tools/tweak-compile-commands.md new file mode 100644 index 00000000000..25514872d8f --- /dev/null +++ b/tools/tweak-compile-commands.md @@ -0,0 +1,167 @@ +# tweak-compile-commands.py + +## NAME + +tweak-compile-commands.py - tweak `-isystem`/`--sysroot`/`-I` options in a `compile_commands.json` file + +## SYNOPSIS + +``` +tools/tweak-compile-commands.py COMPILE_COMMANDS [-o OUTPUT | -i] + [--isystem-to-i] [--exclude-folder FOLDER ...] + [--remove-include-path PATH ...] +``` + +## DESCRIPTION + +In many cases the system headers should not be included in Cppcheck +analysis, it is preferable to use `--library` instead. The headers +do not provide the knowledge needed for static analysis, i.e. they +can say what types the arguments to a function has but the header +do not provide the semantics of the functions. + +However sometimes you do want to include system headers in Cppcheck +analysis. And you need to have handling of `--sysroot` and +`-isystem`. This script will tweak the compile_commands.json file. + +### SYSROOT + +Example build command such as: + +``` +gcc --sysroot /a/b -isystem /opt/x -c foo.c +``` + +gcc searches both `/opt/x` *and* `/a/b/opt/x` for headers. + +`tweak-compile-commands.py` rewrites each build command in a +`compile_commands.json` file so this implicit behaviour is spelled out +explicitly: for every command that has a `--sysroot` argument, every +existing `-isystem PATH` argument gets a matching, explicit +`-isystem SYSROOT/PATH` argument added right after it, and the `--sysroot` +argument is then removed (it is no longer needed since the sysroot-relative +paths are now spelled out explicitly). Commands without a `--sysroot` +argument are left unchanged. + +### ISYSTEM + +The script has an option `--isystem-to-i`, this tells the script to +convert `-isystem` arguments to `-I`. + +The option `--exclude-folder` can be used to skip certain folders. Use +that for a folder if Cppcheck option `--library` can be used instead. + +### REMOVE -I + +The script also has `--remove-include-path`, the script will remove +any `-I PATH` argument whose path contains a given string. This is +useful for stripping include paths that Cppcheck should not see at all. + +## ARGUMENTS + +`COMPILE_COMMANDS` +: Path to the `compile_commands.json` file to read. + +## OPTIONS + +`-o OUTPUT`, `--output OUTPUT` +: Write the result to `OUTPUT` instead of stdout. Cannot be combined with + `-i`. + +`-i`, `--in-place` +: Overwrite `COMPILE_COMMANDS` with the result. Cannot be combined with + `-o`. + +`--isystem-to-i` +: Also convert `-isystem PATH` arguments to `-I PATH`, except for paths + excluded with `--exclude-folder`. Has no effect on its own if not given + (the sysroot tweak still applies). + +`--exclude-folder FOLDER` +: When used with `--isystem-to-i`, keep any `-isystem` argument as + `-isystem` (instead of converting it to `-I`) if `FOLDER` is one of the + path's folder components (an exact match of a path segment, not a + substring). May be given multiple times. Ignored if `--isystem-to-i` is + not given. + +`--remove-include-path PATH` +: Remove any `-I` argument whose path contains `PATH` as a substring. May be + given multiple times; a path is removed if it matches any of them. + Independent of `--isystem-to-i`/`--exclude-folder`, and applies after them, + so a path converted from `-isystem` to `-I` can also be removed by this + option. + +With neither `-o` nor `-i`, the resulting JSON is written to stdout, and +the input file is left untouched. A summary (`tweaked N of M entries`) is +always printed to stderr. + +## EXAMPLES + +Preview the sysroot tweak without touching any file: + +``` +$ tools/tweak-compile-commands.py compile_commands.json +``` + +Apply the sysroot tweak in place: + +``` +$ tools/tweak-compile-commands.py -i compile_commands.json +``` + +Apply the sysroot tweak and convert `-isystem` to `-I`, keeping any path +that goes through a `lib1` or `lib2` folder as `-isystem`: + +``` +$ tools/tweak-compile-commands.py -i compile_commands.json \ + --isystem-to-i --exclude-folder lib1 --exclude-folder lib2 +``` + +Given this input entry: + +```json +{ + "command": "gcc --sysroot /a/b -isystem /opt/x -isystem /path/lib1/include -c foo.c -o foo.o" +} +``` + +the last command above produces: + +```json +{ + "command": "gcc -I /opt/x -I /a/b/opt/x -isystem /path/lib1/include -isystem /a/b/path/lib1/include -c foo.c -o foo.o" +} +``` + +Note that `/path/lib1/include` is kept as `-isystem` (matching +`--exclude-folder lib1`), and so is its sysroot-relative duplicate +`/a/b/path/lib1/include`, since it also contains a `lib1` folder component. + +Remove all `-I` include paths that go through `/path/lib1`: + +``` +$ tools/tweak-compile-commands.py -i compile_commands.json \ + --remove-include-path /path/lib1 +``` + +Given this input entry: + +```json +{ + "command": "gcc -I /opt/x -I /path/lib1/include -c foo.c -o foo.o" +} +``` + +the command above produces: + +```json +{ + "command": "gcc -I /opt/x -c foo.c -o foo.o" +} +``` + +## EXIT STATUS + +Exits with a non-zero status and a traceback if `COMPILE_COMMANDS` cannot +be read or does not contain valid JSON. Otherwise exits 0, even if no +entries needed changes. diff --git a/tools/tweak-compile-commands.py b/tools/tweak-compile-commands.py new file mode 100755 index 00000000000..4e57cfaf33f --- /dev/null +++ b/tools/tweak-compile-commands.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +# +# Tweaks a compile_commands.json file: for every build command that has a +# --sysroot argument, each existing -isystem argument gets a matching extra +# -isystem argument pointing into the sysroot, and the --sysroot argument +# itself is then removed. This is useful when a compiler resolves -isystem +# paths relative to --sysroot internally (as part of its built-in search +# path handling) but a tool consuming compile_commands.json (such as +# Cppcheck) does not, so the sysroot-relative path needs to be spelled out +# explicitly instead. +# +# Example: +# --sysroot /a/b -isystem /opt/x +# => +# -isystem /opt/x -isystem /a/b/opt/x +# +# Optionally, --isystem-to-i converts -isystem arguments to -I, which can be +# useful since Cppcheck otherwise treats -isystem headers as "system" +# headers and skips some checks in them. Paths whose folder name matches one +# of the --exclude-folder values are left as -isystem. For example, with +# --isystem-to-i --exclude-folder lib1: +# -isystem /opt/x -isystem /path/lib1/include +# => +# -I /opt/x -isystem /path/lib1/include +# +# Optionally, --remove-include-path removes -I arguments whose path contains +# the given path. For example, with --remove-include-path /path/lib1: +# -I /opt/x -I /path/lib1/include +# => +# -I /opt/x +# +# Usage: +# tools/tweak-compile-commands.py compile_commands.json -o out.json +# tools/tweak-compile-commands.py -i compile_commands.json +# tools/tweak-compile-commands.py -i compile_commands.json --isystem-to-i --exclude-folder lib1 +# tools/tweak-compile-commands.py -i compile_commands.json --remove-include-path /path/lib1 +# +# With neither -o nor -i, the result is written to stdout. + +import argparse +import json +import shlex +import sys + + +def find_sysroot(tokens): + for i, tok in enumerate(tokens): + if tok == '--sysroot' and i + 1 < len(tokens): + return tokens[i + 1] + if tok.startswith('--sysroot='): + return tok[len('--sysroot='):] + return None + + +def join_sysroot(sysroot, path): + return sysroot.rstrip('/') + '/' + path.lstrip('/') + + +def remove_sysroot_arg(tokens): + result = [] + i = 0 + n = len(tokens) + while i < n: + tok = tokens[i] + if tok == '--sysroot' and i + 1 < n: + i += 2 + continue + if tok.startswith('--sysroot='): + i += 1 + continue + result.append(tok) + i += 1 + return result + + +def add_isystem_sysroot(tokens, sysroot): + result = [] + i = 0 + n = len(tokens) + while i < n: + tok = tokens[i] + if tok == '-isystem' and i + 1 < n: + path = tokens[i + 1] + result.append(tok) + result.append(path) + result.append('-isystem') + result.append(join_sysroot(sysroot, path)) + i += 2 + continue + if tok.startswith('-isystem') and tok != '-isystem': + path = tok[len('-isystem'):] + result.append(tok) + result.append('-isystem' + join_sysroot(sysroot, path)) + i += 1 + continue + result.append(tok) + i += 1 + return result + + +def is_excluded(path, exclude_folders): + parts = path.replace('\\', '/').split('/') + return any(part in exclude_folders for part in parts if part) + + +def convert_isystem_to_i(tokens, exclude_folders): + result = [] + i = 0 + n = len(tokens) + while i < n: + tok = tokens[i] + if tok == '-isystem' and i + 1 < n: + path = tokens[i + 1] + result.append(tok if is_excluded(path, exclude_folders) else '-I') + result.append(path) + i += 2 + continue + if tok.startswith('-isystem') and tok != '-isystem': + path = tok[len('-isystem'):] + result.append(tok if is_excluded(path, exclude_folders) else '-I' + path) + i += 1 + continue + result.append(tok) + i += 1 + return result + + +def matches_remove_path(path, remove_paths): + normalized = path.replace('\\', '/') + return any(remove_path.replace('\\', '/') in normalized for remove_path in remove_paths) + + +def remove_include_paths(tokens, remove_paths): + result = [] + i = 0 + n = len(tokens) + while i < n: + tok = tokens[i] + if tok == '-I' and i + 1 < n: + path = tokens[i + 1] + if matches_remove_path(path, remove_paths): + i += 2 + continue + result.append(tok) + result.append(path) + i += 2 + continue + if tok.startswith('-I') and tok != '-I': + path = tok[len('-I'):] + if matches_remove_path(path, remove_paths): + i += 1 + continue + result.append(tok) + i += 1 + continue + result.append(tok) + i += 1 + return result + + +def tweak_entry(entry, isystem_to_i, exclude_folders, remove_include_path): + if 'command' in entry: + tokens = shlex.split(entry['command']) + elif 'arguments' in entry: + tokens = entry['arguments'] + else: + return False + + changed = False + + sysroot = find_sysroot(tokens) + if sysroot is not None: + tokens = add_isystem_sysroot(tokens, sysroot) + tokens = remove_sysroot_arg(tokens) + changed = True + + if isystem_to_i: + new_tokens = convert_isystem_to_i(tokens, exclude_folders) + if new_tokens != tokens: + tokens = new_tokens + changed = True + + if remove_include_path: + new_tokens = remove_include_paths(tokens, remove_include_path) + if new_tokens != tokens: + tokens = new_tokens + changed = True + + if not changed: + return False + + if 'command' in entry: + entry['command'] = shlex.join(tokens) + else: + entry['arguments'] = tokens + return True + + +def main(): + parser = argparse.ArgumentParser( + description='Add sysroot-relative -isystem arguments to build commands in a compile_commands.json file.') + parser.add_argument('compile_commands', help='path to the compile_commands.json file to read') + group = parser.add_mutually_exclusive_group() + group.add_argument('-o', '--output', help='write the result to this file instead of stdout') + group.add_argument('-i', '--in-place', action='store_true', help='overwrite the input file with the result') + parser.add_argument('--isystem-to-i', action='store_true', + help='convert -isystem arguments to -I (except excluded folders)') + parser.add_argument('--exclude-folder', action='append', default=[], metavar='FOLDER', + help='folder name to keep as -isystem when using --isystem-to-i; can be given multiple times') + parser.add_argument('--remove-include-path', action='append', default=[], metavar='PATH', + help='remove -I arguments whose path contains PATH; can be given multiple times') + args = parser.parse_args() + + with open(args.compile_commands, encoding='utf-8') as f: + entries = json.load(f) + + changed = 0 + for entry in entries: + if tweak_entry(entry, args.isystem_to_i, args.exclude_folder, args.remove_include_path): + changed += 1 + + out = json.dumps(entries, indent=2) + '\n' + + if args.in_place: + with open(args.compile_commands, 'w', encoding='utf-8') as f: + f.write(out) + elif args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(out) + else: + sys.stdout.write(out) + + print('tweaked {} of {} entries'.format(changed, len(entries)), file=sys.stderr) + + +if __name__ == '__main__': + main()