From 1fac839d0e5d053e26597c09c4451aac7f227ca2 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 10 Apr 2024 17:08:05 -0400 Subject: [PATCH 001/113] fix: don't create a log file by default --- _appmap/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_appmap/env.py b/_appmap/env.py index a4ffd719..91acaf48 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -126,7 +126,7 @@ def _configure_logging(self): trace_logger.install() log_level = self.get("APPMAP_LOG_LEVEL", "warn").upper() - disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "false").upper() != "FALSE" + disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "true").upper() != "FALSE" log_config = self.get("APPMAP_LOG_CONFIG") now = datetime.now() config_dict = { From a8f82c615739cc26105da4d4912521d12c31dd4e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 10 Apr 2024 23:27:30 +0000 Subject: [PATCH 002/113] chore(release): 1.20.1 [skip ci] ## [1.20.1](https://github.com/getappmap/appmap-python/compare/v1.20.0...v1.20.1) (2024-04-10) ### Bug Fixes * don't create a log file by default ([1fac839](https://github.com/getappmap/appmap-python/commit/1fac839d0e5d053e26597c09c4451aac7f227ca2)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe8521c..dbebe904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.20.1](https://github.com/getappmap/appmap-python/compare/v1.20.0...v1.20.1) (2024-04-10) + + +### Bug Fixes + +* don't create a log file by default ([1fac839](https://github.com/getappmap/appmap-python/commit/1fac839d0e5d053e26597c09c4451aac7f227ca2)) + # [1.20.0](https://github.com/getappmap/appmap-python/compare/v1.19.1...v1.20.0) (2024-03-15) diff --git a/pyproject.toml b/pyproject.toml index 27a3f1ed..39b2447e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.20.0" +version = "1.20.1" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From fbcbd5ac3ed5b6319ebae343fe9643360d89aea7 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 22 Apr 2024 13:39:08 -0400 Subject: [PATCH 003/113] refactor: a little housekeeping --- _appmap/test/test_fastapi.py | 4 ---- _appmap/test/test_sqlalchemy.py | 6 ++++-- appmap/fastapi.py | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/_appmap/test/test_fastapi.py b/_appmap/test/test_fastapi.py index c5daee90..d6ab0b9f 100644 --- a/_appmap/test/test_fastapi.py +++ b/_appmap/test/test_fastapi.py @@ -1,13 +1,9 @@ import importlib -import socket -import sys from importlib.metadata import version -from pathlib import Path from types import SimpleNamespace as NS import pytest from fastapi.testclient import TestClient -from xprocess import ProcessStarter import appmap from _appmap.env import Env diff --git a/_appmap/test/test_sqlalchemy.py b/_appmap/test/test_sqlalchemy.py index 7258f9c2..f0d97a7a 100644 --- a/_appmap/test/test_sqlalchemy.py +++ b/_appmap/test/test_sqlalchemy.py @@ -13,7 +13,7 @@ create_engine, ) -import appmap.sqlalchemy # pylint: disable=unused-import +import appmap.sqlalchemy # pylint: disable=unused-import # noqa: F401 from _appmap.metadata import Metadata from ..test.helpers import DictIncluding @@ -28,7 +28,9 @@ def test_sql_capture(connection, events): {"sql": "SELECT 1", "database_type": "sqlite"} ) assert events[0].sql_query["server_version"].startswith("3.") - assert Metadata()["frameworks"] == [{"name": "SQLAlchemy", "version": version("sqlalchemy")}] + assert Metadata()["frameworks"] == [ + {"name": "SQLAlchemy", "version": version("sqlalchemy")}, + ] @staticmethod # pylint: disable=unused-argument diff --git a/appmap/fastapi.py b/appmap/fastapi.py index 949c7a72..170ae331 100644 --- a/appmap/fastapi.py +++ b/appmap/fastapi.py @@ -59,7 +59,6 @@ def __init__(self, app, remote_enabled=None): def init_app(self): # pylint: disable=import-outside-toplevel - from fastapi.middleware.wsgi import WSGIMiddleware from starlette.routing import Mount, Router # pylint: enable=import-outside-toplevel From 670660f4f1202f0a255d8f3ebcd11a4970090cca Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 15 Apr 2024 07:27:11 -0400 Subject: [PATCH 004/113] feat: add runner, get ready for v2 Starting with v2, the agent will no longer be enabled by default when installed in a Python environment. To instrument code and create recordings, the `appmap-python` script must be used, or the environment variables explicitly managed. --- _appmap/env.py | 30 +++++- _appmap/recording.py | 5 +- _appmap/test/conftest.py | 7 +- _appmap/test/test_configuration.py | 8 +- _appmap/test/test_django.py | 4 +- _appmap/test/test_env.py | 2 +- _appmap/test/test_flask.py | 10 +- _appmap/test/test_runner.py | 52 +++++++++++ _appmap/test/test_test_frameworks.py | 4 +- _appmap/unittest.py | 3 + _appmap/web_framework.py | 2 + appmap/__init__.py | 80 ++++++++-------- appmap/command/runner.py | 133 +++++++++++++++++++++++++++ appmap/pytest.py | 4 + ci/smoketest.sh | 2 +- pyproject.toml | 2 + tox.ini | 12 +-- 17 files changed, 291 insertions(+), 69 deletions(-) create mode 100644 _appmap/test/test_runner.py create mode 100644 appmap/command/runner.py diff --git a/_appmap/env.py b/_appmap/env.py index 91acaf48..1bd33471 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -3,6 +3,7 @@ import logging import logging.config import os +import warnings from contextlib import contextmanager from datetime import datetime from os import environ @@ -11,6 +12,20 @@ from . import trace_logger +_ENABLED_BY_DEFAULT_MSG = """ + +The APPMAP environment variable is unset. Your code will be +instrumented and recorded according to the configuration in appmap.yml. + +Starting with version 2, this behavior will change: when APPMAP is +unset, no code will be instrumented. You will need to use the +appmap-python script to run your application, or explicitly set +APPMAP. + +Visit https://appmap.io/docs/reference/appmap-python.html#appmap-python-script for more +details. +""" + _cwd = Path.cwd() _bootenv = environ.copy() @@ -37,16 +52,21 @@ def reset(cls, **kwargs): class Env(metaclass=_EnvMeta): def __init__(self, env=None, cwd=None): + warnings.filterwarnings("once", _ENABLED_BY_DEFAULT_MSG) + # root_dir and root_dir_len are going to be used when # instrumenting every function, so preprocess them as # much as possible. + self._cwd = cwd or _cwd self._env = _bootenv.copy() if env: self._env.update(env) self._configure_logging() - self._enabled = self._env.get("APPMAP", "").lower() != "false" + enabled = self._env.get("_APPMAP", None) + self._enabled_by_default = enabled is None + self._enabled = enabled is None or enabled.lower() != "false" self._root_dir = str(self._cwd) + "/" self._root_dir_len = len(self._root_dir) @@ -82,6 +102,14 @@ def root_dir_len(self): def output_dir(self): return self._output_dir + @property + def enabled_by_default(self): + return self._enabled_by_default + + def warn_enabled_by_default(self): + if self._enabled_by_default: + warnings.warn(_ENABLED_BY_DEFAULT_MSG, category=DeprecationWarning, stacklevel=2) + @property def enabled(self): return self._enabled diff --git a/_appmap/recording.py b/_appmap/recording.py index 0ceb2f33..465a5e1b 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -1,6 +1,6 @@ import atexit -from datetime import datetime, timezone import os +from datetime import datetime, timezone from tempfile import NamedTemporaryFile from _appmap import generation @@ -44,6 +44,7 @@ def is_running(self): return Recorder.get_enabled() def __enter__(self): + Env.current.warn_enabled_by_default() self.start() def __exit__(self, exc_type, exc_value, tb): @@ -80,6 +81,8 @@ def write_appmap( def initialize(): if Env.current.enables("process", "false"): + Env.current.warn_enabled_by_default() + r = Recording() r.start() diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index a95100dc..e2dee18e 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -14,7 +14,6 @@ import _appmap import appmap -from _appmap.env import Env from _appmap.test.web_framework import TEST_HOST, TEST_PORT from appmap import generation @@ -62,11 +61,11 @@ def pytest_runtest_setup(item): appmap_enabled = mark.kwargs.get("appmap_enabled", None) if isinstance(appmap_enabled, str): - env["APPMAP"] = appmap_enabled + env["_APPMAP"] = appmap_enabled elif appmap_enabled is False: - env["APPMAP"] = "false" + env["_APPMAP"] = "false" elif appmap_enabled is None: - env.pop("APPMAP", None) + env.pop("_APPMAP", None) _appmap.initialize(env=env) # pylint: disable=protected-access diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index 4c8eed6e..206af5eb 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -44,15 +44,15 @@ def test_reports_invalid(): @pytest.mark.appmap_enabled(config="appmap-broken.yml") def test_is_disabled_when_unset(): """Test that recording is disabled when APPMAP is unset but the config is broken""" - assert Env.current.get("APPMAP", None) is None + assert Env.current.get("_APPMAP", None) is None assert not appmap.enabled() @pytest.mark.appmap_enabled(config="appmap-broken.yml", appmap_enabled="false") def test_is_disabled_when_false(): - """Test that recording is disabled when APPMAP=false""" - Env.current.set("APPMAP", "false") + """Test that recording is disabled when _APPMAP=false""" + Env.current.set("_APPMAP", "false") assert not appmap.enabled() @@ -204,7 +204,7 @@ def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch assert not path.is_file() # pylint: disable=protected-access - _appmap.initialize(cwd=repo_root, env={"APPMAP": "false"}) + _appmap.initialize(cwd=repo_root, env={"_APPMAP": "false"}) c = Config() assert not path.is_file() diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index cf22a7c4..bc1f40c8 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -3,7 +3,6 @@ import json import os -import socket import sys from pathlib import Path from types import SimpleNamespace as NS @@ -16,7 +15,6 @@ import pytest from django.template.loader import render_to_string from django.test.client import MULTIPART_CONTENT -from xprocess import ProcessStarter import appmap import appmap.django # noqa: F401 @@ -212,7 +210,7 @@ def test_enabled(self, pytester): assert "http_server_request" in events[0] def test_disabled(self, pytester, monkeypatch): - monkeypatch.setenv("APPMAP", "false") + monkeypatch.setenv("_APPMAP", "false") result = pytester.runpytest("-svv", "-p", "no:randomly", "-k", "test_request") result.assert_outcomes(passed=1, failed=0, errors=0) assert not (pytester.path / "tmp").exists() diff --git a/_appmap/test/test_env.py b/_appmap/test/test_env.py index 2cfff6b8..f961ae7a 100644 --- a/_appmap/test/test_env.py +++ b/_appmap/test/test_env.py @@ -2,7 +2,7 @@ def test_disable_temporarily(): - env = Env({"APPMAP": "true"}) + env = Env({"_APPMAP": "true"}) assert env.enables("requests") try: with env.disabled("requests"): diff --git a/_appmap/test/test_flask.py b/_appmap/test/test_flask.py index 0d8a5a4e..9bd1eb5a 100644 --- a/_appmap/test/test_flask.py +++ b/_appmap/test/test_flask.py @@ -3,21 +3,15 @@ import importlib import os -import socket -import sys -from functools import partial from importlib.metadata import version -from pathlib import Path from types import SimpleNamespace as NS import flask import pytest -from attr import dataclass -from xprocess import ProcessStarter +from appmap.flask import AppmapFlask from _appmap.env import Env from _appmap.metadata import Metadata -from appmap.flask import AppmapFlask from ..test.helpers import DictIncluding from .web_framework import ( @@ -154,7 +148,7 @@ def test_enabled(self, pytester): assert appmap_file.exists() def test_disabled(self, pytester, monkeypatch): - monkeypatch.setenv("APPMAP", "false") + monkeypatch.setenv("_APPMAP", "false") result = pytester.runpytest("-svv") diff --git a/_appmap/test/test_runner.py b/_appmap/test/test_runner.py new file mode 100644 index 00000000..176c9401 --- /dev/null +++ b/_appmap/test/test_runner.py @@ -0,0 +1,52 @@ +import re + +import pytest + + +def test_runner_noargs(script_runner): + result = script_runner.run(["appmap-python"]) + assert result.returncode != 0 + assert result.stdout.startswith("usage") + + +def test_runner_help(script_runner): + result = script_runner.run(["appmap-python", "--help"]) + assert result.returncode == 0 + assert result.stdout.startswith("usage") + + +@pytest.mark.parametrize("recording_type", ["process", "pytest", "remote", "requests", "unittest"]) +def test_runner_recording_type(script_runner, recording_type): + result = script_runner.run(["appmap-python", "--record", recording_type]) + assert result.returncode == 0 + assert ( + re.search(f"(?m)^APPMAP_RECORD_{recording_type.upper()}=true$", result.stdout) is not None + ) + + result = script_runner.run(["appmap-python", "--no-record", recording_type]) + assert result.returncode == 0 + assert re.search(f"(?m)^APPMAP_RECORD_{recording_type.upper()}=true$", result.stdout) is None + + +@pytest.mark.parametrize("flag,expected", [("--record", 1), ("--no-record", 0)]) +def test_runner_multi_recording_type(script_runner, flag, expected): + types = "process,pytest" + result = script_runner.run(["appmap-python", flag, types]) + assert result.returncode == 0 + assert len(re.findall("(?m)^APPMAP_RECORD_PROCESS=true$", result.stdout)) == expected + assert len(re.findall("(?m)^APPMAP_RECORD_PYTEST=true$", result.stdout)) == expected + + +@pytest.mark.script_launch_mode("subprocess") +class TestEnv: + def test_appmap_present(self, script_runner): + result = script_runner.run(["appmap-python", "printenv", "APPMAP"]) + assert result.returncode == 0 + assert re.match(r"true", result.stdout) is not None + + def test_recording_type_present(self, script_runner): + result = script_runner.run( + ["appmap-python", "--record", "process", "printenv", "APPMAP_RECORD_PROCESS"] + ) + assert result.returncode == 0 + assert re.match(r"true", result.stdout) is not None diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 0b73216c..6cd5e2f1 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -32,7 +32,7 @@ def run_tests(self, testdir): """Run the tests.""" def test_with_appmap_false(self, testdir, monkeypatch): - monkeypatch.setenv("APPMAP", "false") + monkeypatch.setenv("_APPMAP", "false") self.run_tests(testdir) @@ -165,7 +165,7 @@ def fixture_runner_testdir(request, data_dir, pytester, monkeypatch): # Make sure APPMAP isn't the environment, to test that recording-by-default is working as # expected. Individual test cases may set it as necessary. - monkeypatch.delenv("APPMAP", raising=False) + monkeypatch.delenv("_APPMAP", raising=False) marker = request.node.get_closest_marker("example_dir") test_type = "unittest" if marker is None else marker.args[0] diff --git a/_appmap/unittest.py b/_appmap/unittest.py index 7641a86b..18fcfed6 100644 --- a/_appmap/unittest.py +++ b/_appmap/unittest.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from _appmap import noappmap, testing_framework, wrapt +from _appmap.env import Env from _appmap.utils import get_function_location _session = testing_framework.session("unittest", "tests") @@ -42,6 +43,7 @@ def _args(test_case, *_, isTest=False, **__): with _session.record( test_case.__class__, method_name, location=location ) as metadata: + Env.current.warn_enabled_by_default() if metadata: with wrapped( *args, **kwargs @@ -67,6 +69,7 @@ def callTestMethod(wrapped, test_case, args, kwargs): method_name = test_case.id().split(".")[-1] location = _get_test_location(test_case.__class__, method_name) with _session.record(test_case.__class__, method_name, location=location) as metadata: + Env.current.warn_enabled_by_default() if metadata: with testing_framework.collect_result_metadata(metadata): wrapped(*args, **kwargs) diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index b5e67aa8..4bb25078 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -247,6 +247,8 @@ def remote_enabled(self): """Return True if the AppMap middleware has enabled remote recording, False otherwise.""" def run(self): + Env.current.warn_enabled_by_default() + if not self.middleware_present(): return self.insert_middleware() diff --git a/appmap/__init__.py b/appmap/__init__.py index 65e38973..4f457133 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -1,38 +1,44 @@ """AppMap recorder for Python""" - -from _appmap import generation # noqa: F401 -from _appmap.env import Env # noqa: F401 -from _appmap.importer import instrument_module # noqa: F401 -from _appmap.labels import labels # noqa: F401 -from _appmap.noappmap import decorator as noappmap -from _appmap.recording import Recording # noqa: F401 - -try: - from . import django # noqa: F401 -except ImportError: - pass - -try: - from . import flask # noqa: F401 -except ImportError: - pass - -try: - from . import fastapi # noqa: F401 -except ImportError: - pass - -try: - from . import uvicorn # noqa: F401 -except ImportError: - pass - -# Note: pytest integration is configured as a pytest plugin, so it doesn't need to be imported here - -# unittest is part of the standard library, so it should always be importable (and therefore doesn't -# need to be in a try .. except block) -from . import unittest # noqa: F401 - - -def enabled(): - return Env.current.enabled +import os + +# Note that we need to guard these imports with a conditional, rather than +# putting them in a function and conditionally calling the function. If we +# execute the imports in a function, the modules all get put into the funtion's +# globals, rather than into appmap's globals. +if os.environ.get("APPMAP", "true").upper() == "TRUE": + from _appmap import generation # noqa: F401 + from _appmap.env import Env # noqa: F401 + from _appmap.importer import instrument_module # noqa: F401 + from _appmap.labels import labels # noqa: F401 + from _appmap.noappmap import decorator as noappmap # noqa: F401 + from _appmap.recording import Recording # noqa: F401 + + try: + from . import django # noqa: F401 + except ImportError: + pass + + try: + from . import flask # noqa: F401 + except ImportError: + pass + + try: + from . import fastapi # noqa: F401 + except ImportError: + pass + + try: + from . import uvicorn # noqa: F401 + except ImportError: + pass + + # Note: pytest integration is configured as a pytest plugin, so it doesn't + # need to be imported here + + # unittest is part of the standard library, so it should always be + # importable (and therefore doesn't need to be in a try .. except block) + from . import unittest # noqa: F401 + + def enabled(): + return Env.current.enabled diff --git a/appmap/command/runner.py b/appmap/command/runner.py new file mode 100644 index 00000000..4fbf6c08 --- /dev/null +++ b/appmap/command/runner.py @@ -0,0 +1,133 @@ +import argparse +import getopt +import os +import sys +import textwrap + +_parser = argparse.ArgumentParser( + description=textwrap.dedent(""" +Enable recording of the provided command, optionally specifying the +type(s) of recording to enable and disable. If a recording type is +specified as both enabled and disabled, it will be enabled. + +This command sets the environment variables described here: +https://appmap.io/docs/reference/appmap-python.html#controlling-recording. +For any recording type that is not explicitly specified, the +corresponding environment variable will not be set. + +If no command is provided, the computed set of environment variables +will be displayed. + """), + formatter_class=argparse.RawDescriptionHelpFormatter, +) + +_RECORDING_TYPES = set( + [ + "process", + "pytest", + "remote", + "requests", + "unittest", + ] +) + + +def recording_types(v: str): + values = set(v.split(",")) + if not values & _RECORDING_TYPES: + raise argparse.ArgumentTypeError(v) + return values + + +_parser.add_argument( + "--record", + help="recording types to enable", + metavar=",".join(_RECORDING_TYPES), + type=recording_types, + default=argparse.SUPPRESS, +) +_parser.add_argument( + "--no-record", + help="recording types to disable", + metavar=",".join(_RECORDING_TYPES), + type=recording_types, + default=argparse.SUPPRESS, +) + +if sys.version_info >= (3, 9): + _parser.add_argument( + "--enable-log", + help="create a log file", + action=argparse.BooleanOptionalAction, + default=False, + ) +else: + # You can see why BooleanOptionalAction was added. This is close, though not + # really as good.... + _enable_log_group = _parser.add_mutually_exclusive_group() + _enable_log_group.add_argument( + "--enable-log", + help="create a log file", + dest="enable_log", + action="store_true", + ) + _enable_log_group.add_argument( + "--no-enable-log", + help="don't create a log file", + dest="enable_log", + action="store_false", + ) + +_parser.add_argument( + "command", + nargs="*", + help="the command to run (default: display the environment variables)", + default=argparse.SUPPRESS, +) + + +def run(): + if len(sys.argv) == 1: + _parser.print_help() + sys.exit(1) + + # Use gnu_getopt to separate the command line into args we know about, + # followed by the command to run (and its args) + try: + getopt_flags = ["help", "record=", "no-record=", "enable-log", "no-enable-log"] + opts, cmd = getopt.gnu_getopt(sys.argv[1:], "+h", getopt_flags) + except getopt.GetoptError as exc: + print(exc, file=sys.stderr) + _parser.print_help() + sys.exit(1) + + # parse the args after flattening the tuples returned from gnu_getopt + flags = [f for opt in opts for f in opt if len(f) > 0] + parsed_args = _parser.parse_args(flags) + parsed_args = vars(parsed_args) + + # our settings override those in the environment + envvars = {"APPMAP": "true"} + + # Set the environment variables based on the the flags. A recording type in + # --record overrides one set in --no-record. The environment variable for a + # type that doesn't appear in either will be unset. + record = parsed_args.get("record", set()) + no_record = parsed_args.get("no_record", set()) - record + for enabled in record: + envvars[f"APPMAP_RECORD_{enabled.upper()}"] = "true" + for disabled in no_record: + envvars[f"APPMAP_RECORD_{disabled.upper()}"] = "false" + + envvars["APPMAP_DISABLE_LOG_FILE"] = "false" if parsed_args["enable_log"] else "true" + + if len(cmd) == 0: + for k, v in sorted(envvars.items()): + print(f"{k}={v}") + sys.exit(0) + + os.execvpe(cmd[0], cmd, {**os.environ, **envvars}) + + +if __name__ == "__main__": + run() diff --git a/appmap/pytest.py b/appmap/pytest.py index 6a1d3f13..e25547a0 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -25,6 +25,10 @@ def __call__(self, wrapped, _, args, kwargs): if not Env.current.is_appmap_repo and Env.current.enables("pytest"): logger.debug("Test recording is enabled (Pytest)") + @pytest.hookimpl + def pytest_configure(config): + Env.current.warn_enabled_by_default() + @pytest.hookimpl def pytest_sessionstart(session): session.appmap = testing_framework.session( diff --git a/ci/smoketest.sh b/ci/smoketest.sh index 07ba0981..ab8089a4 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -12,7 +12,7 @@ cat /tmp/appmap.yml python -m appmap.command.appmap_agent_validate -$RUNNER pytest -k test_hello_world +$RUNNER appmap-python pytest -k test_hello_world if [[ -f tmp/appmap/pytest/simple_test_simple_UnitTestTest_test_hello_world.appmap.json ]]; then echo 'Success' diff --git a/pyproject.toml b/pyproject.toml index 39b2447e..ae80ff9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ uvicorn = "^0.27.1" fastapi = "^0.110.0" httpx = "^0.27.0" pytest-env = "^1.1.3" +pytest-console-scripts = "^1.4.1" [build-system] requires = ["poetry-core>=1.1.0"] @@ -91,6 +92,7 @@ appmap = "appmap.pytest" appmap-agent-init = "appmap.command.appmap_agent_init:run" appmap-agent-status = "appmap.command.appmap_agent_status:run" appmap-agent-validate = "appmap.command.appmap_agent_validate:run" +appmap-python = "appmap.command.runner:run" [tool.black] line-length = 102 diff --git a/tox.ini b/tox.ini index aa889dd4..a25a18d4 100644 --- a/tox.ini +++ b/tox.ini @@ -18,13 +18,11 @@ deps= commands = - # Turn off recording while installing. It's not necessary, and the warning messages that come - # out of the agent confuse poetry. - env APPMAP_LOG_LEVEL=warning APPMAP=false poetry install -v - py310-web: bash -c "poetry run pylint -j 0 appmap _appmap || pylint-exit $?" - web: poetry run {posargs:pytest} - django3: poetry run pytest _appmap/test/test_django.py - flask2: poetry run pytest _appmap/test/test_flask.py + poetry install -v + py310-web: poetry run pylint -j 0 appmap _appmap + web: poetry run appmap-python {posargs:pytest} + django3: poetry run appmap-python pytest _appmap/test/test_django.py + flask2: poetry run appmap-python pytest _appmap/test/test_flask.py [testenv:vendoring] skip_install = True From bddbe701c5651b2aa81471fd3a6cf76262b89e28 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 29 Apr 2024 08:38:06 -0400 Subject: [PATCH 005/113] docs: add doc for recording env vars --- doc/recording-env-vars.md | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 doc/recording-env-vars.md diff --git a/doc/recording-env-vars.md b/doc/recording-env-vars.md new file mode 100644 index 00000000..92c5297c --- /dev/null +++ b/doc/recording-env-vars.md @@ -0,0 +1,44 @@ +The tables below describe how the variable environment variables control the various +recording types. In each case, ✓ means that the corresponding recording type +will be produced, ❌ means that it will not. + +## Web Apps +These tables describe how `APPMAP_RECORD_REQUEST` and `APPMAP_RECORD_REMOTE` are +handled when running a web app. "web app, debug on" means a Flask app run as `flask --debug`, +a FastAPI app run using `uvicorn --reload` and, a Django app run with `DEBUG = True` in `settings.py`. + +| | `APPMAP_RECORD_REQUEST` is unset | `APPMAP_RECORD_REQUEST` == "true" | `APPMAP_RECORD_REQUEST` == "false" | +| -------------------- | :----------------------------: | :------------------------------: | :-------------------------------: | +| "web app, debug on" | ✓ | ✓ | ❌ | +| "web app, debug off" | ✓ | ✓ | ❌ | + + +| | `APPMAP_RECORD_REMOTE` is unset | `APPMAP_RECORD_REMOTE` == "true" | `APPMAP_RECORD_REMOTE` == "false" | +| -------------------- | :---------------------------: | :----------------------------: | :------------------------------: | +| "web app, debug on" | ✓ | ✓ | ❌ | +| "web app, debug off" | ❌ | ✓(with warning) | ❌ | + + +## Testing +This table shows how `APPMAP_RECORD_PYTEST`, `APPMAP_RECORD_UNITTEST`, and +`APPMAP_RECORD_REQUEST` are handled when running tests in. Note that in v2, in +v2, `APPMAP_RECORD_PYTEST` and `APPMAP_RECORD_UNITTEST` will be replaced with +APPMAP_RECORD_TEST. + +| | `APPMAP_RECORD_PYTEST` is unset | `APPMAP_RECORD_PYTEST` == "true" | `APPMAP_RECORD_PYTEST` == "false" | `APPMAP_RECORD_REQUEST` is unset | `APPMAP_RECORD_REQUEST` == "true" | `APPMAP_RECORD_REQUEST` == "false" | +| ------ | :---------------------------: | :-----------------------------: | :------------------------------: | :----------------------------: | :-----------------------------: | :------------------------------: | +| pytest | ✓ | ✓ | ❌ | ✓in v1, ❌ in v2 | ✓ | ❌ | + + + + +## Process Recording +`APPMAP_RECORD_PROCESS` creates recordings as described in this table. Note +that, in v1, `APPMAP_RECORD_PROCESS` doesn't change the handling of any of the +other variables. As a result, setting it when running a either web app or when +running tests will result in an error. Whether this behavior should change in v2 +is TBD. + +| | `APPMAP_RECORD_PROCESS` is unset | `APPMAP_RECORD_PROCESS` == "true" | `APPMAP_RECORD_PROCESS` == "false" | +| ----------------- | :----------------------------: | :---------------------------------: | :----------------------------------: | +| process recording | ❌ | ✓ | ❌ | \ No newline at end of file From 4ada3fe6c05169c2241adca438451aec16e9f1d5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 29 Apr 2024 14:29:15 +0000 Subject: [PATCH 006/113] chore(release): 1.21.0 [skip ci] # [1.21.0](https://github.com/getappmap/appmap-python/compare/v1.20.1...v1.21.0) (2024-04-29) ### Features * add runner, get ready for v2 ([670660f](https://github.com/getappmap/appmap-python/commit/670660f4f1202f0a255d8f3ebcd11a4970090cca)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbebe904..76f2ca06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [1.21.0](https://github.com/getappmap/appmap-python/compare/v1.20.1...v1.21.0) (2024-04-29) + + +### Features + +* add runner, get ready for v2 ([670660f](https://github.com/getappmap/appmap-python/commit/670660f4f1202f0a255d8f3ebcd11a4970090cca)) + ## [1.20.1](https://github.com/getappmap/appmap-python/compare/v1.20.0...v1.20.1) (2024-04-10) diff --git a/pyproject.toml b/pyproject.toml index ae80ff9c..feebd8ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.20.1" +version = "1.21.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 4555c82c156d24475a5974566f5d531f5cc2fd69 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Mon, 13 May 2024 19:29:52 +0300 Subject: [PATCH 007/113] feat: search for config file --- _appmap/configuration.py | 42 +++++++++++++++++-- _appmap/env.py | 4 ++ _appmap/test/data/config-up/appmap.yml | 1 + .../data/config-up/subprojects/p1/__init__.py | 0 .../config-up/subprojects/p2/sub1/__init__.py | 0 _appmap/test/test_configuration.py | 41 ++++++++++++++++++ _appmap/test/test_util.py | 17 +++++++- _appmap/utils.py | 31 ++++++++++++++ 8 files changed, 131 insertions(+), 5 deletions(-) create mode 100644 _appmap/test/data/config-up/appmap.yml create mode 100644 _appmap/test/data/config-up/subprojects/p1/__init__.py create mode 100644 _appmap/test/data/config-up/subprojects/p2/sub1/__init__.py diff --git a/_appmap/configuration.py b/_appmap/configuration.py index bbfc8324..35b3bbf1 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -26,13 +26,21 @@ def default_app_name(rootdir): rootdir = Path(rootdir) - if not (rootdir / ".git").exists(): - return rootdir.name + if (rootdir / ".git").exists(): + repo_root = _get_repo_root(rootdir) + if repo_root: + return repo_root.name + return rootdir.name +def _get_repo_root(rootdir): git = utils.git(cwd=str(rootdir)) repo_root = git("rev-parse --show-toplevel") - return Path(repo_root).name + if repo_root: + return Path(repo_root) + return None +def _resolve_relative_to(path1: Path, path2: Path): + return (path2 / path1).resolve(strict=False) # Make it easy to mock sys.prefix def _get_sys_prefix(): @@ -201,7 +209,20 @@ def _load_config(self): if use_default_config: env_config_filename = "appmap.yml" - path = Path(env_config_filename).resolve() + env = Env.current + config_dir = env.root_dir + + path = _resolve_relative_to(Path(env_config_filename), Path(config_dir)) + if not path.is_file(): + # search config file in parent directories up to + # repo root (if exists) or up to file system root + repo_root = _get_repo_root(env.root_dir) + config_dir = utils.locate_file_up( + env_config_filename, env.root_dir, repo_root + ) + if config_dir: + path = _resolve_relative_to(Path(env_config_filename), Path(config_dir)) + if path.is_file(): self.file_present = True @@ -220,6 +241,19 @@ def _load_config(self): if "packages" not in self._config: self._config["packages"] = self.default_packages + # Is appmap_dir specified? + appmap_dir = ( + self._config["appmap_dir"] + if "appmap_dir" in self._config else "tmp/appmap" + ) + + # appmap_dir must be resolved relative to the location of config file + # unless APPMAP_OUTPUT_DIR is set by tests. + if config_dir and Env.current.get("APPMAP_OUTPUT_DIR", None) is None: + Env.current.output_dir = _resolve_relative_to( + Path(appmap_dir), Path(config_dir) + ) + self.file_valid = True Env.current.enabled = should_enable except ParserError: diff --git a/_appmap/env.py b/_appmap/env.py index 1bd33471..a4f88431 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -102,6 +102,10 @@ def root_dir_len(self): def output_dir(self): return self._output_dir + @output_dir.setter + def output_dir(self, value): + self._output_dir = value + @property def enabled_by_default(self): return self._enabled_by_default diff --git a/_appmap/test/data/config-up/appmap.yml b/_appmap/test/data/config-up/appmap.yml new file mode 100644 index 00000000..7621fe2d --- /dev/null +++ b/_appmap/test/data/config-up/appmap.yml @@ -0,0 +1 @@ +name: config-up-name \ No newline at end of file diff --git a/_appmap/test/data/config-up/subprojects/p1/__init__.py b/_appmap/test/data/config-up/subprojects/p1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/_appmap/test/data/config-up/subprojects/p2/sub1/__init__.py b/_appmap/test/data/config-up/subprojects/p2/sub1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index 206af5eb..dab8f4cb 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -248,3 +248,44 @@ def test_missing_packages(self, tmpdir): env={"APPMAP_CONFIG": "appmap-incomplete.yml"}, ) self.check_default_config(Path(tmpdir).name) + +class TestSearchConfig: + def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): + copy_tree(data_dir / "config-up", str(tmpdir)) + project_root = tmpdir / "subprojects" / "p1" + monkeypatch.chdir(project_root) + + # pylint: disable=protected-access + _appmap.initialize(cwd=project_root) + assert Config().name == "config-up-name" + assert str(Env.current.output_dir).endswith(str(tmpdir / "tmp" / "appmap")) + + def test_config_not_found_until_repo_root(self, data_dir, tmpdir, git_directory, monkeypatch): + copy_tree(data_dir / "config-up", str(tmpdir)) + repo_root = tmpdir / "subprojects" / "p2" + copy_tree(git_directory, str(repo_root)) + project_root = repo_root / "sub1" + monkeypatch.chdir(project_root) + + # pylint: disable=protected-access + _appmap.initialize(cwd=project_root) + # It should stop searching at repo_root. + # Check that it did not find appmap.yml + # in config-up folder. + assert Config().name != "config-up-name" + # It should go on with default config + assert Env.current.enabled + + def test_config_not_found_in_path_hierarchy(self, data_dir, tmpdir, monkeypatch): + copy_tree(data_dir / "config-up", str(tmpdir)) + project_root = tmpdir / "subprojects" / "p1" + monkeypatch.chdir(project_root) + + # pylint: disable=protected-access + _appmap.initialize( + cwd=project_root, + env={"APPMAP_CONFIG": "notfound.yml"}, + ) + Config() + # No default config since we specified APPMAP_CONFIG + assert not Env.current.enabled diff --git a/_appmap/test/test_util.py b/_appmap/test/test_util.py index 2c37857c..466bb6e0 100644 --- a/_appmap/test/test_util.py +++ b/_appmap/test/test_util.py @@ -2,7 +2,11 @@ Test util functionality """ -from _appmap.utils import scenario_filename +import os +from pathlib import Path +import uuid + +from _appmap.utils import locate_file_up, scenario_filename def test_scenario_filename__short(): @@ -13,3 +17,14 @@ def test_scenario_filename__short(): def test_scenario_filename__special_character(): """has a customizable suffix""" assert scenario_filename("foobar?=65") == "foobar_65" + +def test_locate_file_up(data_dir): + result = locate_file_up("appmap.yml", Path(data_dir) / "package1" / "package2") + assert result.parts[-3:] == ("_appmap", "test", "data") + + result = locate_file_up("test_util.py", Path(data_dir) / "package1" / "package2") + assert result.parts[-2:] == ("_appmap", "test") + + impossible_file_name = str(uuid.uuid4()) + ".yml" + result = locate_file_up(impossible_file_name, data_dir) + assert result is None diff --git a/_appmap/utils.py b/_appmap/utils.py index cf321c66..8defa962 100644 --- a/_appmap/utils.py +++ b/_appmap/utils.py @@ -1,5 +1,6 @@ import inspect import os +from pathlib import Path import re import shlex import subprocess @@ -222,3 +223,33 @@ def scenario_filename(name, separator="_"): pattern = r"[^a-z0-9\-_]+" replacement = separator return re.sub(pattern, replacement, name, flags=re.IGNORECASE) + + +def locate_file_up(filename, start_dir=None, stop_dir=None): + """ + Search for a file in the current directory and recursively up to the root directory. + + :param filename: The name of the file to locate. + :param start_dir: The directory to start the search from. Defaults to the current. + :param stop_idr: The directory to stop the search. If None search is performed until + the root of the file system. + :return: The path to the directory containing the file or None if the file cannot be found. + """ + + if start_dir is None: + start_dir = Path.cwd() + elif isinstance(start_dir, str): + start_dir = Path(start_dir) + + file_path = start_dir.joinpath(filename) + if Path.exists(file_path): + return start_dir + + if isinstance(stop_dir, str): + stop_dir = Path(stop_dir) + + parent_dir = start_dir.parent + if parent_dir in (start_dir, stop_dir): + return None + + return locate_file_up(filename, parent_dir, stop_dir) From 33db9af5dd33703b23153176232e4dd818b939c7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 15 May 2024 14:02:43 +0000 Subject: [PATCH 008/113] chore(release): 1.22.0 [skip ci] # [1.22.0](https://github.com/getappmap/appmap-python/compare/v1.21.0...v1.22.0) (2024-05-15) ### Features * search for config file ([4555c82](https://github.com/getappmap/appmap-python/commit/4555c82c156d24475a5974566f5d531f5cc2fd69)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f2ca06..83cacb41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [1.22.0](https://github.com/getappmap/appmap-python/compare/v1.21.0...v1.22.0) (2024-05-15) + + +### Features + +* search for config file ([4555c82](https://github.com/getappmap/appmap-python/commit/4555c82c156d24475a5974566f5d531f5cc2fd69)) + # [1.21.0](https://github.com/getappmap/appmap-python/compare/v1.20.1...v1.21.0) (2024-04-29) diff --git a/pyproject.toml b/pyproject.toml index feebd8ba..e1aa1918 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.21.0" +version = "1.22.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 9b221bd0f8d634028d389282bfb65bbfe225b01f Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Wed, 15 May 2024 09:58:27 +0300 Subject: [PATCH 009/113] test: fix sqlalchemy deprecated execute param --- _appmap/test/test_sqlalchemy.py | 32 +++++++++++++++++++------------- appmap/sqlalchemy.py | 2 +- pyproject.toml | 1 - requirements-dev.txt | 3 ++- requirements-test.txt | 1 + tox.ini | 6 ++++-- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/_appmap/test/test_sqlalchemy.py b/_appmap/test/test_sqlalchemy.py index f0d97a7a..5da284d9 100644 --- a/_appmap/test/test_sqlalchemy.py +++ b/_appmap/test/test_sqlalchemy.py @@ -10,6 +10,7 @@ MetaData, String, Table, + text, create_engine, ) @@ -23,7 +24,10 @@ class TestSQLAlchemy(AppMapTestBase): @staticmethod def test_sql_capture(connection, events): - connection.execute("SELECT 1") + # Passing a string to execute is deprecated in 1.4 + # and removed in 2.0. We wrap it with text(). + # https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.Connection.execute + connection.execute(text("SELECT 1")) assert events[0].sql_query == DictIncluding( {"sql": "SELECT 1", "database_type": "sqlite"} ) @@ -38,25 +42,27 @@ def test_capture_ddl(events, schema): assert "CREATE TABLE addresses" in events[-2].sql_query["sql"] # pylint: disable=unused-argument - def test_capture_insert(self, connection, schema, events): + def test_capture_insert(self, engine, schema, events): ins = self.users.insert().values(name="jack", fullname="Jack Jones") - connection.execute(ins) + with engine.begin() as conn: + conn.execute(ins) assert ( events[-2].sql_query["sql"] == "INSERT INTO users (name, fullname) VALUES (?, ?)" ) # pylint: disable=unused-argument - def test_capture_insert_many(self, connection, schema, events): - connection.execute( - self.addresses.insert(), - [ - {"user_id": 1, "email_address": "jack@yahoo.com"}, - {"user_id": 1, "email_address": "jack@msn.com"}, - {"user_id": 2, "email_address": "www@www.org"}, - {"user_id": 2, "email_address": "wendy@aol.com"}, - ], - ) + def test_capture_insert_many(self, engine, schema, events): + with engine.begin() as conn: + conn.execute( + self.addresses.insert(), + [ + {"user_id": 1, "email_address": "jack@yahoo.com"}, + {"user_id": 1, "email_address": "jack@msn.com"}, + {"user_id": 2, "email_address": "www@www.org"}, + {"user_id": 2, "email_address": "wendy@aol.com"}, + ], + ) assert ( events[-2].sql_query["sql"] == "-- 4 times\nINSERT INTO addresses (user_id, email_address) VALUES (?, ?)" diff --git a/appmap/sqlalchemy.py b/appmap/sqlalchemy.py index 3358b7c4..2e4c382f 100644 --- a/appmap/sqlalchemy.py +++ b/appmap/sqlalchemy.py @@ -15,7 +15,7 @@ @event.listens_for(Engine, "before_cursor_execute") # pylint: disable=too-many-arguments,unused-argument def capture_sql_call(conn, cursor, statement, parameters, context, executemany): - """Capture SQL query callinto appmap.""" + """Capture SQL query call into appmap.""" if is_instrumentation_disabled(): # We must be in the middle of fetching object representation. # Don't record this query in the appmap. diff --git a/pyproject.toml b/pyproject.toml index e1aa1918..33df765a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ packaging = ">=19.0" # install it and the rest of the dev dependencies. [tool.poetry.group.dev.dependencies] -SQLAlchemy = "^1.4.11" Twisted = "^22.4.0" asgiref = "^3.7.2" black = "^24.2.0" diff --git a/requirements-dev.txt b/requirements-dev.txt index c2d5913e..bc06c81f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,5 @@ django flask >=2, <= 3 pytest-django<4.8 fastapi -httpx \ No newline at end of file +httpx +sqlalchemy \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index 04392b28..16b853d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ django ~= 3.2 pytest-django < 4.8 +sqlalchemy < 2.0 diff --git a/tox.ini b/tox.ini index a25a18d4..ed1c73b7 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = true # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{8,9,10,11,12}-{web,django3,flask2} +envlist = py3{8,9,10,11,12}-{web,django3,flask2,sqlalchemy1} [testenv] allowlist_externals = @@ -13,9 +13,10 @@ deps= poetry web: Django >=4.0, <5.0 web: Flask >=3.0 + web: sqlalchemy >=2.0, <3.0 flask2: Flask >= 2.0, <3.0 django3: Django >=3.2, <4.0 - + sqlalchemy1: sqlalchemy >=1.4.11, <2.0 commands = poetry install -v @@ -23,6 +24,7 @@ commands = web: poetry run appmap-python {posargs:pytest} django3: poetry run appmap-python pytest _appmap/test/test_django.py flask2: poetry run appmap-python pytest _appmap/test/test_flask.py + sqlalchemy1: poetry run appmap-python pytest _appmap/test/test_sqlalchemy.py [testenv:vendoring] skip_install = True From 208ab4b043718ca7443e3cb0d0b5c9d98473540d Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Thu, 16 May 2024 06:01:17 -0400 Subject: [PATCH 010/113] refactor: clean up Config Update Config so it matches the singleton style of Env. --- _appmap/__init__.py | 3 +-- _appmap/configuration.py | 31 ++++++--------------------- _appmap/env.py | 20 +++-------------- _appmap/importer.py | 2 +- _appmap/singleton.py | 14 ++++++++++++ _appmap/test/test_configuration.py | 26 +++++++++++----------- _appmap/testing_framework.py | 2 +- appmap/command/appmap_agent_init.py | 2 +- appmap/command/appmap_agent_status.py | 2 +- 9 files changed, 41 insertions(+), 61 deletions(-) create mode 100644 _appmap/singleton.py diff --git a/_appmap/__init__.py b/_appmap/__init__.py index 4317f826..c98d0cfc 100644 --- a/_appmap/__init__.py +++ b/_appmap/__init__.py @@ -1,6 +1,5 @@ -from . import configuration +from . import configuration, event, importer, metadata, recorder, recording, web_framework from . import env as appmapenv -from . import event, importer, metadata, recorder, recording, web_framework from .py_version_check import check_py_version diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 35b3bbf1..74831bc5 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -14,6 +14,7 @@ from yaml.parser import ParserError from _appmap.labels import LabelSet +from _appmap.singleton import SingletonMeta from appmap.labeling import presets as label_presets from . import utils @@ -125,24 +126,10 @@ def excluded(d): return packages -class Config: +class Config(metaclass=SingletonMeta): """Singleton Config class""" - _instance = None - - def __new__(cls): - if cls._instance is None: - logger.trace("Creating the Config object") - cls._instance = super(Config, cls).__new__(cls) - - cls._instance._initialized = False - - return cls._instance - def __init__(self): - if self._initialized: - return - self.file_present = False self.file_valid = False self.package_functions = {} @@ -154,12 +141,6 @@ def __init__(self): if "labels" in self._config: self.labels.append(self._config["labels"]) - self._initialized = True - - @classmethod - def initialize(cls): - cls._instance = None - @property def name(self): return self._config["name"] @@ -395,7 +376,7 @@ def wrap(self, filterable): wrapped = getattr(filterable.obj, "_appmap_wrapped", None) if wrapped is None: logger.trace(" wrapping %s", filterable.fqname) - Config().labels.apply(filterable) + Config.current.labels.apply(filterable) ret = instrument(filterable) if rule and rule.shallow: setattr(ret, "_appmap_shallow", rule) @@ -428,7 +409,7 @@ class ConfigFilter(MatcherFilter): def __init__(self, *args, **kwargs): matchers = [] if Env.current.enabled: - matchers = [matcher_of_config(p) for p in Config().packages] + matchers = [matcher_of_config(p) for p in Config.current.packages] super().__init__(matchers, *args, **kwargs) @@ -441,14 +422,14 @@ def __init__(self, *args, **kwargs): def initialize(): - Config().initialize() + Config.reset() Importer.use_filter(BuiltinFilter) Importer.use_filter(ConfigFilter) initialize() -c = Config() +c = Config.current logger.info("config: %s", c._config) logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) diff --git a/_appmap/env.py b/_appmap/env.py index a4f88431..6805fa55 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import cast +from _appmap.singleton import SingletonMeta + from . import trace_logger _ENABLED_BY_DEFAULT_MSG = """ @@ -34,23 +36,7 @@ def _recording_method_key(recording_method): return f"APPMAP_RECORD_{recording_method.upper()}" -class _EnvMeta(type): - def __init__(cls, *args, **kwargs): - type.__init__(cls, *args, **kwargs) - cls._instance = None - - @property - def current(cls): - if not cls._instance: - cls._instance = Env() - - return cls._instance - - def reset(cls, **kwargs): - cls._instance = Env(**kwargs) - - -class Env(metaclass=_EnvMeta): +class Env(metaclass=SingletonMeta): def __init__(self, env=None, cwd=None): warnings.filterwarnings("once", _ENABLED_BY_DEFAULT_MSG) diff --git a/_appmap/importer.py b/_appmap/importer.py index b853c384..cecdf48a 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -189,7 +189,7 @@ def instrument_functions(filterable, selected_functions=None): # Import Config here, to avoid circular top-level imports. from .configuration import Config # pylint: disable=import-outside-toplevel - package_functions = Config().package_functions + package_functions = Config.current.package_functions fm = FilterableMod(mod) if fm.fqname in package_functions: instrument_functions(fm, package_functions.get(fm.fqname)) diff --git a/_appmap/singleton.py b/_appmap/singleton.py new file mode 100644 index 00000000..a029a4e9 --- /dev/null +++ b/_appmap/singleton.py @@ -0,0 +1,14 @@ +class SingletonMeta(type): + def __init__(cls, *args, **kwargs): + type.__init__(cls, *args, **kwargs) + cls._instance = None + + @property + def current(cls): + if not cls._instance: + cls._instance = cls() + + return cls._instance + + def reset(cls, **kwargs): + cls._instance = cls(**kwargs) \ No newline at end of file diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index dab8f4cb..e59e7f2a 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -26,7 +26,7 @@ def test_can_be_configured(): """ assert appmap.enabled() - c = Config() + c = Config.current assert c.file_present assert c.file_valid @@ -38,7 +38,7 @@ def test_reports_invalid(): indicates that the config is not valid. """ assert not appmap.enabled() - assert not Config().file_valid + assert not Config.current.file_valid @pytest.mark.appmap_enabled(config="appmap-broken.yml") @@ -62,9 +62,9 @@ def test_config_not_found(caplog): "APPMAP_CONFIG": "notfound.yml", } ) - assert Config().name is None - assert not Config().file_present - assert not Config().file_valid + assert Config.current.name is None + assert not Config.current.file_present + assert not Config.current.file_valid assert not appmap.enabled() not_found = Path("notfound.yml").resolve() @@ -80,7 +80,7 @@ def test_config_no_message(caplog): """ assert not appmap.enabled() - assert Config().name is None + assert Config.current.name is None assert caplog.text == "" @@ -129,7 +129,7 @@ def check_default_packages(self, actual_packages): def check_default_config(self, expected_name): assert appmap.enabled() - default_config = Config() + default_config = Config.current assert default_config.name == expected_name self.check_default_packages(default_config.packages) assert default_config.default["appmap_dir"] == "tmp/appmap" @@ -160,7 +160,7 @@ def test_skipped_when_overridden(self): "APPMAP_CONFIG": "/tmp/appmap.yml", } ) - assert not Config().name + assert not Config.current.name assert not appmap.enabled() def test_exclusions(self, data_dir, tmpdir, mocker, monkeypatch): @@ -186,7 +186,7 @@ def test_created_if_missing_and_enabled(self, git, data_dir, monkeypatch, tmpdir # pylint: disable=protected-access _appmap.initialize(cwd=repo_root) - Config() # write the file as a side-effect + Config.current # write the file as a side-effect assert path.is_file() with open(path, encoding="utf-8") as cfg: actual_config = yaml.safe_load(cfg) @@ -206,7 +206,7 @@ def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch # pylint: disable=protected-access _appmap.initialize(cwd=repo_root, env={"_APPMAP": "false"}) - c = Config() + c = Config.current assert not path.is_file() @@ -257,7 +257,7 @@ def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): # pylint: disable=protected-access _appmap.initialize(cwd=project_root) - assert Config().name == "config-up-name" + assert Config.current.name == "config-up-name" assert str(Env.current.output_dir).endswith(str(tmpdir / "tmp" / "appmap")) def test_config_not_found_until_repo_root(self, data_dir, tmpdir, git_directory, monkeypatch): @@ -272,7 +272,7 @@ def test_config_not_found_until_repo_root(self, data_dir, tmpdir, git_directory, # It should stop searching at repo_root. # Check that it did not find appmap.yml # in config-up folder. - assert Config().name != "config-up-name" + assert Config.current.name != "config-up-name" # It should go on with default config assert Env.current.enabled @@ -286,6 +286,6 @@ def test_config_not_found_in_path_hierarchy(self, data_dir, tmpdir, monkeypatch) cwd=project_root, env={"APPMAP_CONFIG": "notfound.yml"}, ) - Config() + Config.current # No default config since we specified APPMAP_CONFIG assert not Env.current.enabled diff --git a/_appmap/testing_framework.py b/_appmap/testing_framework.py index 77f93f88..eeffe45c 100644 --- a/_appmap/testing_framework.py +++ b/_appmap/testing_framework.py @@ -106,7 +106,7 @@ def record(self, klass, method, **kwds): metadata = item.metadata metadata.update( { - "app": configuration.Config().name, + "app": configuration.Config.current.name, "recorder": { "name": self.name, "type": self.recorder_type, diff --git a/appmap/command/appmap_agent_init.py b/appmap/command/appmap_agent_init.py index dbc780a0..397fba57 100644 --- a/appmap/command/appmap_agent_init.py +++ b/appmap/command/appmap_agent_init.py @@ -12,7 +12,7 @@ def _run(): { "configuration": { "filename": "appmap.yml", - "contents": yaml.dump(Config().default), + "contents": yaml.dump(Config.current.default), } } ) diff --git a/appmap/command/appmap_agent_status.py b/appmap/command/appmap_agent_status.py index 3d36745a..a1ca4981 100644 --- a/appmap/command/appmap_agent_status.py +++ b/appmap/command/appmap_agent_status.py @@ -68,7 +68,7 @@ def has_unittest_tests(): def _run(*, discover_tests): - config = Config() + config = Config.current uses_pytest = has_dist("pytest") has_tests = None From f7937eeffa4e690c57b3154847cc4b93c6187068 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Mon, 13 May 2024 09:56:30 +0300 Subject: [PATCH 011/113] feat: check malformed path entries --- _appmap/configuration.py | 47 ++++++++++++++++++- .../test/data/appmap-all-paths-malformed.yml | 9 ++++ _appmap/test/data/appmap-empty-path.yml | 5 ++ _appmap/test/data/appmap-malformed-path.yml | 4 ++ _appmap/test/test_configuration.py | 21 +++++++++ 5 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 _appmap/test/data/appmap-all-paths-malformed.yml create mode 100644 _appmap/test/data/appmap-empty-path.yml create mode 100644 _appmap/test/data/appmap-malformed-path.yml diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 74831bc5..d7dadf69 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -2,6 +2,7 @@ Manage Configuration AppMap recorder for Python. """ +import ast import importlib.metadata import inspect import os @@ -125,6 +126,8 @@ def excluded(d): return packages +class AppMapInvalidConfigException(Exception): + pass class Config(metaclass=SingletonMeta): """Singleton Config class""" @@ -179,7 +182,7 @@ def default_packages(self): root_dir = Env.current.root_dir return [{"path": p} for p in find_top_packages(root_dir)] - def _load_config(self): + def _load_config(self, show_warnings=False): self._config = {"name": None, "packages": []} # Only use a default config if the user hasn't specified a @@ -192,7 +195,7 @@ def _load_config(self): env = Env.current config_dir = env.root_dir - + path = _resolve_relative_to(Path(env_config_filename), Path(config_dir)) if not path.is_file(): # search config file in parent directories up to @@ -221,6 +224,8 @@ def _load_config(self): self._config["name"] = self.default_name if "packages" not in self._config: self._config["packages"] = self.default_packages + else: + self._drop_malformed_package_paths(show_warnings) # Is appmap_dir specified? appmap_dir = ( @@ -284,6 +289,43 @@ def _load_functions(self): self.package_functions.update(modules) + def _drop_malformed_package_paths(self, show_warnings): + invalid_items = [] + for item in self._config["packages"]: + # it can be a "dist" entry + if "path" not in item: + continue + + path = item.get("path") + if path is None: + if show_warnings: + logger.warning("Missing path value in configuration file.") + invalid_items.append(item) + continue + + if not self._check_path_value(path): + has_separator = isinstance(path, str) and ('/' in path or '\\' in path) + if show_warnings: + logger.warning( + f"Malformed path value '{path}' in configuration file. " + "Path entries must be module names" + f"{' not directory paths' if has_separator else ''}.", + stack_info=False, + ) + invalid_items.append(item) + continue + + if len(invalid_items) > 0: + self._config["packages"] = [item for item in self._config["packages"] + if item not in invalid_items] + + def _check_path_value(self, value): + try: + ast.parse(f"import {value}") + return True + except SyntaxError: + return False + def startswith(prefix, sequence): """ @@ -430,6 +472,7 @@ def initialize(): initialize() c = Config.current +c._load_config(show_warnings=True) logger.info("config: %s", c._config) logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) diff --git a/_appmap/test/data/appmap-all-paths-malformed.yml b/_appmap/test/data/appmap-all-paths-malformed.yml new file mode 100644 index 00000000..a962f97d --- /dev/null +++ b/_appmap/test/data/appmap-all-paths-malformed.yml @@ -0,0 +1,9 @@ +name: TestApp +packages: +- path: abc/xyz +- path: abc\xyz +- path: \abc +- path: xyz/ +- path: 42 +- path: . +- path: diff --git a/_appmap/test/data/appmap-empty-path.yml b/_appmap/test/data/appmap-empty-path.yml new file mode 100644 index 00000000..9cca7544 --- /dev/null +++ b/_appmap/test/data/appmap-empty-path.yml @@ -0,0 +1,5 @@ +name: TestApp +packages: + - path: example_class + - path: + diff --git a/_appmap/test/data/appmap-malformed-path.yml b/_appmap/test/data/appmap-malformed-path.yml new file mode 100644 index 00000000..e33dd6fc --- /dev/null +++ b/_appmap/test/data/appmap-malformed-path.yml @@ -0,0 +1,4 @@ +name: TestApp +packages: +- path: example_class +- path: package1/package2/Mod1Class diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index e59e7f2a..40a5434b 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -120,6 +120,27 @@ def test_class_prefix_doesnt_match(self): f = Filterable(None, "package1_prefix.cls", None) assert cf().filter(f) is False + def test_malformed_path(self, data_dir, caplog): + _appmap.initialize(env={"APPMAP_CONFIG": "appmap-malformed-path.yml"}, cwd=data_dir) + Config.current._load_config(show_warnings=True) + assert ( + "Malformed path value 'package1/package2/Mod1Class' in configuration file. " + "Path entries must be module names not directory paths." + in caplog.text + ) + + def test_all_paths_malformed(self, data_dir): + _appmap.initialize(env={"APPMAP_CONFIG": "appmap-all-paths-malformed.yml"}, cwd=data_dir) + assert len(Config().packages) == 0 + + def test_empty_path(self, data_dir, caplog): + _appmap.initialize(env={"APPMAP_CONFIG": "appmap-empty-path.yml"}, cwd=data_dir) + Config.current._load_config(show_warnings=True) + assert ( + "Missing path value in configuration file." + in caplog.text + ) + class DefaultHelpers: def check_default_packages(self, actual_packages): From c6928ef97cefb8d3b522d1dda7d6623538ce9f73 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 16 May 2024 12:13:24 +0000 Subject: [PATCH 012/113] chore(release): 1.23.0 [skip ci] # [1.23.0](https://github.com/getappmap/appmap-python/compare/v1.22.0...v1.23.0) (2024-05-16) ### Features * check malformed path entries ([f7937ee](https://github.com/getappmap/appmap-python/commit/f7937eeffa4e690c57b3154847cc4b93c6187068)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83cacb41..425b5366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [1.23.0](https://github.com/getappmap/appmap-python/compare/v1.22.0...v1.23.0) (2024-05-16) + + +### Features + +* check malformed path entries ([f7937ee](https://github.com/getappmap/appmap-python/commit/f7937eeffa4e690c57b3154847cc4b93c6187068)) + # [1.22.0](https://github.com/getappmap/appmap-python/compare/v1.21.0...v1.22.0) (2024-05-15) diff --git a/pyproject.toml b/pyproject.toml index 33df765a..27678631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.22.0" +version = "1.23.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From cacc62f9ba6811c45e3bebe8849ad48800be60f9 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 16 May 2024 09:50:38 +0300 Subject: [PATCH 013/113] feat: append to a single log file --- _appmap/configuration.py | 13 +++++++++---- _appmap/env.py | 35 ++++++++++++++++++++++++++--------- appmap/command/runner.py | 4 +++- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index d7dadf69..b7be26d9 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -472,7 +472,12 @@ def initialize(): initialize() c = Config.current -c._load_config(show_warnings=True) -logger.info("config: %s", c._config) -logger.debug("package_functions: %s", c.package_functions) -logger.info("env: %r", os.environ) +# For various reasons, this code runs more than once on startup. Use an +# environment variable to make sure the user only sees startup messages once. +_startup_messages_shown = os.environ.get("_APPMAP_MESSAGES_SHOWN") +if _startup_messages_shown is None: + c._load_config(show_warnings=True) + logger.info("config: %s", c._config) + logger.debug("package_functions: %s", c.package_functions) + logger.info("env: %r", os.environ) + os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" \ No newline at end of file diff --git a/_appmap/env.py b/_appmap/env.py index 6805fa55..88ab2759 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -5,7 +5,6 @@ import os import warnings from contextlib import contextmanager -from datetime import datetime from os import environ from pathlib import Path from typing import cast @@ -144,9 +143,8 @@ def _configure_logging(self): trace_logger.install() log_level = self.get("APPMAP_LOG_LEVEL", "warn").upper() - disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "true").upper() != "FALSE" + disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "false").upper() != "FALSE" log_config = self.get("APPMAP_LOG_CONFIG") - now = datetime.now() config_dict = { "version": 1, "disable_existing_loggers": False, @@ -156,16 +154,27 @@ def _configure_logging(self): "format": "[{asctime}] {levelname} {name}: {message}", } }, - "handlers": {"default": {"class": "logging.StreamHandler", "formatter": "default"}}, + "handlers": { + "default": { + "class": "logging.StreamHandler", + "formatter": "default", + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "WARNING", + "formatter": "default", + "stream": "ext://sys.stderr", + }, + }, "loggers": { "appmap": { "level": log_level, - "handlers": ["default"], + "handlers": ["default", "stderr"], "propagate": True, }, "_appmap": { "level": log_level, - "handlers": ["default"], + "handlers": ["default", "stderr"], "propagate": True, }, }, @@ -178,10 +187,18 @@ def _configure_logging(self): loggers["appmap"]["level"] = loggers["_appmap"]["level"] = log_level config_dict["handlers"] = { "default": { - "class": "logging.FileHandler", + "class": "logging.handlers.RotatingFileHandler", "formatter": "default", - "filename": f"appmap-{now:%Y%m%d%H%M%S}-{os.getpid()}.log", - } + "filename": "appmap.log", + "maxBytes": 50 * 1024 * 1024, + "backupCount": 1, + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "WARNING", + "formatter": "default", + "stream": "ext://sys.stderr", + }, } if log_config is not None: diff --git a/appmap/command/runner.py b/appmap/command/runner.py index 4fbf6c08..2841b467 100644 --- a/appmap/command/runner.py +++ b/appmap/command/runner.py @@ -119,7 +119,9 @@ def run(): for disabled in no_record: envvars[f"APPMAP_RECORD_{disabled.upper()}"] = "false" - envvars["APPMAP_DISABLE_LOG_FILE"] = "false" if parsed_args["enable_log"] else "true" + envvars["APPMAP_DISABLE_LOG_FILE"] = ( + "true" if parsed_args.get("no_enable_log", set()) else "false" + ) if len(cmd) == 0: for k, v in sorted(envvars.items()): From 0069afd85a1cd44735d93d6acdf24133942c4d9d Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 16 May 2024 09:56:52 +0300 Subject: [PATCH 014/113] refactor: extract update output dir in config load Extract update output dir part of Config._load_config to prevent pylint 'R0912: Too many branches' --- _appmap/configuration.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index b7be26d9..d26c018a 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -182,6 +182,19 @@ def default_packages(self): root_dir = Env.current.root_dir return [{"path": p} for p in find_top_packages(root_dir)] + def _update_output_dir(self, config_dir): + # appmap_dir must be resolved relative to the location of config file + # unless APPMAP_OUTPUT_DIR is set by tests. + if config_dir and Env.current.get("APPMAP_OUTPUT_DIR", None) is None: + # Is appmap_dir specified? + appmap_dir = ( + self._config["appmap_dir"] + if "appmap_dir" in self._config else "tmp/appmap" + ) + Env.current.output_dir = _resolve_relative_to( + Path(appmap_dir), Path(config_dir) + ) + def _load_config(self, show_warnings=False): self._config = {"name": None, "packages": []} @@ -227,18 +240,7 @@ def _load_config(self, show_warnings=False): else: self._drop_malformed_package_paths(show_warnings) - # Is appmap_dir specified? - appmap_dir = ( - self._config["appmap_dir"] - if "appmap_dir" in self._config else "tmp/appmap" - ) - - # appmap_dir must be resolved relative to the location of config file - # unless APPMAP_OUTPUT_DIR is set by tests. - if config_dir and Env.current.get("APPMAP_OUTPUT_DIR", None) is None: - Env.current.output_dir = _resolve_relative_to( - Path(appmap_dir), Path(config_dir) - ) + self._update_output_dir(config_dir) self.file_valid = True Env.current.enabled = should_enable From bbeee653a04df9dafb7e4b8c04db023b3f9be210 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Thu, 16 May 2024 18:25:59 -0400 Subject: [PATCH 015/113] fix: improve handling of unset APPMAP Prior to these changes, the agent still wasn't doing a very good job of letting the user what was going to happen when APPMAP was unset. Now it does. These changes also clean up the tests that check how APPMAP is handled (set to "true", set to "false", and unset). --- _appmap/env.py | 6 +++++- appmap/__init__.py | 6 +++++- tox.ini | 8 ++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/_appmap/env.py b/_appmap/env.py index 88ab2759..4de46514 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -46,7 +46,11 @@ def __init__(self, env=None, cwd=None): self._cwd = cwd or _cwd self._env = _bootenv.copy() if env: - self._env.update(env) + for k, v in env.items(): + if v is not None: + self._env[k] = v + else: + self._env.pop(k, None) self._configure_logging() enabled = self._env.get("_APPMAP", None) diff --git a/appmap/__init__.py b/appmap/__init__.py index 4f457133..7c58b247 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -5,7 +5,11 @@ # putting them in a function and conditionally calling the function. If we # execute the imports in a function, the modules all get put into the funtion's # globals, rather than into appmap's globals. -if os.environ.get("APPMAP", "true").upper() == "TRUE": +_enabled = os.environ.get("APPMAP", None) +if _enabled is None or _enabled.upper() == "TRUE": + if _enabled is not None: + # Use setdefault so tests can manage _APPMAP as necessary + os.environ["_APPMAP"] = _enabled from _appmap import generation # noqa: F401 from _appmap.env import Env # noqa: F401 from _appmap.importer import instrument_module # noqa: F401 diff --git a/tox.ini b/tox.ini index ed1c73b7..93180edf 100644 --- a/tox.ini +++ b/tox.ini @@ -21,10 +21,10 @@ deps= commands = poetry install -v py310-web: poetry run pylint -j 0 appmap _appmap - web: poetry run appmap-python {posargs:pytest} - django3: poetry run appmap-python pytest _appmap/test/test_django.py - flask2: poetry run appmap-python pytest _appmap/test/test_flask.py - sqlalchemy1: poetry run appmap-python pytest _appmap/test/test_sqlalchemy.py + web: poetry run {posargs:pytest} + django3: poetry run pytest _appmap/test/test_django.py + flask2: poetry run pytest _appmap/test/test_flask.py + sqlalchemy1: poetry run pytest _appmap/test/test_sqlalchemy.py [testenv:vendoring] skip_install = True From 01b5c03a33764c13b865004ca2025d92d4852606 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Thu, 16 May 2024 18:57:46 -0400 Subject: [PATCH 016/113] refactor: bump pylint score --- _appmap/configuration.py | 3 ++- _appmap/singleton.py | 2 +- _appmap/test/appmap_test_base.py | 1 + _appmap/test/conftest.py | 1 + _appmap/test/test_configuration.py | 6 +++--- pylintrc | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index d26c018a..13b42c6c 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -478,8 +478,9 @@ def initialize(): # environment variable to make sure the user only sees startup messages once. _startup_messages_shown = os.environ.get("_APPMAP_MESSAGES_SHOWN") if _startup_messages_shown is None: + # pylint: disable=protected-access c._load_config(show_warnings=True) logger.info("config: %s", c._config) logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) - os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" \ No newline at end of file + os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" diff --git a/_appmap/singleton.py b/_appmap/singleton.py index a029a4e9..e8c275c4 100644 --- a/_appmap/singleton.py +++ b/_appmap/singleton.py @@ -11,4 +11,4 @@ def current(cls): return cls._instance def reset(cls, **kwargs): - cls._instance = cls(**kwargs) \ No newline at end of file + cls._instance = cls(**kwargs) diff --git a/_appmap/test/appmap_test_base.py b/_appmap/test/appmap_test_base.py index 161bce39..550c2f21 100644 --- a/_appmap/test/appmap_test_base.py +++ b/_appmap/test/appmap_test_base.py @@ -25,6 +25,7 @@ def setup_method(self, _): @staticmethod @pytest.fixture def events(): + # pylint: disable=protected-access rec = Recorder.get_current() rec.clear() rec._enabled = True diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index e2dee18e..939f8eca 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -38,6 +38,7 @@ def fixture_with_data_dir(data_dir, monkeypatch): @pytest.fixture def events(): + # pylint: disable=protected-access rec = Recorder.get_current() rec.clear() rec._enabled = True diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index 40a5434b..2100439e 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -122,7 +122,7 @@ def test_class_prefix_doesnt_match(self): def test_malformed_path(self, data_dir, caplog): _appmap.initialize(env={"APPMAP_CONFIG": "appmap-malformed-path.yml"}, cwd=data_dir) - Config.current._load_config(show_warnings=True) + Config.current._load_config(show_warnings=True) # pylint: disable=protected-access assert ( "Malformed path value 'package1/package2/Mod1Class' in configuration file. " "Path entries must be module names not directory paths." @@ -135,7 +135,7 @@ def test_all_paths_malformed(self, data_dir): def test_empty_path(self, data_dir, caplog): _appmap.initialize(env={"APPMAP_CONFIG": "appmap-empty-path.yml"}, cwd=data_dir) - Config.current._load_config(show_warnings=True) + Config.current._load_config(show_warnings=True) # pylint: disable=protected-access assert ( "Missing path value in configuration file." in caplog.text @@ -227,7 +227,7 @@ def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch # pylint: disable=protected-access _appmap.initialize(cwd=repo_root, env={"_APPMAP": "false"}) - c = Config.current + Config.current assert not path.is_file() diff --git a/pylintrc b/pylintrc index 06d340d4..3bd9b874 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,6 @@ [MAIN] # Specify a score threshold under which the program will exit with error. -fail-under=9.83 +fail-under=9.85 # Analyse import fallback blocks. This can be used to support both Python 2 and From 017532949c6e0f8145b0a5d4c7e9c2c0dffb4e76 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 May 2024 13:27:12 +0000 Subject: [PATCH 017/113] chore(release): 1.24.0 [skip ci] # [1.24.0](https://github.com/getappmap/appmap-python/compare/v1.23.0...v1.24.0) (2024-05-17) ### Bug Fixes * improve handling of unset APPMAP ([bbeee65](https://github.com/getappmap/appmap-python/commit/bbeee653a04df9dafb7e4b8c04db023b3f9be210)) ### Features * append to a single log file ([cacc62f](https://github.com/getappmap/appmap-python/commit/cacc62f9ba6811c45e3bebe8849ad48800be60f9)) --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425b5366..5943cf92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# [1.24.0](https://github.com/getappmap/appmap-python/compare/v1.23.0...v1.24.0) (2024-05-17) + + +### Bug Fixes + +* improve handling of unset APPMAP ([bbeee65](https://github.com/getappmap/appmap-python/commit/bbeee653a04df9dafb7e4b8c04db023b3f9be210)) + + +### Features + +* append to a single log file ([cacc62f](https://github.com/getappmap/appmap-python/commit/cacc62f9ba6811c45e3bebe8849ad48800be60f9)) + # [1.23.0](https://github.com/getappmap/appmap-python/compare/v1.22.0...v1.23.0) (2024-05-16) diff --git a/pyproject.toml b/pyproject.toml index 27678631..3119d75a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.23.0" +version = "1.24.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From b9ecced9407e59a302750615be66b08ad679ddb4 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 17 May 2024 17:13:30 -0400 Subject: [PATCH 018/113] fix: find a config in the repo root Make sure a config that exists in the repo root will be found in any subdirectory, including an immediate child. --- _appmap/configuration.py | 2 + .../{subprojects => project}/p1/__init__.py | 0 .../p2/sub1/__init__.py | 0 _appmap/test/test_configuration.py | 63 ++++++++++++++----- _appmap/utils.py | 19 +++--- requirements-dev.txt | 3 +- 6 files changed, 61 insertions(+), 26 deletions(-) rename _appmap/test/data/config-up/{subprojects => project}/p1/__init__.py (100%) rename _appmap/test/data/config-up/{subprojects => project}/p2/sub1/__init__.py (100%) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 13b42c6c..13973c2d 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -221,6 +221,7 @@ def _load_config(self, show_warnings=False): path = _resolve_relative_to(Path(env_config_filename), Path(config_dir)) if path.is_file(): + self._file = path self.file_present = True should_enable = Env.current.enabled @@ -480,6 +481,7 @@ def initialize(): if _startup_messages_shown is None: # pylint: disable=protected-access c._load_config(show_warnings=True) + logger.info("file: %s", c._file) logger.info("config: %s", c._config) logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) diff --git a/_appmap/test/data/config-up/subprojects/p1/__init__.py b/_appmap/test/data/config-up/project/p1/__init__.py similarity index 100% rename from _appmap/test/data/config-up/subprojects/p1/__init__.py rename to _appmap/test/data/config-up/project/p1/__init__.py diff --git a/_appmap/test/data/config-up/subprojects/p2/sub1/__init__.py b/_appmap/test/data/config-up/project/p2/sub1/__init__.py similarity index 100% rename from _appmap/test/data/config-up/subprojects/p2/sub1/__init__.py rename to _appmap/test/data/config-up/project/p2/sub1/__init__.py diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index 2100439e..bb348a3c 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from distutils.dir_util import copy_tree from pathlib import Path +from textwrap import dedent import pytest import yaml @@ -271,40 +272,72 @@ def test_missing_packages(self, tmpdir): self.check_default_config(Path(tmpdir).name) class TestSearchConfig: + # pylint: disable=too-many-arguments + def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): copy_tree(data_dir / "config-up", str(tmpdir)) - project_root = tmpdir / "subprojects" / "p1" - monkeypatch.chdir(project_root) + wd = tmpdir / "project" / "p1" + monkeypatch.chdir(wd) # pylint: disable=protected-access - _appmap.initialize(cwd=project_root) + _appmap.initialize(cwd=wd) assert Config.current.name == "config-up-name" assert str(Env.current.output_dir).endswith(str(tmpdir / "tmp" / "appmap")) - def test_config_not_found_until_repo_root(self, data_dir, tmpdir, git_directory, monkeypatch): + def _init_repo(self, data_dir, tmpdir, git_directory, repo_root, appmapdir): copy_tree(data_dir / "config-up", str(tmpdir)) - repo_root = tmpdir / "subprojects" / "p2" copy_tree(git_directory, str(repo_root)) - project_root = repo_root / "sub1" - monkeypatch.chdir(project_root) + with open(appmapdir / "appmap.yml", "w+", encoding="utf-8") as f: + f.writelines( + dedent(""" + name: project + packages: [] + """) + ) + + @pytest.mark.parametrize("subdir", [Path("p1"), Path("p2", "sub1")]) + def test_config_in_repo_root(self, data_dir, tmpdir, git_directory, monkeypatch, subdir): + repo_root = tmpdir / "project" + self._init_repo(data_dir, tmpdir, git_directory, repo_root, repo_root) + + wd = repo_root / subdir + monkeypatch.chdir(wd) # pylint: disable=protected-access - _appmap.initialize(cwd=project_root) - # It should stop searching at repo_root. - # Check that it did not find appmap.yml - # in config-up folder. - assert Config.current.name != "config-up-name" + _appmap.initialize(cwd=wd) + + # There's a config in the repo root. It should have been found, and have + # the correct contents. + assert Config.current.file_present + assert Config.current.name == "project" + + assert Env.current.enabled + + @pytest.mark.parametrize("subdir", [Path("p1"), Path("p2", "sub1")]) + def test_config_above_repo_root(self, data_dir, tmpdir, git_directory, monkeypatch, subdir): + repo_root = tmpdir / "project" + self._init_repo(data_dir, tmpdir, git_directory, repo_root, tmpdir) + + wd = repo_root / subdir + monkeypatch.chdir(wd) + + # pylint: disable=protected-access + _appmap.initialize(cwd=wd) + + # We should have stopped at the repo root without finding a config. + assert not Config.current.file_present + # It should go on with default config assert Env.current.enabled def test_config_not_found_in_path_hierarchy(self, data_dir, tmpdir, monkeypatch): copy_tree(data_dir / "config-up", str(tmpdir)) - project_root = tmpdir / "subprojects" / "p1" - monkeypatch.chdir(project_root) + wd = tmpdir / "project" / "p1" + monkeypatch.chdir(wd) # pylint: disable=protected-access _appmap.initialize( - cwd=project_root, + cwd=wd, env={"APPMAP_CONFIG": "notfound.yml"}, ) Config.current diff --git a/_appmap/utils.py b/_appmap/utils.py index 8defa962..7ac193a6 100644 --- a/_appmap/utils.py +++ b/_appmap/utils.py @@ -1,6 +1,5 @@ import inspect import os -from pathlib import Path import re import shlex import subprocess @@ -8,6 +7,7 @@ from contextlib import contextmanager from contextvars import ContextVar from enum import Enum, IntFlag, auto +from pathlib import Path from typing import Any, Callable from .env import Env @@ -228,10 +228,10 @@ def scenario_filename(name, separator="_"): def locate_file_up(filename, start_dir=None, stop_dir=None): """ Search for a file in the current directory and recursively up to the root directory. - + :param filename: The name of the file to locate. :param start_dir: The directory to start the search from. Defaults to the current. - :param stop_idr: The directory to stop the search. If None search is performed until + :param stop_dir: The directory to stop the search. If None search is performed until the root of the file system. :return: The path to the directory containing the file or None if the file cannot be found. """ @@ -245,11 +245,10 @@ def locate_file_up(filename, start_dir=None, stop_dir=None): if Path.exists(file_path): return start_dir - if isinstance(stop_dir, str): - stop_dir = Path(stop_dir) - - parent_dir = start_dir.parent - if parent_dir in (start_dir, stop_dir): - return None + for p in start_dir.parents: + if Path.exists(p.joinpath(filename)): + return p + if p == stop_dir: + return None - return locate_file_up(filename, parent_dir, stop_dir) + return None diff --git a/requirements-dev.txt b/requirements-dev.txt index bc06c81f..2210a524 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,5 @@ flask >=2, <= 3 pytest-django<4.8 fastapi httpx -sqlalchemy \ No newline at end of file +sqlalchemy +debugpy \ No newline at end of file From 0542f976f1a42d1a30cb70d924b2fd086a84fabe Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 28 Feb 2024 12:52:51 -0500 Subject: [PATCH 019/113] test: GH action for linting Break linting out into a separate tox environment. Add a GitHub action config to run it, to make it more obvious that it's linting that's causing the build to fail. Also, do a little more clean up to bump the pylint score. --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++++ pyproject.toml | 3 +-- tox.ini | 27 ++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..5d2fd055 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: Build +on: + pull_request: + schedule: + - cron: "0 0 * * 0" + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + python: ["3.12"] + include: + - python: "3.12" + tox_env: "lint" + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install tox + run: | + python -m pip install --upgrade pip setuptools + pip install tox + - name: Test + run: | + tox -e ${{ matrix.tox_env }} + diff --git a/pyproject.toml b/pyproject.toml index 3119d75a..5deead8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,8 +60,7 @@ httpretty = "^1.0.5" isort = "^5.10.1" pprintpp = ">=0.4.0" pyfakefs = "^5.3.5" -pylint = "^2.6.0" -pylint-exit = "^1.2.0" +pylint = "^3.0" pytest = "^7.3.2" pytest-django = "~4.7" pytest-mock = "^3.5.1" diff --git a/tox.ini b/tox.ini index 93180edf..3fe7c822 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,13 @@ isolated_build = true # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{8,9,10,11,12}-{web,django3,flask2,sqlalchemy1} +envlist = py3{8,9,10,11,12}-{web,django3,flask2,sqlalchemy1},lint + +[web-deps] +deps= + Django >=4.0, <5.0 + Flask >=3.0 + sqlalchemy >=2.0, <3.0 [testenv] allowlist_externals = @@ -11,21 +17,32 @@ allowlist_externals = deps= poetry - web: Django >=4.0, <5.0 - web: Flask >=3.0 - web: sqlalchemy >=2.0, <3.0 + web: {[web-deps]deps} flask2: Flask >= 2.0, <3.0 django3: Django >=3.2, <4.0 sqlalchemy1: sqlalchemy >=1.4.11, <2.0 commands = poetry install -v - py310-web: poetry run pylint -j 0 appmap _appmap web: poetry run {posargs:pytest} django3: poetry run pytest _appmap/test/test_django.py flask2: poetry run pytest _appmap/test/test_flask.py sqlalchemy1: poetry run pytest _appmap/test/test_sqlalchemy.py +[testenv:lint] +setenv = + APPMAP=false +skip_install = True +deps = + poetry + {[web-deps]deps} +commands = + poetry install + # It doesn't seem great to disable cyclic-import checking, but the imports + # aren't currently causing any problems. They should probably get fixed + # sometime soon. + {posargs:poetry run pylint --disable=cyclic-import -j 0 appmap _appmap} + [testenv:vendoring] skip_install = True deps = vendoring From 2849c2b90baa366a9b3338900fcda79edda3a32c Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sat, 18 May 2024 06:40:49 -0400 Subject: [PATCH 020/113] refactor: bump pylint score --- _appmap/configuration.py | 1 + _appmap/event.py | 3 +++ _appmap/test/data/trial/test/test_deferred.py | 2 +- .../test/data/unittest/simple/test_simple.py | 4 ++-- _appmap/test/normalize.py | 1 + _appmap/test/test_command.py | 24 +++++++++++-------- _appmap/test/test_configuration.py | 8 +++---- _appmap/test/test_describe_value.py | 2 +- _appmap/test/test_django.py | 3 ++- _appmap/test/test_env.py | 4 ++-- _appmap/test/test_http.py | 4 ++-- _appmap/test/test_util.py | 3 +-- _appmap/test/web_framework.py | 1 + _appmap/web_framework.py | 3 +++ appmap/http.py | 2 +- appmap/labeling/__init__.py | 2 +- appmap/pytest.py | 4 ++-- appmap/unittest.py | 3 ++- conftest.py | 3 --- pylintrc | 5 ++-- 20 files changed, 47 insertions(+), 35 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 13973c2d..3d255ce6 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -196,6 +196,7 @@ def _update_output_dir(self, config_dir): ) def _load_config(self, show_warnings=False): + # pylint: disable=too-many-branches self._config = {"name": None, "packages": []} # Only use a default config if the user hasn't specified a diff --git a/_appmap/event.py b/_appmap/event.py index 0b1964c1..4af73cbf 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -173,6 +173,7 @@ def to_dict(self, value): class CallEvent(Event): + # pylint: disable=method-cache-max-size-none __slots__ = ["_fn", "_fqfn", "static", "receiver", "parameters", "labels"] @staticmethod @@ -217,6 +218,7 @@ def make_params(filterable): @staticmethod def set_params(params, instance, args, kwargs): + # pylint: disable=too-many-branches # Note that set_params expects args and kwargs as a tuple and # dict, respectively. It operates on them as collections, so # it doesn't unpack them. @@ -400,6 +402,7 @@ class HttpServerRequestEvent(MessageEvent): __slots__ = ["http_server_request"] + # pylint: disable=too-many-arguments def __init__( self, request_method, diff --git a/_appmap/test/data/trial/test/test_deferred.py b/_appmap/test/data/trial/test/test_deferred.py index edc7d8b9..9d8b4df4 100644 --- a/_appmap/test/data/trial/test/test_deferred.py +++ b/_appmap/test/data/trial/test/test_deferred.py @@ -1,4 +1,4 @@ -import time +import time # noqa: F401 from twisted.internet import defer, reactor from twisted.trial import unittest diff --git a/_appmap/test/data/unittest/simple/test_simple.py b/_appmap/test/data/unittest/simple/test_simple.py index 5267ee78..264047a6 100644 --- a/_appmap/test/data/unittest/simple/test_simple.py +++ b/_appmap/test/data/unittest/simple/test_simple.py @@ -1,11 +1,11 @@ import unittest from unittest.mock import patch -import simple +import simple # isort: skip # Importing from decouple will cause a failure if we're not hooking # finders correctly. -from decouple import config +from decouple import config # noqa: F401 import appmap diff --git a/_appmap/test/normalize.py b/_appmap/test/normalize.py index 77f72a71..7207d3b4 100644 --- a/_appmap/test/normalize.py +++ b/_appmap/test/normalize.py @@ -67,6 +67,7 @@ def normalize_appmap(generated_appmap): """ def normalize(dct): + # pylint: disable=too-many-branches if "classMap" in dct: dct["classMap"].sort(key=itemgetter("name")) if "children" in dct: diff --git a/_appmap/test/test_command.py b/_appmap/test/test_command.py index fbadff34..6da6abc6 100644 --- a/_appmap/test/test_command.py +++ b/_appmap/test/test_command.py @@ -11,7 +11,7 @@ from .helpers import DictIncluding -@pytest.fixture(name="cmd_setup") +@pytest.fixture(name="_cmd_setup") def _cmd_setup(request, git, data_dir, monkeypatch): repo_root = git.cwd copy_tree(data_dir / request.param, str(repo_root)) @@ -23,8 +23,8 @@ def _cmd_setup(request, git, data_dir, monkeypatch): return monkeypatch -@pytest.mark.parametrize("cmd_setup", ["config"], indirect=True) -def test_agent_init(cmd_setup, capsys): +@pytest.mark.parametrize("_cmd_setup", ["config"], indirect=True) +def test_agent_init(_cmd_setup, capsys): rc = appmap_agent_init._run() # pylint: disable=protected-access assert rc == 0 @@ -39,19 +39,23 @@ def test_agent_init(cmd_setup, capsys): class TestAgentStatus: - @pytest.mark.parametrize("cmd_setup", ["pytest"], indirect=True) + @pytest.mark.parametrize("_cmd_setup", ["pytest"], indirect=True) @pytest.mark.parametrize("do_discovery", [True, False]) - def test_test_discovery_control(self, cmd_setup, do_discovery, mocker): + def test_test_discovery_control(self, _cmd_setup, do_discovery, mocker): mocker.patch("appmap.command.appmap_agent_status.discover_pytest_tests") rc = appmap_agent_status._run( # pylint: disable=protected-access discover_tests=do_discovery ) assert rc == 0 call_count = 1 if do_discovery else 0 + + # Well, pylint, if it didn't have call_count, assertion would fail, + # wouldn't it? + # pylint: disable=no-member assert appmap_agent_status.discover_pytest_tests.call_count == call_count - @pytest.mark.parametrize("cmd_setup", ["pytest"], indirect=True) - def test_agent_status(self, cmd_setup, capsys): + @pytest.mark.parametrize("_cmd_setup", ["pytest"], indirect=True) + def test_agent_status(self, _cmd_setup, capsys): rc = appmap_agent_status._run(discover_tests=True) # pylint: disable=protected-access assert rc == 0 @@ -77,8 +81,8 @@ def test_agent_status(self, cmd_setup, capsys): {"args": [], "framework": "pytest", "command": "pytest"} ) - @pytest.mark.parametrize("cmd_setup", ["package1"], indirect=True) - def test_agent_status_no_commands(self, cmd_setup, capsys): + @pytest.mark.parametrize("_cmd_setup", ["package1"], indirect=True) + def test_agent_status_no_commands(self, _cmd_setup, capsys): rc = appmap_agent_status._run(discover_tests=True) # pylint: disable=protected-access assert rc == 0 @@ -103,7 +107,7 @@ def check_errors(self, capsys, status, count, msg): assert err["level"] == "error" assert re.match(msg, err["message"]) is not None - def test_no_errors(self, capsys, mocker): + def test_no_errors(self, capsys): # Both Django and flask are installed in a dev environment, so # validation will succeed. self.check_errors(capsys, 0, 0, None) diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index bb348a3c..71b26a8a 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -208,7 +208,7 @@ def test_created_if_missing_and_enabled(self, git, data_dir, monkeypatch, tmpdir # pylint: disable=protected-access _appmap.initialize(cwd=repo_root) - Config.current # write the file as a side-effect + Config.current # pylint: disable=pointless-statement assert path.is_file() with open(path, encoding="utf-8") as cfg: actual_config = yaml.safe_load(cfg) @@ -228,7 +228,7 @@ def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch # pylint: disable=protected-access _appmap.initialize(cwd=repo_root, env={"_APPMAP": "false"}) - Config.current + Config.current # pylint: disable=pointless-statement assert not path.is_file() @@ -241,7 +241,7 @@ def setup_config(self, data_dir, monkeypatch, tmpdir): @contextmanager def incomplete_config(self): # pylint: disable=protected-access - with open("appmap-incomplete.yml", mode="w", buffering=1) as f: + with open("appmap-incomplete.yml", mode="w", buffering=1, encoding="utf-8") as f: print("# Incomplete file", file=f) yield f @@ -340,6 +340,6 @@ def test_config_not_found_in_path_hierarchy(self, data_dir, tmpdir, monkeypatch) cwd=wd, env={"APPMAP_CONFIG": "notfound.yml"}, ) - Config.current + Config.current # pylint: disable=pointless-statement # No default config since we specified APPMAP_CONFIG assert not Env.current.enabled diff --git a/_appmap/test/test_describe_value.py b/_appmap/test/test_describe_value.py index 9d790774..fe5706d3 100644 --- a/_appmap/test/test_describe_value.py +++ b/_appmap/test/test_describe_value.py @@ -13,7 +13,7 @@ class WithOverloadedClass: # pylint: disable=missing-class-docstring,too-few-public-methods @property def __class__(self): - raise Exception("__class__ called") + raise RuntimeError("__class__ called") describe_value(None, WithOverloadedClass()) diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index bc1f40c8..d26bb73a 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -32,7 +32,8 @@ sys.path += [str(Path(__file__).parent / "data" / "django")] # Import app just for the side-effects. It must happen after sys.path has been modified. -import djangoapp # pyright: ignore pylint: disable=import-error, unused-import,wrong-import-order,wrong-import-position +# pylint: disable=import-error, unused-import,wrong-import-order,wrong-import-position +import djangoapp # pyright: ignore # noqa: F401 class TestFormCapture(_TestFormCapture): diff --git a/_appmap/test/test_env.py b/_appmap/test/test_env.py index f961ae7a..ff212fb7 100644 --- a/_appmap/test/test_env.py +++ b/_appmap/test/test_env.py @@ -7,7 +7,7 @@ def test_disable_temporarily(): try: with env.disabled("requests"): assert not env.enables("requests") - raise 'hell' - except: + raise RuntimeError("hell") + except RuntimeError: ... assert env.enables("requests") diff --git a/_appmap/test/test_http.py b/_appmap/test/test_http.py index fdecccc4..b0cc048b 100644 --- a/_appmap/test/test_http.py +++ b/_appmap/test/test_http.py @@ -4,13 +4,13 @@ import pytest import requests -import appmap.http +import appmap.http # noqa: F401 from ..test.helpers import DictIncluding def test_http_client_capture(mock_requests, events): - requests.get("https://example.test/foo/bar?q=one&q=two&q2=%F0%9F%A6%A0") + requests.get("https://example.test/foo/bar?q=one&q=two&q2=%F0%9F%A6%A0", timeout=1) assert events[0].to_dict() == DictIncluding( { diff --git a/_appmap/test/test_util.py b/_appmap/test/test_util.py index 466bb6e0..25c45592 100644 --- a/_appmap/test/test_util.py +++ b/_appmap/test/test_util.py @@ -2,9 +2,8 @@ Test util functionality """ -import os -from pathlib import Path import uuid +from pathlib import Path from _appmap.utils import locate_file_up, scenario_filename diff --git a/_appmap/test/web_framework.py b/_appmap/test/web_framework.py index 7e7185ba..699b4dfa 100644 --- a/_appmap/test/web_framework.py +++ b/_appmap/test/web_framework.py @@ -339,6 +339,7 @@ def record_request_thread(cls): return requests.get(cls.server_url() + "/test", timeout=30) def record_requests(self, record_remote): + # pylint: disable=too-many-locals if record_remote: # when remote recording is enabled, this test also # verifies the global recorder doesn't save duplicate diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index 4bb25078..9034f474 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -102,6 +102,7 @@ def name_hash(namepart): return sha256(os.fsencode(namepart)).hexdigest() +# pylint: disable=too-many-arguments def create_appmap_file( output_dir, request_method, @@ -141,6 +142,7 @@ def before_request_main(self, rec, req: Any) -> Tuple[float, int]: """Specify the main operations to be performed by a request is processed.""" raise NotImplementedError + # pylint: disable=too-many-arguments def after_request_main(self, rec, status, headers, start, call_event_id) -> None: duration = time.monotonic() - start @@ -179,6 +181,7 @@ def before_request_hook(self, request) -> Tuple[Optional[Recorder], float, int]: return rec, start, call_event_id + # pylint: disable=too-many-arguments def after_request_hook( self, request_path, diff --git a/appmap/http.py b/appmap/http.py index 3e5c4273..81d9fc9a 100644 --- a/appmap/http.py +++ b/appmap/http.py @@ -52,7 +52,7 @@ def putheader(self, orig, header, *values): if not hasattr(request, "headers"): request["headers"] = {} headers = request["headers"] - if not header in headers: + if header not in headers: headers[header] = [] headers[header].extend(values) orig(self, header, *values) diff --git a/appmap/labeling/__init__.py b/appmap/labeling/__init__.py index 2d80994d..73b64571 100644 --- a/appmap/labeling/__init__.py +++ b/appmap/labeling/__init__.py @@ -8,7 +8,7 @@ import yaml from importlib_resources import files -from _appmap.labels import LabelSet +from _appmap.labels import LabelSet # noqa: F401 @lru_cache(maxsize=None) diff --git a/appmap/pytest.py b/appmap/pytest.py index e25547a0..14c8d9f9 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -26,7 +26,7 @@ def __call__(self, wrapped, _, args, kwargs): logger.debug("Test recording is enabled (Pytest)") @pytest.hookimpl - def pytest_configure(config): + def pytest_configure(config): # pylint: disable=unused-argument Env.current.warn_enabled_by_default() @pytest.hookimpl @@ -84,5 +84,5 @@ def pytest_pyfunc_call(pyfuncitem): try: with testing_framework.collect_result_metadata(metadata): result.get_result() - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 pass # exception got recorded in metadata diff --git a/appmap/unittest.py b/appmap/unittest.py index 368b3089..6734cf9c 100644 --- a/appmap/unittest.py +++ b/appmap/unittest.py @@ -4,4 +4,5 @@ if not Env.current.is_appmap_repo and Env.current.enables("unittest"): logger.debug("Test recording is enabled (unittest)") - import _appmap.unittest # pyright: ignore pylint: disable=unused-import + # pylint: disable=unused-import + import _appmap.unittest # pyright: ignore # noqa: F401 diff --git a/conftest.py b/conftest.py index 5bae8ebc..01f0ad16 100644 --- a/conftest.py +++ b/conftest.py @@ -1,7 +1,4 @@ import os -import sys - -import pytest collect_ignore = [os.path.join("_appmap", "test", "data")] pytest_plugins = ["pytester"] diff --git a/pylintrc b/pylintrc index 3bd9b874..b8959689 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,6 @@ [MAIN] # Specify a score threshold under which the program will exit with error. -fail-under=9.85 +fail-under=9.86 # Analyse import fallback blocks. This can be used to support both Python 2 and @@ -429,7 +429,8 @@ disable=raw-checker-failed, missing-class-docstring, missing-module-docstring, consider-using-f-string, - fixme + fixme, + similarities # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From e4b8d78638e97527d30605643ddd2903aa063d55 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 20 May 2024 11:26:14 +0000 Subject: [PATCH 021/113] chore(release): 1.24.1 [skip ci] ## [1.24.1](https://github.com/getappmap/appmap-python/compare/v1.24.0...v1.24.1) (2024-05-20) ### Bug Fixes * find a config in the repo root ([b9ecced](https://github.com/getappmap/appmap-python/commit/b9ecced9407e59a302750615be66b08ad679ddb4)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5943cf92..bae4ba62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.24.1](https://github.com/getappmap/appmap-python/compare/v1.24.0...v1.24.1) (2024-05-20) + + +### Bug Fixes + +* find a config in the repo root ([b9ecced](https://github.com/getappmap/appmap-python/commit/b9ecced9407e59a302750615be66b08ad679ddb4)) + # [1.24.0](https://github.com/getappmap/appmap-python/compare/v1.23.0...v1.24.0) (2024-05-17) diff --git a/pyproject.toml b/pyproject.toml index 5deead8f..b41aef6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.24.0" +version = "1.24.1" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 2df0f37474d1cd26bdfdbb45baf4fd2c9c9c982f Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sun, 12 May 2024 06:24:54 -0400 Subject: [PATCH 022/113] fix: honor APPMAP_RECORD_REQUESTS when testing Make sure `APPMAP_RECORD_REQUESTS=true` will generate request recordings when running tests. Previously, it was ignored, and request recordings were never generated when tests were run. --- _appmap/env.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_appmap/env.py b/_appmap/env.py index 4de46514..899c4c03 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -73,6 +73,9 @@ def __init__(self, env=None, cwd=None): def set(self, name, value): self._env[name] = value + def setdefault(self, name, default_value): + self._env.setdefault(name, default_value) + def get(self, name, default=None): return self._env.get(name, default) @@ -122,7 +125,7 @@ def enables(self, recording_method, default="true"): def disabled(self, recording_method: str): key = _recording_method_key(recording_method) value = self.get(key) - self.set(key, "false") + self.setdefault(key, "false") try: yield finally: From 500fe55f06c536611e3e292b22a8fade62101afe Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sun, 12 May 2024 07:49:45 -0400 Subject: [PATCH 023/113] fix: combine testing-related env vars Combine APPMAP_RECORD_PYTEST and APPMAP_RECORD_UNITTEST into APPMAP_RECORD_TESTS. --- _appmap/test/test_runner.py | 2 +- _appmap/test/test_test_frameworks.py | 2 +- appmap/command/runner.py | 3 +-- appmap/pytest.py | 2 +- appmap/unittest.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/_appmap/test/test_runner.py b/_appmap/test/test_runner.py index 176c9401..140df2e2 100644 --- a/_appmap/test/test_runner.py +++ b/_appmap/test/test_runner.py @@ -15,7 +15,7 @@ def test_runner_help(script_runner): assert result.stdout.startswith("usage") -@pytest.mark.parametrize("recording_type", ["process", "pytest", "remote", "requests", "unittest"]) +@pytest.mark.parametrize("recording_type", ["process", "remote", "requests", "tests"]) def test_runner_recording_type(script_runner, recording_type): result = script_runner.run(["appmap-python", "--record", recording_type]) assert result.returncode == 0 diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 6cd5e2f1..fbdfb9ec 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -39,7 +39,7 @@ def test_with_appmap_false(self, testdir, monkeypatch): assert not testdir.output().exists() def test_disabled(self, testdir, monkeypatch): - monkeypatch.setenv(f"APPMAP_RECORD_{self._test_type.upper()}", "false") + monkeypatch.setenv("APPMAP_RECORD_TESTS", "false") self.run_tests(testdir) assert not testdir.output().exists() diff --git a/appmap/command/runner.py b/appmap/command/runner.py index 2841b467..20bd5614 100644 --- a/appmap/command/runner.py +++ b/appmap/command/runner.py @@ -24,10 +24,9 @@ _RECORDING_TYPES = set( [ "process", - "pytest", "remote", "requests", - "unittest", + "tests", ] ) diff --git a/appmap/pytest.py b/appmap/pytest.py index 14c8d9f9..5638e063 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -22,7 +22,7 @@ def __call__(self, wrapped, _, args, kwargs): return wrapped(*args, **kwargs) -if not Env.current.is_appmap_repo and Env.current.enables("pytest"): +if not Env.current.is_appmap_repo and Env.current.enables("tests"): logger.debug("Test recording is enabled (Pytest)") @pytest.hookimpl diff --git a/appmap/unittest.py b/appmap/unittest.py index 6734cf9c..79951d95 100644 --- a/appmap/unittest.py +++ b/appmap/unittest.py @@ -2,7 +2,7 @@ logger = Env.current.getLogger(__name__) -if not Env.current.is_appmap_repo and Env.current.enables("unittest"): +if not Env.current.is_appmap_repo and Env.current.enables("tests"): logger.debug("Test recording is enabled (unittest)") # pylint: disable=unused-import import _appmap.unittest # pyright: ignore # noqa: F401 From aa89150970ed94d4c2819f8bbe1ab84c1e80c705 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 13 May 2024 16:55:55 -0400 Subject: [PATCH 024/113] refactor: rename doc to docs Also, fix a typo. --- {doc => docs}/recording-env-vars.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename {doc => docs}/recording-env-vars.md (81%) diff --git a/doc/recording-env-vars.md b/docs/recording-env-vars.md similarity index 81% rename from doc/recording-env-vars.md rename to docs/recording-env-vars.md index 92c5297c..89ba7df0 100644 --- a/doc/recording-env-vars.md +++ b/docs/recording-env-vars.md @@ -3,11 +3,11 @@ recording types. In each case, ✓ means that the corresponding recording type will be produced, ❌ means that it will not. ## Web Apps -These tables describe how `APPMAP_RECORD_REQUEST` and `APPMAP_RECORD_REMOTE` are +These tables describe how `APPMAP_RECORD_REQUESTS` and `APPMAP_RECORD_REMOTE` are handled when running a web app. "web app, debug on" means a Flask app run as `flask --debug`, a FastAPI app run using `uvicorn --reload` and, a Django app run with `DEBUG = True` in `settings.py`. -| | `APPMAP_RECORD_REQUEST` is unset | `APPMAP_RECORD_REQUEST` == "true" | `APPMAP_RECORD_REQUEST` == "false" | +| | `APPMAP_RECORD_REQUESTS` is unset | `APPMAP_RECORD_REQUESTS` == "true" | `APPMAP_RECORD_REQUESTS` == "false" | | -------------------- | :----------------------------: | :------------------------------: | :-------------------------------: | | "web app, debug on" | ✓ | ✓ | ❌ | | "web app, debug off" | ✓ | ✓ | ❌ | @@ -21,13 +21,13 @@ a FastAPI app run using `uvicorn --reload` and, a Django app run with `DEBUG = T ## Testing This table shows how `APPMAP_RECORD_PYTEST`, `APPMAP_RECORD_UNITTEST`, and -`APPMAP_RECORD_REQUEST` are handled when running tests in. Note that in v2, in +`APPMAP_RECORD_REQUESTS` are handled when running tests in. Note that in v2, in v2, `APPMAP_RECORD_PYTEST` and `APPMAP_RECORD_UNITTEST` will be replaced with -APPMAP_RECORD_TEST. +`APPMAP_RECORD_TESTS`. -| | `APPMAP_RECORD_PYTEST` is unset | `APPMAP_RECORD_PYTEST` == "true" | `APPMAP_RECORD_PYTEST` == "false" | `APPMAP_RECORD_REQUEST` is unset | `APPMAP_RECORD_REQUEST` == "true" | `APPMAP_RECORD_REQUEST` == "false" | +| | `APPMAP_RECORD_PYTEST` is unset | `APPMAP_RECORD_PYTEST` == "true" | `APPMAP_RECORD_PYTEST` == "false" | `APPMAP_RECORD_REQUESTS` is unset | `APPMAP_RECORD_REQUESTS` == "true" | `APPMAP_RECORD_REQUESTS` == "false" | | ------ | :---------------------------: | :-----------------------------: | :------------------------------: | :----------------------------: | :-----------------------------: | :------------------------------: | -| pytest | ✓ | ✓ | ❌ | ✓in v1, ❌ in v2 | ✓ | ❌ | +| pytest | ✓ | ✓ | ❌ | ❌ | ignored in v1, ✓ in v2 | ❌ | From 57b3910a48cea8582612772d79abacc53b5b73d5 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 13 May 2024 16:58:14 -0400 Subject: [PATCH 025/113] feat: disable record by default Recording is no longer enabled by default. It is now necessary to run scripts with appmap-python, or explicitly set APPMAP=true BREAKING CHANGE: disable record by default --- README.md | 6 ++- _appmap/env.py | 27 +---------- _appmap/recording.py | 3 -- _appmap/test/test_configuration.py | 6 --- _appmap/unittest.py | 3 -- _appmap/web_framework.py | 2 - appmap/__init__.py | 74 +++++++++++++++--------------- appmap/pytest.py | 4 -- tox.ini | 10 ++-- 9 files changed, 47 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5d16a3ab..cf878433 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,13 @@ reenabled as soon as possible.] ### pytest Note that you must install the dependencies contained in -[requirements-test.txt](requirements-test.txt) before running tests. See the explanation in +[requirements-dev.txt](requirements-dev.txt) before running tests. See the explanation in [pyproject.toml](pyproject.toml) for details. +Additionally, the tests currently require that you set `APPMAP=true`. You can +either run `pytest` with `appmap-python` (see [tox.ini](tox.ini)), or you can explicitly +set the environment variable. + [pytest](https://docs.pytest.org/en/stable/) for testing: ``` diff --git a/_appmap/env.py b/_appmap/env.py index 899c4c03..645412f2 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -3,7 +3,6 @@ import logging import logging.config import os -import warnings from contextlib import contextmanager from os import environ from pathlib import Path @@ -13,20 +12,6 @@ from . import trace_logger -_ENABLED_BY_DEFAULT_MSG = """ - -The APPMAP environment variable is unset. Your code will be -instrumented and recorded according to the configuration in appmap.yml. - -Starting with version 2, this behavior will change: when APPMAP is -unset, no code will be instrumented. You will need to use the -appmap-python script to run your application, or explicitly set -APPMAP. - -Visit https://appmap.io/docs/reference/appmap-python.html#appmap-python-script for more -details. -""" - _cwd = Path.cwd() _bootenv = environ.copy() @@ -36,9 +21,8 @@ def _recording_method_key(recording_method): class Env(metaclass=SingletonMeta): - def __init__(self, env=None, cwd=None): - warnings.filterwarnings("once", _ENABLED_BY_DEFAULT_MSG) + def __init__(self, env=None, cwd=None): # root_dir and root_dir_len are going to be used when # instrumenting every function, so preprocess them as # much as possible. @@ -54,7 +38,6 @@ def __init__(self, env=None, cwd=None): self._configure_logging() enabled = self._env.get("_APPMAP", None) - self._enabled_by_default = enabled is None self._enabled = enabled is None or enabled.lower() != "false" self._root_dir = str(self._cwd) + "/" @@ -98,14 +81,6 @@ def output_dir(self): def output_dir(self, value): self._output_dir = value - @property - def enabled_by_default(self): - return self._enabled_by_default - - def warn_enabled_by_default(self): - if self._enabled_by_default: - warnings.warn(_ENABLED_BY_DEFAULT_MSG, category=DeprecationWarning, stacklevel=2) - @property def enabled(self): return self._enabled diff --git a/_appmap/recording.py b/_appmap/recording.py index 465a5e1b..8aed253e 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -44,7 +44,6 @@ def is_running(self): return Recorder.get_enabled() def __enter__(self): - Env.current.warn_enabled_by_default() self.start() def __exit__(self, exc_type, exc_value, tb): @@ -81,8 +80,6 @@ def write_appmap( def initialize(): if Env.current.enables("process", "false"): - Env.current.warn_enabled_by_default() - r = Recording() r.start() diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index 71b26a8a..ecd73cb4 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -16,10 +16,6 @@ from _appmap.importer import Filterable, NullFilter -def test_enabled_by_default(): - assert appmap.enabled() - - @pytest.mark.appmap_enabled def test_can_be_configured(): """ @@ -45,8 +41,6 @@ def test_reports_invalid(): @pytest.mark.appmap_enabled(config="appmap-broken.yml") def test_is_disabled_when_unset(): """Test that recording is disabled when APPMAP is unset but the config is broken""" - assert Env.current.get("_APPMAP", None) is None - assert not appmap.enabled() diff --git a/_appmap/unittest.py b/_appmap/unittest.py index 18fcfed6..7641a86b 100644 --- a/_appmap/unittest.py +++ b/_appmap/unittest.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from _appmap import noappmap, testing_framework, wrapt -from _appmap.env import Env from _appmap.utils import get_function_location _session = testing_framework.session("unittest", "tests") @@ -43,7 +42,6 @@ def _args(test_case, *_, isTest=False, **__): with _session.record( test_case.__class__, method_name, location=location ) as metadata: - Env.current.warn_enabled_by_default() if metadata: with wrapped( *args, **kwargs @@ -69,7 +67,6 @@ def callTestMethod(wrapped, test_case, args, kwargs): method_name = test_case.id().split(".")[-1] location = _get_test_location(test_case.__class__, method_name) with _session.record(test_case.__class__, method_name, location=location) as metadata: - Env.current.warn_enabled_by_default() if metadata: with testing_framework.collect_result_metadata(metadata): wrapped(*args, **kwargs) diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index 9034f474..aebf08a7 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -250,8 +250,6 @@ def remote_enabled(self): """Return True if the AppMap middleware has enabled remote recording, False otherwise.""" def run(self): - Env.current.warn_enabled_by_default() - if not self.middleware_present(): return self.insert_middleware() diff --git a/appmap/__init__.py b/appmap/__init__.py index 7c58b247..5d271a40 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -9,40 +9,40 @@ if _enabled is None or _enabled.upper() == "TRUE": if _enabled is not None: # Use setdefault so tests can manage _APPMAP as necessary - os.environ["_APPMAP"] = _enabled - from _appmap import generation # noqa: F401 - from _appmap.env import Env # noqa: F401 - from _appmap.importer import instrument_module # noqa: F401 - from _appmap.labels import labels # noqa: F401 - from _appmap.noappmap import decorator as noappmap # noqa: F401 - from _appmap.recording import Recording # noqa: F401 - - try: - from . import django # noqa: F401 - except ImportError: - pass - - try: - from . import flask # noqa: F401 - except ImportError: - pass - - try: - from . import fastapi # noqa: F401 - except ImportError: - pass - - try: - from . import uvicorn # noqa: F401 - except ImportError: - pass - - # Note: pytest integration is configured as a pytest plugin, so it doesn't - # need to be imported here - - # unittest is part of the standard library, so it should always be - # importable (and therefore doesn't need to be in a try .. except block) - from . import unittest # noqa: F401 - - def enabled(): - return Env.current.enabled + os.environ.setdefault("_APPMAP", _enabled) + from _appmap import generation # noqa: F401 + from _appmap.env import Env # noqa: F401 + from _appmap.importer import instrument_module # noqa: F401 + from _appmap.labels import labels # noqa: F401 + from _appmap.noappmap import decorator as noappmap # noqa: F401 + from _appmap.recording import Recording # noqa: F401 + + try: + from . import django # noqa: F401 + except ImportError: + pass + + try: + from . import flask # noqa: F401 + except ImportError: + pass + + try: + from . import fastapi # noqa: F401 + except ImportError: + pass + + try: + from . import uvicorn # noqa: F401 + except ImportError: + pass + + # Note: pytest integration is configured as a pytest plugin, so it doesn't + # need to be imported here + + # unittest is part of the standard library, so it should always be + # importable (and therefore doesn't need to be in a try .. except block) + from . import unittest # noqa: F401 + + def enabled(): + return Env.current.enabled diff --git a/appmap/pytest.py b/appmap/pytest.py index 5638e063..8c555b52 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -25,10 +25,6 @@ def __call__(self, wrapped, _, args, kwargs): if not Env.current.is_appmap_repo and Env.current.enables("tests"): logger.debug("Test recording is enabled (Pytest)") - @pytest.hookimpl - def pytest_configure(config): # pylint: disable=unused-argument - Env.current.warn_enabled_by_default() - @pytest.hookimpl def pytest_sessionstart(session): session.appmap = testing_framework.session( diff --git a/tox.ini b/tox.ini index 3fe7c822..ea4ec756 100644 --- a/tox.ini +++ b/tox.ini @@ -24,14 +24,12 @@ deps= commands = poetry install -v - web: poetry run {posargs:pytest} - django3: poetry run pytest _appmap/test/test_django.py - flask2: poetry run pytest _appmap/test/test_flask.py - sqlalchemy1: poetry run pytest _appmap/test/test_sqlalchemy.py + web: poetry run appmap-python {posargs:pytest} + django3: poetry run appmap-python pytest _appmap/test/test_django.py + flask2: poetry run appmap-python pytest _appmap/test/test_flask.py + sqlalchemy1: poetry run appmap-python pytest _appmap/test/test_sqlalchemy.py [testenv:lint] -setenv = - APPMAP=false skip_install = True deps = poetry From 74b2ee15bfc380ee44ef74905e880541028f4c3b Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 20 May 2024 17:32:11 -0400 Subject: [PATCH 026/113] fix: enabling process recording disables others It doesn't make sense (and doesn't work) to try to capture other recording types when the user says they want a process recording (by setting APPMAP_RECORD_PROCESS=true). --- _appmap/env.py | 16 ++++++++++++++-- _appmap/recording.py | 2 +- _appmap/test/test_django.py | 13 +++++++++++++ _appmap/test/test_flask.py | 17 ++++++++++++++++- _appmap/test/test_test_frameworks.py | 7 +++++++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/_appmap/env.py b/_appmap/env.py index 645412f2..8d95aa3e 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -21,6 +21,7 @@ def _recording_method_key(recording_method): class Env(metaclass=SingletonMeta): + RECORD_PROCESS_DEFAULT = "false" def __init__(self, env=None, cwd=None): # root_dir and root_dir_len are going to be used when @@ -93,8 +94,19 @@ def enables(self, recording_method, default="true"): if not self.enabled: return False - v = self.get(_recording_method_key(recording_method), default).lower() - return v != "false" + process_enabled = self._enables("process", self.RECORD_PROCESS_DEFAULT) + if recording_method == "process": + return process_enabled + + # If process recording is enabled, others should be disabled + if process_enabled: + return False + + # Otherwise, check the environment variable + return self._enables(recording_method, default) + + def _enables(self, recording_method, default): + return self.get(_recording_method_key(recording_method), default).lower() != "false" @contextmanager def disabled(self, recording_method: str): diff --git a/_appmap/recording.py b/_appmap/recording.py index 8aed253e..ea05b0d7 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -79,7 +79,7 @@ def write_appmap( def initialize(): - if Env.current.enables("process", "false"): + if Env.current.enables("process", Env.RECORD_PROCESS_DEFAULT): r = Recording() r.start() diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index d26bb73a..1e9ee1dc 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -216,6 +216,19 @@ def test_disabled(self, pytester, monkeypatch): result.assert_outcomes(passed=1, failed=0, errors=0) assert not (pytester.path / "tmp").exists() + def test_disabled_for_process(self, pytester, monkeypatch): + monkeypatch.setenv("APPMAP_RECORD_PROCESS", "true") + + result = pytester.runpytest("-svv") + + # There are two tests for remote recording. They should both fail, + # because process recording should disable remote recording. + result.assert_outcomes(passed=2, failed=2, errors=0) + + assert (pytester.path / "tmp" / "appmap" / "process").exists() + assert not (pytester.path / "tmp" / "appmap" / "requests").exists() + assert not (pytester.path / "tmp" / "appmap" / "pytest").exists() + @pytest.fixture(name="server") def django_server(xprocess, server_base): diff --git a/_appmap/test/test_flask.py b/_appmap/test/test_flask.py index 9bd1eb5a..9bb20c50 100644 --- a/_appmap/test/test_flask.py +++ b/_appmap/test/test_flask.py @@ -8,10 +8,10 @@ import flask import pytest -from appmap.flask import AppmapFlask from _appmap.env import Env from _appmap.metadata import Metadata +from appmap.flask import AppmapFlask from ..test.helpers import DictIncluding from .web_framework import ( @@ -144,7 +144,11 @@ def test_enabled(self, pytester): appmap_file = ( pytester.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json" ) + + # No request recordings should have been created assert not os.path.exists(pytester.path / "tmp" / "appmap" / "requests") + + # but there should be a test recording assert appmap_file.exists() def test_disabled(self, pytester, monkeypatch): @@ -154,3 +158,14 @@ def test_disabled(self, pytester, monkeypatch): result.assert_outcomes(passed=1, failed=0, errors=0) assert not (pytester.path / "tmp" / "appmap").exists() + + def test_disabled_for_process(self, pytester, monkeypatch): + monkeypatch.setenv("APPMAP_RECORD_PROCESS", "true") + + result = pytester.runpytest("-svv") + + result.assert_outcomes(passed=1, failed=0, errors=0) + + assert (pytester.path / "tmp" / "appmap" / "process").exists() + assert not (pytester.path / "tmp" / "appmap" / "requests").exists() + assert not (pytester.path / "tmp" / "appmap" / "pytest").exists() diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index fbdfb9ec..e5093585 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -44,6 +44,13 @@ def test_disabled(self, testdir, monkeypatch): self.run_tests(testdir) assert not testdir.output().exists() + def test_disabled_for_process(self, testdir, monkeypatch): + monkeypatch.setenv("APPMAP_RECORD_PROCESS", "true") + + self.run_tests(testdir) + assert (testdir.path / "tmp" / "appmap" / "process").exists() + assert not testdir.output().exists() + class TestUnittestRunner(_TestTestRunner): @classmethod From e5f7b4167b08b4bf7a15e380828961b02b582ce4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 23 May 2024 19:29:36 +0000 Subject: [PATCH 027/113] chore(release): 2.0.0 [skip ci] # [2.0.0](https://github.com/getappmap/appmap-python/compare/v1.24.1...v2.0.0) (2024-05-23) ### Bug Fixes * combine testing-related env vars ([500fe55](https://github.com/getappmap/appmap-python/commit/500fe55f06c536611e3e292b22a8fade62101afe)) * enabling process recording disables others ([74b2ee1](https://github.com/getappmap/appmap-python/commit/74b2ee15bfc380ee44ef74905e880541028f4c3b)) * honor APPMAP_RECORD_REQUESTS when testing ([2df0f37](https://github.com/getappmap/appmap-python/commit/2df0f37474d1cd26bdfdbb45baf4fd2c9c9c982f)) ### Features * disable record by default ([57b3910](https://github.com/getappmap/appmap-python/commit/57b3910a48cea8582612772d79abacc53b5b73d5)) ### BREAKING CHANGES * disable record by default --- CHANGELOG.md | 19 +++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bae4ba62..1236f594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +# [2.0.0](https://github.com/getappmap/appmap-python/compare/v1.24.1...v2.0.0) (2024-05-23) + + +### Bug Fixes + +* combine testing-related env vars ([500fe55](https://github.com/getappmap/appmap-python/commit/500fe55f06c536611e3e292b22a8fade62101afe)) +* enabling process recording disables others ([74b2ee1](https://github.com/getappmap/appmap-python/commit/74b2ee15bfc380ee44ef74905e880541028f4c3b)) +* honor APPMAP_RECORD_REQUESTS when testing ([2df0f37](https://github.com/getappmap/appmap-python/commit/2df0f37474d1cd26bdfdbb45baf4fd2c9c9c982f)) + + +### Features + +* disable record by default ([57b3910](https://github.com/getappmap/appmap-python/commit/57b3910a48cea8582612772d79abacc53b5b73d5)) + + +### BREAKING CHANGES + +* disable record by default + ## [1.24.1](https://github.com/getappmap/appmap-python/compare/v1.24.0...v1.24.1) (2024-05-20) diff --git a/pyproject.toml b/pyproject.toml index b41aef6c..4b2b0f8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "1.24.1" +version = "2.0.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From b4fedc6c8d22082c9640d03caa8fbf12121fe8f9 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 23 May 2024 15:39:48 +0300 Subject: [PATCH 028/113] fix: handle non json serializable types With these changes, the string representation of the object will appear in the AppMap. This works well for numpy.int64, for example, but it's possible that it will fail for other types. --- _appmap/generation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_appmap/generation.py b/_appmap/generation.py index 8c4dd46f..d9929928 100644 --- a/_appmap/generation.py +++ b/_appmap/generation.py @@ -111,7 +111,10 @@ def default(self, o): if isinstance(o, ClassMapEntry): return o.to_dict() - return json.JSONEncoder.default(self, o) + try: + return json.JSONEncoder.default(self, o) + except TypeError: + return str(o) def dump(recording, metadata=None, indent=None): From 27e1bb40734213ad100f27e024337e3d2e484c3e Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Thu, 23 May 2024 16:49:36 -0400 Subject: [PATCH 029/113] fix: completely disable record-by-default There were still cases where the agent was trying to record by default. These changes handle those cases. --- _appmap/env.py | 2 +- appmap/__init__.py | 4 ++++ appmap/command/runner.py | 5 ++++- ci/smoketest.sh | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/_appmap/env.py b/_appmap/env.py index 8d95aa3e..ad1f7290 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -38,7 +38,7 @@ def __init__(self, env=None, cwd=None): self._env.pop(k, None) self._configure_logging() - enabled = self._env.get("_APPMAP", None) + enabled = self._env.get("_APPMAP", "false") self._enabled = enabled is None or enabled.lower() != "false" self._root_dir = str(self._cwd) + "/" diff --git a/appmap/__init__.py b/appmap/__init__.py index 5d271a40..2ddac2b9 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -46,3 +46,7 @@ def enabled(): return Env.current.enabled + else: + os.environ.pop("_APPMAP", None) +else: + os.environ.setdefault("_APPMAP", "false") \ No newline at end of file diff --git a/appmap/command/runner.py b/appmap/command/runner.py index 20bd5614..dd5a8e4b 100644 --- a/appmap/command/runner.py +++ b/appmap/command/runner.py @@ -106,7 +106,10 @@ def run(): parsed_args = vars(parsed_args) # our settings override those in the environment - envvars = {"APPMAP": "true"} + envvars = { + "APPMAP": "true", + "_APPMAP": "true", + } # Set the environment variables based on the the flags. A recording type in # --record overrides one set in --no-record. The environment variable for a diff --git a/ci/smoketest.sh b/ci/smoketest.sh index ab8089a4..66e3fc4f 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -6,13 +6,15 @@ pip -q install /dist/appmap-*-py3-none-any.whl cp -R /_appmap/test/data/unittest/simple ./. +export APPMAP=true + python -m appmap.command.appmap_agent_init |\ python -c 'import json,sys; print(json.load(sys.stdin)["configuration"]["contents"])' > /tmp/appmap.yml cat /tmp/appmap.yml python -m appmap.command.appmap_agent_validate -$RUNNER appmap-python pytest -k test_hello_world +$RUNNER pytest -k test_hello_world if [[ -f tmp/appmap/pytest/simple_test_simple_UnitTestTest_test_hello_world.appmap.json ]]; then echo 'Success' From 4e16d635fc708e20a9f96ce397c04a544e855c47 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 23 May 2024 22:57:10 +0000 Subject: [PATCH 030/113] chore(release): 2.0.1 [skip ci] ## [2.0.1](https://github.com/getappmap/appmap-python/compare/v2.0.0...v2.0.1) (2024-05-23) ### Bug Fixes * completely disable record-by-default ([27e1bb4](https://github.com/getappmap/appmap-python/commit/27e1bb40734213ad100f27e024337e3d2e484c3e)) * handle non json serializable types ([b4fedc6](https://github.com/getappmap/appmap-python/commit/b4fedc6c8d22082c9640d03caa8fbf12121fe8f9)) --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1236f594..94636d09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [2.0.1](https://github.com/getappmap/appmap-python/compare/v2.0.0...v2.0.1) (2024-05-23) + + +### Bug Fixes + +* completely disable record-by-default ([27e1bb4](https://github.com/getappmap/appmap-python/commit/27e1bb40734213ad100f27e024337e3d2e484c3e)) +* handle non json serializable types ([b4fedc6](https://github.com/getappmap/appmap-python/commit/b4fedc6c8d22082c9640d03caa8fbf12121fe8f9)) + # [2.0.0](https://github.com/getappmap/appmap-python/compare/v1.24.1...v2.0.0) (2024-05-23) diff --git a/pyproject.toml b/pyproject.toml index 4b2b0f8a..03ab08e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.0" +version = "2.0.1" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 9cb20a4e09f4084bb11fa117016d6b418bc03651 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sun, 26 May 2024 05:50:08 -0400 Subject: [PATCH 031/113] fix: expect a missing config file If mapping is disabled, attempting to load file config won't create a new file. Expect this can happen, and don't crash. --- _appmap/configuration.py | 3 +-- ci/smoketest.sh | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 3d255ce6..e83fd0ca 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -482,8 +482,7 @@ def initialize(): if _startup_messages_shown is None: # pylint: disable=protected-access c._load_config(show_warnings=True) - logger.info("file: %s", c._file) - logger.info("config: %s", c._config) + logger.info("file: %s", c._file if c.file_present else "[no appmap.yml]") logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" diff --git a/ci/smoketest.sh b/ci/smoketest.sh index 66e3fc4f..ebd9f31c 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -6,6 +6,9 @@ pip -q install /dist/appmap-*-py3-none-any.whl cp -R /_appmap/test/data/unittest/simple ./. +# Before we enable, run a command that tries to load the config +python -m appmap.command.appmap_agent_status + export APPMAP=true python -m appmap.command.appmap_agent_init |\ From 6ecfe518e0c7fb0fd461a69cf50f8ecd1ad23a70 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 27 May 2024 10:41:47 +0000 Subject: [PATCH 032/113] chore(release): 2.0.2 [skip ci] ## [2.0.2](https://github.com/getappmap/appmap-python/compare/v2.0.1...v2.0.2) (2024-05-27) ### Bug Fixes * expect a missing config file ([9cb20a4](https://github.com/getappmap/appmap-python/commit/9cb20a4e09f4084bb11fa117016d6b418bc03651)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94636d09..4f8daf19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.2](https://github.com/getappmap/appmap-python/compare/v2.0.1...v2.0.2) (2024-05-27) + + +### Bug Fixes + +* expect a missing config file ([9cb20a4](https://github.com/getappmap/appmap-python/commit/9cb20a4e09f4084bb11fa117016d6b418bc03651)) + ## [2.0.1](https://github.com/getappmap/appmap-python/compare/v2.0.0...v2.0.1) (2024-05-23) diff --git a/pyproject.toml b/pyproject.toml index 03ab08e9..4eef02ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.1" +version = "2.0.2" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From aae5dea50217568c67ccd312558c5e818f49b4ca Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Tue, 28 May 2024 05:56:07 -0400 Subject: [PATCH 033/113] fix: ask pytest not to rewrite our modules Make sure that when we get loaded as a pytest plugin, pytest doesn't try to rewrite our modules. This avoids a warning about being unable to do the rewrite, which will cause test failures if warnings have been promoted to errors (e.g. with warnings.simplefilter("error")). As far as I can tell, this change doesn't affect our internal tests, and we still get proper error messages when one of our assertions fails. Even if this turns out not be true, though, pytest rewriting is cosmetic only: an assertion failure still raises an exception and the test will fail. It just won't have a pretty message. --- _appmap/__init__.py | 2 ++ appmap/__init__.py | 4 +++- ci/smoketest.sh | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/_appmap/__init__.py b/_appmap/__init__.py index c98d0cfc..94e6c59f 100644 --- a/_appmap/__init__.py +++ b/_appmap/__init__.py @@ -1,3 +1,5 @@ +"""PYTEST_DONT_REWRITE""" + from . import configuration, event, importer, metadata, recorder, recording, web_framework from . import env as appmapenv from .py_version_check import check_py_version diff --git a/appmap/__init__.py b/appmap/__init__.py index 2ddac2b9..a1638b17 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -1,4 +1,6 @@ -"""AppMap recorder for Python""" +"""AppMap recorder for Python +PYTEST_DONT_REWRITE +""" import os # Note that we need to guard these imports with a conditional, rather than diff --git a/ci/smoketest.sh b/ci/smoketest.sh index ebd9f31c..79ad567f 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -17,7 +17,8 @@ cat /tmp/appmap.yml python -m appmap.command.appmap_agent_validate -$RUNNER pytest -k test_hello_world +# Promote warnings to errors, so we'll fail if pytest warns it can't rewrite appmap +$RUNNER pytest -Werror -k test_hello_world if [[ -f tmp/appmap/pytest/simple_test_simple_UnitTestTest_test_hello_world.appmap.json ]]; then echo 'Success' From 6cb8ffa4b710a9b0e174f5e0751557dd562127d1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 28 May 2024 16:17:13 +0000 Subject: [PATCH 034/113] chore(release): 2.0.3 [skip ci] ## [2.0.3](https://github.com/getappmap/appmap-python/compare/v2.0.2...v2.0.3) (2024-05-28) ### Bug Fixes * ask pytest not to rewrite our modules ([aae5dea](https://github.com/getappmap/appmap-python/commit/aae5dea50217568c67ccd312558c5e818f49b4ca)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f8daf19..71235efd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.3](https://github.com/getappmap/appmap-python/compare/v2.0.2...v2.0.3) (2024-05-28) + + +### Bug Fixes + +* ask pytest not to rewrite our modules ([aae5dea](https://github.com/getappmap/appmap-python/commit/aae5dea50217568c67ccd312558c5e818f49b4ca)) + ## [2.0.2](https://github.com/getappmap/appmap-python/compare/v2.0.1...v2.0.2) (2024-05-27) diff --git a/pyproject.toml b/pyproject.toml index 4eef02ac..819a4baa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.2" +version = "2.0.3" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 61d9aeec38ca677db19b9132c72749d8cf0a6a73 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 23 May 2024 15:39:48 +0300 Subject: [PATCH 035/113] test: non json serializable types --- .../data/pytest/expected/pytest.appmap.json | 118 ++++++++++++------ _appmap/test/data/pytest/simple.py | 11 +- _appmap/test/test_generation.py | 20 +++ requirements-dev.txt | 3 +- tox.ini | 2 + 5 files changed, 115 insertions(+), 39 deletions(-) diff --git a/_appmap/test/data/pytest/expected/pytest.appmap.json b/_appmap/test/data/pytest/expected/pytest.appmap.json index 5f047ad0..8723789a 100644 --- a/_appmap/test/data/pytest/expected/pytest.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest.appmap.json @@ -8,95 +8,133 @@ "name": "appmap", "url": "https://github.com/applandinc/appmap-python" }, + "source_location": "test_simple.py:5", + "name": "hello world", + "feature": "Hello world", "app": "Simple", "recorder": { "name": "pytest", "type": "tests" }, - "source_location": "test_simple.py:5", - "name": "hello world", - "feature": "Hello world", "test_status": "succeeded" }, "events": [ { - "defined_class": "simple.Simple", - "method_id": "hello_world", - "path": "simple.py", - "lineno": 8, "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], "id": 1, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple.py", + "lineno": 16 }, { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 2, + "event": "call", + "thread_id": 1, "defined_class": "simple.Simple", - "method_id": "hello", + "method_id": "get_non_json_serializable", "path": "simple.py", - "lineno": 2, + "lineno": 13 + }, + { "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 2, + "id": 3, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 7 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello'" + "value": "'Hello'", + "class": "builtins.str" }, - "parent_id": 2, - "id": 3, + "parent_id": 3, + "id": 4, "event": "return", "thread_id": 1 }, { - "defined_class": "simple.Simple", - "method_id": "world", - "path": "simple.py", - "lineno": 5, "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 4, + "id": 5, "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 10 + }, + { + "return_value": { + "value": "'world!'", + "class": "builtins.str" + }, + "parent_id": 5, + "id": 6, + "event": "return", "thread_id": 1 }, { "return_value": { - "class": "builtins.str", - "value": "'world!'" + "value": "{0: 'Hello', 1: 'world!'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 }, - "parent_id": 4, - "id": 5, + "parent_id": 2, + "id": 7, "event": "return", "thread_id": 1 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello world!'" + "value": "'Hello world!'", + "class": "builtins.str" }, "parent_id": 1, - "id": 6, + "id": 8, "event": "return", "thread_id": 1 } @@ -110,22 +148,28 @@ "name": "Simple", "type": "class", "children": [ + { + "name": "get_non_json_serializable", + "type": "function", + "location": "simple.py:13", + "static": false + }, { "name": "hello", "type": "function", - "location": "simple.py:2", + "location": "simple.py:7", "static": false }, { "name": "hello_world", "type": "function", - "location": "simple.py:8", + "location": "simple.py:16", "static": false }, { "name": "world", "type": "function", - "location": "simple.py:5", + "location": "simple.py:10", "static": false } ] diff --git a/_appmap/test/data/pytest/simple.py b/_appmap/test/data/pytest/simple.py index eb824400..af0338eb 100644 --- a/_appmap/test/data/pytest/simple.py +++ b/_appmap/test/data/pytest/simple.py @@ -1,3 +1,8 @@ +import numpy + +zero = numpy.int64(0) +one = numpy.int64(1) + class Simple: def hello(self): return "Hello" @@ -5,5 +10,9 @@ def hello(self): def world(self): return "world!" + def get_non_json_serializable(self): + return { zero: self.hello(), one: self.world() } + def hello_world(self): - return "%s %s" % (self.hello(), self.world()) + result = self.get_non_json_serializable() + return "%s %s" % (result[zero], result[one]) diff --git a/_appmap/test/test_generation.py b/_appmap/test/test_generation.py index e7891d60..ee1762fa 100644 --- a/_appmap/test/test_generation.py +++ b/_appmap/test/test_generation.py @@ -1,5 +1,10 @@ +import json import pytest +import numpy as np + +from _appmap.generation import AppMapEncoder + @pytest.mark.appmap_enabled @pytest.mark.usefixtures("with_data_dir") @@ -48,3 +53,18 @@ def check_comment(self, to_dict): return ret verify_example_appmap(check_comment, "instance_method") + +class TestAppMapEncoder: + def test_np_int64_type(self): + data = { + "value": np.int64(42), + } + json_str = json.dumps(data, cls=AppMapEncoder) + assert '{"value": "42"}' == json_str + + def test_np_array_type(self): + data = { + "value": np.array([0, 1, 2, 3]) + } + json_str = json.dumps(data, cls=AppMapEncoder) + assert '{"value": "[0 1 2 3]"}' == json_str diff --git a/requirements-dev.txt b/requirements-dev.txt index 2210a524..f70988fa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ pytest-django<4.8 fastapi httpx sqlalchemy -debugpy \ No newline at end of file +debugpy +numpy \ No newline at end of file diff --git a/tox.ini b/tox.ini index ea4ec756..faeb02af 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,8 @@ allowlist_externals = deps= poetry web: {[web-deps]deps} + py38: numpy==1.24.4 + py3{9,10,11,12}: numpy >=1.26 flask2: Flask >= 2.0, <3.0 django3: Django >=3.2, <4.0 sqlalchemy1: sqlalchemy >=1.4.11, <2.0 From 7c17a383fc849474ac44239abc6fb9f173f97edd Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 29 May 2024 13:17:27 -0400 Subject: [PATCH 036/113] fix: optionally limit number of events collected If the environment variable APPMAP_MAX_EVENTS is set, use it to limit the number of events added to a Recorder. This setting is a suggestion, rather than a hard limit, but the Recorder should get close. --- _appmap/instrument.py | 4 +++- _appmap/recorder.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 0e5b1057..e9fd050c 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -6,7 +6,7 @@ from . import event from .env import Env from .event import CallEvent -from .recorder import Recorder +from .recorder import Recorder, AppMapTooManyEvents from .utils import appmap_tls logger = Env.current.getLogger(__name__) @@ -97,6 +97,8 @@ def call_instrumented(f, instance, args, kwargs): ) Recorder.add_event(return_event) return ret + except AppMapTooManyEvents: + raise except Exception: # noqa: E722 elapsed_time = time.time() - start_time Recorder.add_event( diff --git a/_appmap/recorder.py b/_appmap/recorder.py index a44355a0..b690e7be 100644 --- a/_appmap/recorder.py +++ b/_appmap/recorder.py @@ -10,6 +10,17 @@ # pylint: disable=global-statement _default_recorder = None +# Allow the user to suggest a limit on the number of events that should be added to a Recorder. +# Depending on how exceptions get processed by the framework, there may be some more added, but it +# shouldn't be an enormous number. +_MAX_EVENTS = Env.current.get("APPMAP_MAX_EVENTS") +if _MAX_EVENTS is not None: + _MAX_EVENTS = int(_MAX_EVENTS) + + +class AppMapTooManyEvents(RuntimeError): + """Thrown when a recorder has more than APPMAP_MAX_EVENTS""" + class Recorder(ABC): """ @@ -18,6 +29,8 @@ class Recorder(ABC): Note that the abstract methods have implementations for use by subclasses. """ + _aborting = False + @property @abstractmethod def events(self): @@ -104,6 +117,7 @@ def _get_current(cls): return [perthread, _default_recorder] def clear(self): + Recorder._aborting = False self._events = [] def __init__(self, enabled=False): @@ -130,7 +144,13 @@ def _stop_recording(self): @abstractmethod def _add_event(self, event): + if Recorder._aborting: + return + self._events.append(event) + if _MAX_EVENTS is not None and len(self._events) > _MAX_EVENTS: + Recorder._aborting = True + raise AppMapTooManyEvents() @staticmethod def _initialize(): From fa33b89ad4ddf00e432b921e1b395021706d70d3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 29 May 2024 18:24:14 +0000 Subject: [PATCH 037/113] chore(release): 2.0.4 [skip ci] ## [2.0.4](https://github.com/getappmap/appmap-python/compare/v2.0.3...v2.0.4) (2024-05-29) ### Bug Fixes * optionally limit number of events collected ([7c17a38](https://github.com/getappmap/appmap-python/commit/7c17a383fc849474ac44239abc6fb9f173f97edd)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71235efd..96d24eda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.4](https://github.com/getappmap/appmap-python/compare/v2.0.3...v2.0.4) (2024-05-29) + + +### Bug Fixes + +* optionally limit number of events collected ([7c17a38](https://github.com/getappmap/appmap-python/commit/7c17a383fc849474ac44239abc6fb9f173f97edd)) + ## [2.0.3](https://github.com/getappmap/appmap-python/compare/v2.0.2...v2.0.3) (2024-05-28) diff --git a/pyproject.toml b/pyproject.toml index 819a4baa..dbb66ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.3" +version = "2.0.4" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 6bb7687412808c1d10a5d705ab0b1983868bb576 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Wed, 29 May 2024 15:39:23 +0300 Subject: [PATCH 038/113] fix: appmap.Recording is available even when APPMAP=false --- _appmap/recording.py | 27 +++++++++++++++++++++++++++ appmap/__init__.py | 11 ++++++++++- ci/smoketest.sh | 24 +++++++++++++++++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/_appmap/recording.py b/_appmap/recording.py index ea05b0d7..dafe9c1f 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -54,6 +54,33 @@ def __exit__(self, exc_type, exc_value, tb): return False +class NoopRecording: + """ + A noop context manager to export as "Recording" instead of class + Recording when not Env.current.enabled. + """ + + def __init__(self, exit_hook=None): + self.exit_hook = exit_hook + + def start(self): + pass + + def stop(self): + pass + + def is_running(self): + return False + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, tb): + if self.exit_hook is not None: + self.exit_hook(self) + return False + + def write_appmap( appmap, appmap_fname, recorder_type, metadata=None, basedir=Env.current.output_dir ): diff --git a/appmap/__init__.py b/appmap/__init__.py index a1638b17..b5750595 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -8,6 +8,7 @@ # execute the imports in a function, the modules all get put into the funtion's # globals, rather than into appmap's globals. _enabled = os.environ.get("APPMAP", None) +_recording_exported = False if _enabled is None or _enabled.upper() == "TRUE": if _enabled is not None: # Use setdefault so tests can manage _APPMAP as necessary @@ -18,6 +19,7 @@ from _appmap.labels import labels # noqa: F401 from _appmap.noappmap import decorator as noappmap # noqa: F401 from _appmap.recording import Recording # noqa: F401 + _recording_exported = True try: from . import django # noqa: F401 @@ -51,4 +53,11 @@ def enabled(): else: os.environ.pop("_APPMAP", None) else: - os.environ.setdefault("_APPMAP", "false") \ No newline at end of file + os.environ.setdefault("_APPMAP", "false") + +if not _recording_exported: + # Client code that imports appmap.Recording should run correctly + # even when not Env.current.enabled (not APPMAP=true). + # This prevents: + # ImportError: cannot import name 'Recording' from 'appmap'... + from _appmap.recording import NoopRecording as Recording # noqa: F401 diff --git a/ci/smoketest.sh b/ci/smoketest.sh index 79ad567f..f719533f 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -1,5 +1,24 @@ #!/usr/bin/env bash +test_recording_when_appmap_not_true() +{ + cat < test_client.py +from appmap import Recording + +with Recording(): + print("Hello from appmap library client") +EOF + + python test_client.py + + if [[ $? -eq 0 ]]; then + echo 'Script executed successfully' + else + echo 'Script execution failed' + exit 1 + fi +} + set -ex pip -q install -U pip pytest "flask>=2,<3" python-decouple pip -q install /dist/appmap-*-py3-none-any.whl @@ -9,6 +28,9 @@ cp -R /_appmap/test/data/unittest/simple ./. # Before we enable, run a command that tries to load the config python -m appmap.command.appmap_agent_status +# Ensure that client code using appmap.Recording does not fail when not APPMAP=true +test_recording_when_appmap_not_true + export APPMAP=true python -m appmap.command.appmap_agent_init |\ @@ -26,4 +48,4 @@ else echo 'No appmap generated?' find $PWD exit 1 -fi +fi \ No newline at end of file From 47d8cc32adfbce52d7f935dd68229eb7454e66c1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 30 May 2024 11:43:48 +0000 Subject: [PATCH 039/113] chore(release): 2.0.5 [skip ci] ## [2.0.5](https://github.com/getappmap/appmap-python/compare/v2.0.4...v2.0.5) (2024-05-30) ### Bug Fixes * appmap.Recording is available even when APPMAP=[secure] ([6bb7687](https://github.com/getappmap/appmap-python/commit/6bb7687412808c1d10a5d705ab0b1983868bb576)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96d24eda..f2c73a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.5](https://github.com/getappmap/appmap-python/compare/v2.0.4...v2.0.5) (2024-05-30) + + +### Bug Fixes + +* appmap.Recording is available even when APPMAP=[secure] ([6bb7687](https://github.com/getappmap/appmap-python/commit/6bb7687412808c1d10a5d705ab0b1983868bb576)) + ## [2.0.4](https://github.com/getappmap/appmap-python/compare/v2.0.3...v2.0.4) (2024-05-29) diff --git a/pyproject.toml b/pyproject.toml index dbb66ce1..9b5c1082 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.4" +version = "2.0.5" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From ec1f95debdd3524b688793459be490432895d57c Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 31 May 2024 09:09:39 -0400 Subject: [PATCH 040/113] fix: use an RLock in SharedRecorder._add_event It's possible that when adding an event to the global recorder, the current thread will try to add another event. This happens in Django's tests, because it uses weak references with instrumented finalizers. Calling _add_event can cause the weak reference to get garbage collected, resulting in a call to the finalizer, which will also call _add_event. --- _appmap/recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_appmap/recorder.py b/_appmap/recorder.py index b690e7be..c36147b6 100644 --- a/_appmap/recorder.py +++ b/_appmap/recorder.py @@ -191,7 +191,7 @@ class SharedRecorder(Recorder): A shared Recorder. The global recorder is an instance of this class. """ - _lock = threading.Lock() + _lock = threading.RLock() def __init__(self): super().__init__() From 60f3d145a5e8c65af202a8e03df509a3253f4947 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 31 May 2024 15:20:47 +0000 Subject: [PATCH 041/113] chore(release): 2.0.6 [skip ci] ## [2.0.6](https://github.com/getappmap/appmap-python/compare/v2.0.5...v2.0.6) (2024-05-31) ### Bug Fixes * use an RLock in SharedRecorder._add_event ([ec1f95d](https://github.com/getappmap/appmap-python/commit/ec1f95debdd3524b688793459be490432895d57c)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c73a29..a4cc672b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.6](https://github.com/getappmap/appmap-python/compare/v2.0.5...v2.0.6) (2024-05-31) + + +### Bug Fixes + +* use an RLock in SharedRecorder._add_event ([ec1f95d](https://github.com/getappmap/appmap-python/commit/ec1f95debdd3524b688793459be490432895d57c)) + ## [2.0.5](https://github.com/getappmap/appmap-python/compare/v2.0.4...v2.0.5) (2024-05-30) diff --git a/pyproject.toml b/pyproject.toml index 9b5c1082..8e55f546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.5" +version = "2.0.6" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 42230798cda296e31d437b93593e87760a498fad Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Sat, 1 Jun 2024 13:19:46 +0300 Subject: [PATCH 042/113] fix: max recursion depth exceeded --- _appmap/event.py | 6 +++++- _appmap/instrument.py | 3 ++- _appmap/test/data/example_class.py | 3 +++ _appmap/test/test_events.py | 12 ++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index 4af73cbf..71f2222e 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -455,7 +455,11 @@ class FuncReturnEvent(ReturnEvent): def __init__(self, parent_id, elapsed, return_value): super().__init__(parent_id, elapsed) - self.return_value = describe_value(None, return_value) + # Import here to prevent circular dependency + # pylint: disable=import-outside-toplevel + from _appmap.instrument import recording_disabled # noqa: F401 + with recording_disabled(): + self.return_value = describe_value(None, return_value) class HttpResponseEvent(ReturnEvent): diff --git a/_appmap/instrument.py b/_appmap/instrument.py index e9fd050c..10410e2c 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -15,11 +15,12 @@ @contextmanager def recording_disabled(): tls = appmap_tls() + original_value = tls.get("instrumentation_disabled") tls["instrumentation_disabled"] = True try: yield finally: - tls["instrumentation_disabled"] = False + tls["instrumentation_disabled"] = original_value def is_instrumentation_disabled(): diff --git a/_appmap/test/data/example_class.py b/_appmap/test/data/example_class.py index c84f3fe1..3e61f7ee 100644 --- a/_appmap/test/data/example_class.py +++ b/_appmap/test/data/example_class.py @@ -110,6 +110,9 @@ def with_docstring(self): def with_comment(self): return True + def return_self(self): + return self + def modfunc(): return "Hello world!" diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index ba45a3af..4dba7cd8 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -105,3 +105,15 @@ def test_when_display_disabled(self, mocker): # MagicMock. (If it's broken, we may not get here at all, # because the assertion above may fail.) param.__repr__.assert_called_once_with() + + def test_describe_return_value_recursion_protection(self): + r = appmap.Recording() + with r: + # pylint: disable=import-outside-toplevel + from example_class import ExampleClass + + ExampleClass().return_self() + # There should be no event for method another_method which is called by __repr__. + assert [e.method_id for e in r.events if e.event == "call" and hasattr(e, "method_id")] == [ + "return_self" + ] From 92f5b29cd0c1a1f3bc5418e0d906c589a690894a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 5 Jun 2024 10:39:33 +0000 Subject: [PATCH 043/113] chore(release): 2.0.7 [skip ci] ## [2.0.7](https://github.com/getappmap/appmap-python/compare/v2.0.6...v2.0.7) (2024-06-05) ### Bug Fixes * max recursion depth exceeded ([4223079](https://github.com/getappmap/appmap-python/commit/42230798cda296e31d437b93593e87760a498fad)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4cc672b..d4ee93e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.7](https://github.com/getappmap/appmap-python/compare/v2.0.6...v2.0.7) (2024-06-05) + + +### Bug Fixes + +* max recursion depth exceeded ([4223079](https://github.com/getappmap/appmap-python/commit/42230798cda296e31d437b93593e87760a498fad)) + ## [2.0.6](https://github.com/getappmap/appmap-python/compare/v2.0.5...v2.0.6) (2024-05-31) diff --git a/pyproject.toml b/pyproject.toml index 8e55f546..7d3a5b2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.6" +version = "2.0.7" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From d60c52813aec424de1db2b45bb5f13eb23965d61 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 5 Jun 2024 06:55:25 -0400 Subject: [PATCH 044/113] fix: support APPMAP_MAX_TIME Add APPMAP_MAX_TIME: allows the user to specify the maximum time (in seconds) a recording can be in progress. If the session exceeds this time, AppMapSessionTooLong will be raised. --- _appmap/instrument.py | 5 +++-- _appmap/recorder.py | 26 ++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 10410e2c..191e5ef9 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -6,7 +6,7 @@ from . import event from .env import Env from .event import CallEvent -from .recorder import Recorder, AppMapTooManyEvents +from .recorder import Recorder, AppMapLimitExceeded from .utils import appmap_tls logger = Env.current.getLogger(__name__) @@ -90,6 +90,7 @@ def call_instrumented(f, instance, args, kwargs): call_event_id = call_event.id start_time = time.time() try: + Recorder.check_time(start_time) ret = f.fn(*args, **kwargs) elapsed_time = time.time() - start_time @@ -98,7 +99,7 @@ def call_instrumented(f, instance, args, kwargs): ) Recorder.add_event(return_event) return ret - except AppMapTooManyEvents: + except AppMapLimitExceeded: raise except Exception: # noqa: E722 elapsed_time = time.time() - start_time diff --git a/_appmap/recorder.py b/_appmap/recorder.py index c36147b6..c79c898b 100644 --- a/_appmap/recorder.py +++ b/_appmap/recorder.py @@ -1,4 +1,5 @@ import threading +import time import traceback from abc import ABC, abstractmethod @@ -17,11 +18,23 @@ if _MAX_EVENTS is not None: _MAX_EVENTS = int(_MAX_EVENTS) +_MAX_TIME = Env.current.get("APPMAP_MAX_TIME") +if _MAX_TIME is not None: + _MAX_TIME = int(_MAX_TIME) -class AppMapTooManyEvents(RuntimeError): + +class AppMapLimitExceeded(RuntimeError): + """Class of events thrown when some limit has been exceeded""" + + +class AppMapTooManyEvents(AppMapLimitExceeded): """Thrown when a recorder has more than APPMAP_MAX_EVENTS""" +class AppMapSessionTooLong(AppMapLimitExceeded): + """Throw when an individual recording session has exceeded APPMAP_MAX_TIME""" + + class Recorder(ABC): """ A base class for Recorders. @@ -97,6 +110,13 @@ def start_recording(cls): def stop_recording(cls): return cls.get_current()._stop_recording() # pylint: disable=protected-access + @classmethod + def check_time(cls, event_time): + if _MAX_TIME is None: + return + if event_time - cls.get_current()._start_time > _MAX_TIME: + raise AppMapSessionTooLong(f"Session exceeded {_MAX_TIME} seconds") + @classmethod def add_event(cls, event): """ @@ -124,6 +144,7 @@ def __init__(self, enabled=False): self._events = [] self._enabled = enabled self.start_tb = None + self._start_time = None @abstractmethod def _start_recording(self): @@ -134,6 +155,7 @@ def _start_recording(self): raise RuntimeError("Recording already in progress") self.start_tb = traceback.extract_stack() self._enabled = True + self._start_time = time.time() @abstractmethod def _stop_recording(self): @@ -150,7 +172,7 @@ def _add_event(self, event): self._events.append(event) if _MAX_EVENTS is not None and len(self._events) > _MAX_EVENTS: Recorder._aborting = True - raise AppMapTooManyEvents() + raise AppMapTooManyEvents(f"Session exceeded {_MAX_EVENTS} events") @staticmethod def _initialize(): From 4e29c1321d0e52554a797efd4e5bdda240e2fe82 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 5 Jun 2024 06:57:46 -0400 Subject: [PATCH 045/113] fix: optionally disable schema render When APPMAP_DISPLAY_PARAMS is false, disable rendering of the schema of values (e.g. params, return types). A schema can be deeply nested (as in matplotlib), which blows up memory usage and mapping time. --- _appmap/event.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_appmap/event.py b/_appmap/event.py index 71f2222e..f53102d2 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -111,7 +111,9 @@ def describe_value(name, val, max_depth=5): "object_id": id(val), "value": display_string(val), } - ret.update(_describe_schema(name, val, 0, max_depth)) + if Env.current.display_params: + ret.update(_describe_schema(name, val, 0, max_depth)) + if any(_is_list_or_dict(type(val))): ret["size"] = len(val) From f4618b68bc9bc40ff54120385c1820896094bd2e Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 5 Jun 2024 07:05:50 -0400 Subject: [PATCH 046/113] fix: move __reduce_ex__ up to ObjectProxy The implementation that was previously in FunctionWrapper should work for all subclasses of ObjectProxy. --- vendor/_appmap/wrapt/wrappers.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vendor/_appmap/wrapt/wrappers.py b/vendor/_appmap/wrapt/wrappers.py index a7e9a3d6..f269bbcb 100644 --- a/vendor/_appmap/wrapt/wrappers.py +++ b/vendor/_appmap/wrapt/wrappers.py @@ -457,9 +457,11 @@ def __reduce__(self): raise NotImplementedError( 'object proxy must define __reduce_ex__()') + # Return the qualname of the wrapped function instead of a tuple. This allows an instance of + # subclasses to be pickled as the function it wraps. This seems to be adequate for generating + # AppMaps. def __reduce_ex__(self, protocol): - raise NotImplementedError( - 'object proxy must define __reduce_ex__()') + return self.__wrapped__.__qualname__ class CallableObjectProxy(ObjectProxy): @@ -740,12 +742,9 @@ class FunctionWrapper(_FunctionWrapperBase): # new FunctionWrapper will be created. If it doesn't, then __reduce_ex__ can simply return a # string, which would cause deepcopy to return the original FunctionWrapper. # - # Update: We'll return the qualname of the wrapped function instead of a tuple allows a - # FunctionWrapper to be pickled (as the function it wraps). This seems to be adequate for - # generating AppMaps, so go with that. - def __reduce_ex__(self, protocol): - return self.__wrapped__.__qualname__ + # def __reduce_ex__(self, protocol): + # return self.__wrapped__.__qualname__ # return FunctionWrapper, ( # self.__wrapped__, From 1d5a072962a4993321ef059c76eb0968938b7ffd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 5 Jun 2024 13:49:15 +0000 Subject: [PATCH 047/113] chore(release): 2.0.8 [skip ci] ## [2.0.8](https://github.com/getappmap/appmap-python/compare/v2.0.7...v2.0.8) (2024-06-05) ### Bug Fixes * move __reduce_ex__ up to ObjectProxy ([f4618b6](https://github.com/getappmap/appmap-python/commit/f4618b68bc9bc40ff54120385c1820896094bd2e)) * optionally disable schema render ([4e29c13](https://github.com/getappmap/appmap-python/commit/4e29c1321d0e52554a797efd4e5bdda240e2fe82)) * support APPMAP_MAX_TIME ([d60c528](https://github.com/getappmap/appmap-python/commit/d60c52813aec424de1db2b45bb5f13eb23965d61)) --- CHANGELOG.md | 9 +++++++++ pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ee93e4..b827f1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## [2.0.8](https://github.com/getappmap/appmap-python/compare/v2.0.7...v2.0.8) (2024-06-05) + + +### Bug Fixes + +* move __reduce_ex__ up to ObjectProxy ([f4618b6](https://github.com/getappmap/appmap-python/commit/f4618b68bc9bc40ff54120385c1820896094bd2e)) +* optionally disable schema render ([4e29c13](https://github.com/getappmap/appmap-python/commit/4e29c1321d0e52554a797efd4e5bdda240e2fe82)) +* support APPMAP_MAX_TIME ([d60c528](https://github.com/getappmap/appmap-python/commit/d60c52813aec424de1db2b45bb5f13eb23965d61)) + ## [2.0.7](https://github.com/getappmap/appmap-python/compare/v2.0.6...v2.0.7) (2024-06-05) diff --git a/pyproject.toml b/pyproject.toml index 7d3a5b2f..3fc4be30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.7" +version = "2.0.8" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From c179b86a5769de90775f0d8848e8ad0422961dfe Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 20 Jun 2024 12:30:18 +0300 Subject: [PATCH 048/113] fix: appmap breaks vscode python extension starting a REPL --- _appmap/env.py | 29 ++++++++++++++++++++++++++++- ci/readonly-mount-appmap.log | 1 + ci/run_tests.sh | 1 + ci/smoketest.sh | 20 +++++++++++++++++++- 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 ci/readonly-mount-appmap.log diff --git a/_appmap/env.py b/_appmap/env.py index ad1f7290..b74683bc 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -37,6 +37,7 @@ def __init__(self, env=None, cwd=None): else: self._env.pop(k, None) + self.log_file_creation_failed = False self._configure_logging() enabled = self._env.get("_APPMAP", "false") self._enabled = enabled is None or enabled.lower() != "false" @@ -133,6 +134,21 @@ def display_params(self): def getLogger(self, name) -> trace_logger.TraceLogger: return cast(trace_logger.TraceLogger, logging.getLogger(name)) + def determine_log_file(self): + log_file = "appmap.log" + + # Try creating the log file in the current directory + try: + with open(log_file, 'a', encoding='UTF8'): + pass + except IOError: + # The circumstances in which creation is going to fail + # are also those in which the user doesn't care whether + # there's a log file (e.g. when starting a REPL). + return None + return log_file + + def _configure_logging(self): trace_logger.install() @@ -179,13 +195,19 @@ def _configure_logging(self): log_level = self.get("APPMAP_LOG_LEVEL", "info").upper() loggers = config_dict["loggers"] loggers["appmap"]["level"] = loggers["_appmap"]["level"] = log_level + + log_file = self.determine_log_file() + # Use NullHandler if log_file is None to avoid complicating the configuration + # with the absence of the "default" handler. config_dict["handlers"] = { "default": { "class": "logging.handlers.RotatingFileHandler", "formatter": "default", - "filename": "appmap.log", + "filename": log_file, "maxBytes": 50 * 1024 * 1024, "backupCount": 1, + } if log_file is not None else { + "class": "logging.NullHandler" }, "stderr": { "class": "logging.StreamHandler", @@ -194,6 +216,7 @@ def _configure_logging(self): "stream": "ext://sys.stderr", }, } + self.log_file_creation_failed = log_file is None if log_config is not None: name, level = log_config.split("=", 2) @@ -213,3 +236,7 @@ def initialize(**kwargs): Env.reset(**kwargs) logger = logging.getLogger(__name__) logger.info("appmap enabled: %s", Env.current.enabled) + if Env.current.log_file_creation_failed: + # Writing to stderr makes the REPL fail in vscode-python. + # https://github.com/microsoft/vscode-python/blob/c71c85ebf3749d5fac76899feefb21ee321a4b5b/src/client/common/process/rawProcessApis.ts#L268-L269 + logger.info("appmap.log cannot be created") diff --git a/ci/readonly-mount-appmap.log b/ci/readonly-mount-appmap.log new file mode 100644 index 00000000..249845e3 --- /dev/null +++ b/ci/readonly-mount-appmap.log @@ -0,0 +1 @@ +# For a test in smoketest \ No newline at end of file diff --git a/ci/run_tests.sh b/ci/run_tests.sh index d97299a0..c78d5415 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -6,4 +6,5 @@ docker run -q -i${t} --rm\ -v $PWD/dist:/dist -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ -v $PWD/ci:/ci\ -w /tmp\ + -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ python:3.11 bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" diff --git a/ci/smoketest.sh b/ci/smoketest.sh index f719533f..3d0eeb13 100755 --- a/ci/smoketest.sh +++ b/ci/smoketest.sh @@ -19,6 +19,22 @@ EOF fi } +test_log_file_not_writable() +{ + cat < test_log_file_not_writable.py +import appmap +EOF + + python test_log_file_not_writable.py + + if [[ $? -eq 0 ]]; then + echo 'Script executed successfully' + else + echo 'Script execution failed' + exit 1 + fi +} + set -ex pip -q install -U pip pytest "flask>=2,<3" python-decouple pip -q install /dist/appmap-*-py3-none-any.whl @@ -48,4 +64,6 @@ else echo 'No appmap generated?' find $PWD exit 1 -fi \ No newline at end of file +fi + +test_log_file_not_writable \ No newline at end of file From 965f9875d05e9dda68a89d6717feb11d3e289023 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 20 Jun 2024 13:29:13 +0000 Subject: [PATCH 049/113] chore(release): 2.0.9 [skip ci] ## [2.0.9](https://github.com/getappmap/appmap-python/compare/v2.0.8...v2.0.9) (2024-06-20) ### Bug Fixes * appmap breaks vscode python extension starting a REPL ([c179b86](https://github.com/getappmap/appmap-python/commit/c179b86a5769de90775f0d8848e8ad0422961dfe)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b827f1a9..eb85ff41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.9](https://github.com/getappmap/appmap-python/compare/v2.0.8...v2.0.9) (2024-06-20) + + +### Bug Fixes + +* appmap breaks vscode python extension starting a REPL ([c179b86](https://github.com/getappmap/appmap-python/commit/c179b86a5769de90775f0d8848e8ad0422961dfe)) + ## [2.0.8](https://github.com/getappmap/appmap-python/compare/v2.0.7...v2.0.8) (2024-06-05) diff --git a/pyproject.toml b/pyproject.toml index 3fc4be30..078789f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.8" +version = "2.0.9" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 1ee69cb39b8be8e97b1498218b4ff3a4b9b1c3ee Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Sat, 8 Jun 2024 00:15:32 +0300 Subject: [PATCH 050/113] fix: request recording in unittest setUp method --- _appmap/test/data/django/test/test_unittest_setup.py | 11 +++++++++++ _appmap/test/test_django.py | 10 ++++++---- _appmap/unittest.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 _appmap/test/data/django/test/test_unittest_setup.py diff --git a/_appmap/test/data/django/test/test_unittest_setup.py b/_appmap/test/data/django/test/test_unittest_setup.py new file mode 100644 index 00000000..e535d528 --- /dev/null +++ b/_appmap/test/data/django/test/test_unittest_setup.py @@ -0,0 +1,11 @@ + +from unittest import TestCase + +from django.test import Client + +class DisabledRequestsRecordingTest(TestCase): + def setUp(self) -> None: + Client().get("/") + + def test_request_in_setup(self): + pass diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index 1e9ee1dc..730c7f36 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -201,11 +201,13 @@ def test_enabled(self, pytester): # To really check middleware reset, the tests must run in order, # so disable randomly. result = pytester.runpytest("-svv", "-p", "no:randomly") - result.assert_outcomes(passed=4, failed=0, errors=0) + result.assert_outcomes(passed=5, failed=0, errors=0) # Look for the http_server_request event in test_app's appmap. If # middleware reset is broken, it won't be there. appmap_file = pytester.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json" - assert not os.path.exists(pytester.path / "tmp" / "appmap" / "requests") + assert not os.path.exists( + pytester.path / "tmp" / "appmap" / "requests" + ), "django tests generated request recordings" events = json.loads(appmap_file.read_text())["events"] assert "http_server_request" in events[0] @@ -213,7 +215,7 @@ def test_enabled(self, pytester): def test_disabled(self, pytester, monkeypatch): monkeypatch.setenv("_APPMAP", "false") result = pytester.runpytest("-svv", "-p", "no:randomly", "-k", "test_request") - result.assert_outcomes(passed=1, failed=0, errors=0) + result.assert_outcomes(passed=2, failed=0, errors=0) assert not (pytester.path / "tmp").exists() def test_disabled_for_process(self, pytester, monkeypatch): @@ -223,7 +225,7 @@ def test_disabled_for_process(self, pytester, monkeypatch): # There are two tests for remote recording. They should both fail, # because process recording should disable remote recording. - result.assert_outcomes(passed=2, failed=2, errors=0) + result.assert_outcomes(passed=3, failed=2, errors=0) assert (pytester.path / "tmp" / "appmap" / "process").exists() assert not (pytester.path / "tmp" / "appmap" / "requests").exists() diff --git a/_appmap/unittest.py b/_appmap/unittest.py index 7641a86b..d2a2bd7f 100644 --- a/_appmap/unittest.py +++ b/_appmap/unittest.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from _appmap import noappmap, testing_framework, wrapt +from _appmap.env import Env from _appmap.utils import get_function_location _session = testing_framework.session("unittest", "tests") @@ -52,6 +53,17 @@ def _args(test_case, *_, isTest=False, **__): yield else: + # We need to disable request recording in TestCase._callSetUp too + # in order to prevent creation of a request recording besides test + # recording when requests are made inside setUp method. + # This edge case can be observed in this test in django project: + # $ APPMAP=TRUE ./runtests.py auth_tests.test_views.ChangelistTests.test_user_change_email + #  (ChangelistTests.setUp makes a request) + @wrapt.patch_function_wrapper("unittest.case", "TestCase._callSetUp") + def callSetUp(wrapped, test_case, args, kwargs): # pylint: disable=unused-argument + with Env.current.disabled("requests"): + wrapped(*args, **kwargs) + # As of 3.8, unittest.case.TestCase now calls the test's method indirectly, through # TestCase._callTestMethod. Hook that to manage a recording session. @wrapt.patch_function_wrapper("unittest.case", "TestCase._callTestMethod") From 660e31fda039b67f0d76c21e4ec32542c2253d7c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 21 Jun 2024 10:35:05 +0000 Subject: [PATCH 051/113] chore(release): 2.0.10 [skip ci] ## [2.0.10](https://github.com/getappmap/appmap-python/compare/v2.0.9...v2.0.10) (2024-06-21) ### Bug Fixes * request recording in unittest setUp method ([1ee69cb](https://github.com/getappmap/appmap-python/commit/1ee69cb39b8be8e97b1498218b4ff3a4b9b1c3ee)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb85ff41..e88aba07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.0.10](https://github.com/getappmap/appmap-python/compare/v2.0.9...v2.0.10) (2024-06-21) + + +### Bug Fixes + +* request recording in unittest setUp method ([1ee69cb](https://github.com/getappmap/appmap-python/commit/1ee69cb39b8be8e97b1498218b4ff3a4b9b1c3ee)) + ## [2.0.9](https://github.com/getappmap/appmap-python/compare/v2.0.8...v2.0.9) (2024-06-20) diff --git a/pyproject.toml b/pyproject.toml index 078789f6..c1a369c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.9" +version = "2.0.10" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From d2af099c2aac9c14b78094a98117248aef1eea6d Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Tue, 2 Jul 2024 13:39:32 -0400 Subject: [PATCH 052/113] test: show appmap as JSON when diff fails When an actual appmap doesn't match the expected one, show it as JSON to make it easier to understand why they're different (and to assist with updating the expected, if appropriate). --- _appmap/test/test_test_frameworks.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index e5093585..5f2f0779 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -11,7 +11,7 @@ import pytest -from _appmap import recording +from _appmap import recording, generation from ..test.helpers import DictIncluding from .normalize import normalize_appmap @@ -95,7 +95,7 @@ def setup_class(cls): cls._test_type = "pytest" def run_tests(self, testdir): - result = testdir.runpytest("-vv") + result = testdir.runpytest("-svv") result.assert_outcomes(passed=4, failed=2, xpassed=1, xfailed=1) def test_enabled(self, testdir): @@ -198,7 +198,9 @@ def verify_expected_appmap(testdir): appmap_json = testdir.expected / (f"{testdir.test_type}.appmap.json") expected_appmap = json.loads(appmap_json.read_text()) - assert generated_appmap == expected_appmap, f"expected appmap file {appmap_json}" + assert ( + generated_appmap == expected_appmap + ), f"expected appmap file {appmap_json}\ngenerated appmap: {json.dumps(generated_appmap, indent=2)}" def verify_expected_metadata(testdir): @@ -212,4 +214,6 @@ def verify_expected_metadata(testdir): name = pattern.search(file.name).group(1) metadata = json.loads(file.read_text())["metadata"] expected = testdir.expected / f"{name}.metadata.json" - assert metadata == DictIncluding(json.loads(expected.read_text())) + assert metadata == DictIncluding( + json.loads(expected.read_text()) + ), f"expected appmap: {file}" From 231af726d9302086655378df1e4fff951bf970fc Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Tue, 2 Jul 2024 15:25:02 -0400 Subject: [PATCH 053/113] test: make sure JSON serialization works With the previous function in simple.py, the return event generated to check that numpy.int64 serialized correctly didn't test AppMapEncoder. Keys in a dict that are in a return event are rendered with "repr" before the event gets serialized. "repr(int64(0))" returned "0" in 1.x, but returns "numpy.int64(0)" in 2.x, which caused the test to fail. --- ....appmap.json => pytest-numpy1.appmap.json} | 164 ++++++++---- .../pytest/expected/pytest-numpy2.appmap.json | 242 ++++++++++++++++++ .../expected/status_errored.metadata.json | 2 +- .../expected/status_failed.metadata.json | 2 +- .../expected/status_xfailed.metadata.json | 2 +- _appmap/test/data/pytest/simple.py | 21 +- _appmap/test/data/pytest/test_simple.py | 6 +- _appmap/test/test_test_frameworks.py | 19 +- tox.ini | 2 +- 9 files changed, 385 insertions(+), 75 deletions(-) rename _appmap/test/data/pytest/expected/{pytest.appmap.json => pytest-numpy1.appmap.json} (58%) create mode 100644 _appmap/test/data/pytest/expected/pytest-numpy2.appmap.json diff --git a/_appmap/test/data/pytest/expected/pytest.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json similarity index 58% rename from _appmap/test/data/pytest/expected/pytest.appmap.json rename to _appmap/test/data/pytest/expected/pytest-numpy1.appmap.json index 8723789a..cd3ef53c 100644 --- a/_appmap/test/data/pytest/expected/pytest.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json @@ -8,75 +8,95 @@ "name": "appmap", "url": "https://github.com/applandinc/appmap-python" }, - "source_location": "test_simple.py:5", - "name": "hello world", - "feature": "Hello world", "app": "Simple", "recorder": { "name": "pytest", "type": "tests" }, + "source_location": "test_simple.py:5", + "name": "hello world", + "feature": "Hello world", "test_status": "succeeded" }, "events": [ { + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple.py", + "lineno": 8, "static": false, "receiver": { + "class": "simple.Simple", "kind": "req", - "value": "", "name": "self", - "class": "simple.Simple" + "value": "" }, "parameters": [], "id": 1, "event": "call", - "thread_id": 1, - "defined_class": "simple.Simple", - "method_id": "hello_world", - "path": "simple.py", - "lineno": 16 + "thread_id": 1 }, { + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2, "static": false, "receiver": { + "class": "simple.Simple", "kind": "req", - "value": "", "name": "self", - "class": "simple.Simple" + "value": "" }, "parameters": [], "id": 2, "event": "call", - "thread_id": 1, - "defined_class": "simple.Simple", - "method_id": "get_non_json_serializable", - "path": "simple.py", - "lineno": 13 + "thread_id": 1 }, { + "return_value": { + "class": "builtins.str", + "value": "'Hello'" + }, + "parent_id": 2, + "id": 3, + "event": "return", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5, "static": false, "receiver": { + "class": "simple.Simple", "kind": "req", - "value": "", "name": "self", - "class": "simple.Simple" + "value": "" }, "parameters": [], - "id": 3, + "id": 4, "event": "call", - "thread_id": 1, - "defined_class": "simple.Simple", - "method_id": "hello", - "path": "simple.py", - "lineno": 7 + "thread_id": 1 }, { "return_value": { - "value": "'Hello'", - "class": "builtins.str" + "class": "builtins.str", + "value": "'world!'" }, - "parent_id": 3, - "id": 4, + "parent_id": 4, + "id": 5, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello world!'" + }, + "parent_id": 1, + "id": 6, "event": "return", "thread_id": 1 }, @@ -89,27 +109,52 @@ "class": "simple.Simple" }, "parameters": [], - "id": 5, + "id": 7, "event": "call", "thread_id": 1, "defined_class": "simple.Simple", - "method_id": "world", + "method_id": "show_numpy_dict", "path": "simple.py", - "lineno": 10 + "lineno": 11 }, { - "return_value": { - "value": "'world!'", - "class": "builtins.str" + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" }, - "parent_id": 5, - "id": 6, - "event": "return", - "thread_id": 1 + "parameters": [ + { + "kind": "req", + "value": "{0: 'zero', 1: 'one'}", + "name": "d", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + } + ], + "id": 8, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "get_numpy_dict", + "path": "simple.py", + "lineno": 18 }, { "return_value": { - "value": "{0: 'Hello', 1: 'world!'}", + "value": "{0: 'zero', 1: 'one'}", "class": "builtins.dict", "properties": [ { @@ -123,18 +168,29 @@ ], "size": 2 }, - "parent_id": 2, - "id": 7, + "parent_id": 8, + "id": 9, "event": "return", "thread_id": 1 }, { "return_value": { - "value": "'Hello world!'", - "class": "builtins.str" + "value": "{0: 'zero', 1: 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 }, - "parent_id": 1, - "id": 8, + "parent_id": 7, + "id": 10, "event": "return", "thread_id": 1 } @@ -149,27 +205,33 @@ "type": "class", "children": [ { - "name": "get_non_json_serializable", + "name": "get_numpy_dict", "type": "function", - "location": "simple.py:13", + "location": "simple.py:18", "static": false }, { "name": "hello", "type": "function", - "location": "simple.py:7", + "location": "simple.py:2", "static": false }, { "name": "hello_world", "type": "function", - "location": "simple.py:16", + "location": "simple.py:8", + "static": false + }, + { + "name": "show_numpy_dict", + "type": "function", + "location": "simple.py:11", "static": false }, { "name": "world", "type": "function", - "location": "simple.py:10", + "location": "simple.py:5", "static": false } ] diff --git a/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json new file mode 100644 index 00000000..0f12e30c --- /dev/null +++ b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json @@ -0,0 +1,242 @@ +{ + "version": "1.9", + "metadata": { + "language": { + "name": "python" + }, + "client": { + "name": "appmap", + "url": "https://github.com/applandinc/appmap-python" + }, + "app": "Simple", + "recorder": { + "name": "pytest", + "type": "tests" + }, + "source_location": "test_simple.py:5", + "name": "hello world", + "feature": "Hello world", + "test_status": "succeeded" + }, + "events": [ + { + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple.py", + "lineno": 8, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 1, + "event": "call", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 2, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello'" + }, + "parent_id": 2, + "id": 3, + "event": "return", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 4, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'world!'" + }, + "parent_id": 4, + "id": 5, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello world!'" + }, + "parent_id": 1, + "id": 6, + "event": "return", + "thread_id": 1 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 7, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "show_numpy_dict", + "path": "simple.py", + "lineno": 11 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [ + { + "kind": "req", + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "name": "d", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + } + ], + "id": 8, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "get_numpy_dict", + "path": "simple.py", + "lineno": 18 + }, + { + "return_value": { + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 8, + "id": 9, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 7, + "id": 10, + "event": "return", + "thread_id": 1 + } + ], + "classMap": [ + { + "name": "simple", + "type": "package", + "children": [ + { + "name": "Simple", + "type": "class", + "children": [ + { + "name": "get_numpy_dict", + "type": "function", + "location": "simple.py:18", + "static": false + }, + { + "name": "hello", + "type": "function", + "location": "simple.py:2", + "static": false + }, + { + "name": "hello_world", + "type": "function", + "location": "simple.py:8", + "static": false + }, + { + "name": "show_numpy_dict", + "type": "function", + "location": "simple.py:11", + "static": false + }, + { + "name": "world", + "type": "function", + "location": "simple.py:5", + "static": false + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/status_errored.metadata.json b/_appmap/test/data/pytest/expected/status_errored.metadata.json index 45b3bed1..1c9d0f21 100644 --- a/_appmap/test/data/pytest/expected/status_errored.metadata.json +++ b/_appmap/test/data/pytest/expected/status_errored.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "RuntimeError: test error", - "location": "test_simple.py:28" + "location": "test_simple.py:30" }, "exception": { "class": "RuntimeError", diff --git a/_appmap/test/data/pytest/expected/status_failed.metadata.json b/_appmap/test/data/pytest/expected/status_failed.metadata.json index cc971c33..cca17c0d 100644 --- a/_appmap/test/data/pytest/expected/status_failed.metadata.json +++ b/_appmap/test/data/pytest/expected/status_failed.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "AssertionError: assert False", - "location": "test_simple.py:14" + "location": "test_simple.py:16" }, "exception": { "class": "AssertionError", diff --git a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json index 992d824d..56494885 100644 --- a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json +++ b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "AssertionError: assert False", - "location": "test_simple.py:19" + "location": "test_simple.py:21" }, "exception": { "class": "AssertionError", diff --git a/_appmap/test/data/pytest/simple.py b/_appmap/test/data/pytest/simple.py index af0338eb..455c80a5 100644 --- a/_appmap/test/data/pytest/simple.py +++ b/_appmap/test/data/pytest/simple.py @@ -1,8 +1,3 @@ -import numpy - -zero = numpy.int64(0) -one = numpy.int64(1) - class Simple: def hello(self): return "Hello" @@ -10,9 +5,15 @@ def hello(self): def world(self): return "world!" - def get_non_json_serializable(self): - return { zero: self.hello(), one: self.world() } - def hello_world(self): - result = self.get_non_json_serializable() - return "%s %s" % (result[zero], result[one]) + return "%s %s" % (self.hello(), self.world()) + + def show_numpy_dict(self): + from numpy import int64 + + d = self.get_numpy_dict({int64(0): "zero", int64(1): "one"}) + print(d) + return d + + def get_numpy_dict(self, d): + return d \ No newline at end of file diff --git a/_appmap/test/data/pytest/test_simple.py b/_appmap/test/data/pytest/test_simple.py index c75c3593..05afaf4b 100644 --- a/_appmap/test/data/pytest/test_simple.py +++ b/_appmap/test/data/pytest/test_simple.py @@ -4,10 +4,12 @@ def test_hello_world(): - import simple + from simple import Simple os.chdir("/tmp") - assert simple.Simple().hello_world() == "Hello world!" + assert Simple().hello_world() == "Hello world!" + + assert len(Simple().show_numpy_dict()) > 0 def test_status_failed(): diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 5f2f0779..815e380b 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -7,11 +7,12 @@ import sys import types from abc import ABC, abstractmethod +from importlib.metadata import version as md_version from pathlib import Path import pytest - -from _appmap import recording, generation +from packaging import version +from _appmap import recording from ..test.helpers import DictIncluding from .normalize import normalize_appmap @@ -101,7 +102,8 @@ def run_tests(self, testdir): def test_enabled(self, testdir): self.run_tests(testdir) assert len(list(testdir.output().iterdir())) == 6 - verify_expected_appmap(testdir) + numpy_version = version.parse(md_version("numpy")) + verify_expected_appmap(testdir, f"-numpy{numpy_version.major}") verify_expected_metadata(testdir) @@ -190,17 +192,18 @@ def output_dir(): return pytester -def verify_expected_appmap(testdir): +def verify_expected_appmap(testdir, suffix=""): appmap_json = list(testdir.output().glob("*test_hello_world.appmap.json")) assert len(appmap_json) == 1 # sanity check generated_appmap = normalize_appmap(appmap_json[0].read_text()) - appmap_json = testdir.expected / (f"{testdir.test_type}.appmap.json") + appmap_json = testdir.expected / (f"{testdir.test_type}{suffix}.appmap.json") expected_appmap = json.loads(appmap_json.read_text()) - assert ( - generated_appmap == expected_appmap - ), f"expected appmap file {appmap_json}\ngenerated appmap: {json.dumps(generated_appmap, indent=2)}" + assert generated_appmap == expected_appmap, ( + f"expected appmap file {appmap_json}\n" + + f"generated appmap: {json.dumps(generated_appmap, indent=2)}" + ) def verify_expected_metadata(testdir): diff --git a/tox.ini b/tox.ini index faeb02af..a5ef8a44 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ deps= poetry web: {[web-deps]deps} py38: numpy==1.24.4 - py3{9,10,11,12}: numpy >=1.26 + py3{9,10,11,12}: numpy >=2 flask2: Flask >= 2.0, <3.0 django3: Django >=3.2, <4.0 sqlalchemy1: sqlalchemy >=1.4.11, <2.0 From d69b6e1648bd647b91ca4f9ef75300af7e015bfb Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 1 Jul 2024 06:54:35 -0400 Subject: [PATCH 054/113] feat: instrument properties Any class function decorated with @property, or any class attribute with a property as a value, will now be instrumented. --- _appmap/event.py | 14 +++- _appmap/importer.py | 55 +++++++++++---- _appmap/test/data/example_class.py | 47 +++++++++++++ _appmap/test/test_params.py | 2 +- _appmap/test/test_properties.py | 105 +++++++++++++++++++++++++++++ _appmap/utils.py | 4 ++ 6 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 _appmap/test/test_properties.py diff --git a/_appmap/event.py b/_appmap/event.py index f53102d2..a333fc78 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -176,7 +176,7 @@ def to_dict(self, value): class CallEvent(Event): # pylint: disable=method-cache-max-size-none - __slots__ = ["_fn", "_fqfn", "static", "receiver", "parameters", "labels"] + __slots__ = ["_fn", "_fqfn", "static", "receiver", "parameters", "labels", "auxtype"] @staticmethod def make(fn, fntype): @@ -283,7 +283,10 @@ def defined_class(self): @property @lru_cache(maxsize=None) def method_id(self): - return self._fqfn.fqfn[1] + ret = self._fqfn.fqfn[1] + if self.auxtype is not None: + ret = f"{ret} ({self.auxtype})" + return ret @property @lru_cache(maxsize=None) @@ -319,6 +322,13 @@ def __init__(self, fn, fntype, parameters, labels): parameters = parameters[1:] self.parameters = parameters self.labels = labels + self.auxtype = None + if fntype & FnType.GET: + self.auxtype = "get" + elif fntype & FnType.SET: + self.auxtype = "set" + elif fntype & FnType.DEL: + self.auxtype = "del" def to_dict(self, attrs=None): ret = super().to_dict() # get the attrs defined in __slots__ diff --git a/_appmap/importer.py b/_appmap/importer.py index cecdf48a..6cff6cdd 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -37,14 +37,14 @@ def __new__(cls, clazz): class FilterableFn( namedtuple( "FilterableFn", - Filterable._fields + ("static_fn",), + Filterable._fields + ("static_fn", "auxtype"), ) ): __slots__ = () - def __new__(cls, scope, fn, static_fn): + def __new__(cls, scope, fn, static_fn, auxtype=None): fqname = "%s.%s" % (scope.fqname, fn.__name__) - self = super(FilterableFn, cls).__new__(cls, scope.scope, fqname, fn, static_fn) + self = super(FilterableFn, cls).__new__(cls, scope.scope, fqname, fn, static_fn, auxtype) return self @property @@ -52,7 +52,10 @@ def fntype(self): if self.scope == Scope.MODULE: return FnType.MODULE - return FnType.classify(self.static_fn) + ret = FnType.classify(self.static_fn) + if self.auxtype is not None: + ret |= self.auxtype + return ret class Filter(ABC): # pylint: disable=too-few-public-methods @@ -122,19 +125,31 @@ def is_member_func(m): # instead iterate over dir(cls), we would see functions from # superclasses, too. Those don't need to be instrumented here, # they'll get taken care of when the superclass is imported. - ret = [] + functions = [] + properties = {} modname = cls.__module__ if hasattr(cls, "__module__") else cls.__name__ for key in cls.__dict__: if key.startswith("__"): continue static_value = inspect.getattr_static(cls, key) - if not is_member_func(static_value): - continue - value = getattr(cls, key) - if value.__module__ != modname: - continue - ret.append((key, static_value, value)) - return ret + if isinstance(static_value, property): + properties[key] = ( + static_value, + { + "fget": (static_value.fget, FnType.GET), + "fset": (static_value.fset, FnType.SET), + "fdel": (static_value.fdel, FnType.DEL), + }, + ) + else: + if not is_member_func(static_value): + continue + value = getattr(cls, key) + if value.__module__ != modname: + continue + functions.append((key, static_value, value)) + + return (functions, properties) class Importer: @@ -177,7 +192,7 @@ def do_import(cls, *args, **kwargs): def instrument_functions(filterable, selected_functions=None): logger.trace(" looking for members of %s", filterable.obj) - functions = get_members(filterable.obj) + functions, properties = get_members(filterable.obj) logger.trace(" functions %s", functions) for fn_name, static_fn, fn in functions: @@ -185,6 +200,20 @@ def instrument_functions(filterable, selected_functions=None): new_fn = cls.instrument_function(fn_name, filterableFn, selected_functions) if new_fn != fn: wrapt.wrap_function_wrapper(filterable.obj, fn_name, new_fn) + # Now that we've instrumented all the functions, go through the properties and update + # them + for prop_name, (prop, prop_fns) in properties.items(): + instrumented_fns = {} + for k, (fn, auxtype) in prop_fns.items(): + if fn is None: + continue + filterableFn = FilterableFn(filterable, fn, fn, auxtype) + new_fn = cls.instrument_function(fn.__name__, filterableFn, selected_functions) + if new_fn != fn: + new_fn = wrapt.FunctionWrapper(fn, new_fn) + instrumented_fns[k] = new_fn + instrumented_fns["doc"] = prop.__doc__ + setattr(filterable.obj, prop_name, property(**instrumented_fns)) # Import Config here, to avoid circular top-level imports. from .configuration import Config # pylint: disable=import-outside-toplevel diff --git a/_appmap/test/data/example_class.py b/_appmap/test/data/example_class.py index 3e61f7ee..46c124f5 100644 --- a/_appmap/test/data/example_class.py +++ b/_appmap/test/data/example_class.py @@ -113,6 +113,53 @@ def with_comment(self): def return_self(self): return self + def __init__(self): + self._read_only = "read only" + self._fully_accessible = "fully accessible" + self._undecorated = "undecorated" + + @property + def read_only(self): + """Read-only""" + return self._read_only + + @property + def fully_accessible(self): + """Fully-accessible""" + return self._fully_accessible + + @fully_accessible.setter + def fully_accessible(self, v): + self._fully_accessible = v + + @fully_accessible.deleter + def fully_accessible(self): + del self._fully_accessible + + def get_undecorated(self): + return self._undecorated + + def set_undecorated(self, value): + self._undecorated = value + + def delete_undecorated(self): + del self._undecorated + + undecorated_property = property(get_undecorated, set_undecorated, delete_undecorated) + + def set_write_only(self, v): + self._write_only = v + + def del_write_only(self): + del self._write_only + + write_only = property(None, set_write_only, del_write_only, "Write-only") + def modfunc(): return "Hello world!" + +if __name__ == "__main__": + ec = ExampleClass() + ec.fully_accessible = "updated" + print(ec.fully_accessible) \ No newline at end of file diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index cbe572bc..7026755b 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -1,4 +1,4 @@ -"""Tests for the function parameter handling""" +"""Tests for function parameter handling""" # pylint: disable=missing-function-docstring diff --git a/_appmap/test/test_properties.py b/_appmap/test/test_properties.py new file mode 100644 index 00000000..c22ae029 --- /dev/null +++ b/_appmap/test/test_properties.py @@ -0,0 +1,105 @@ +"""Tests for methods decorated with @property""" + +# pyright: reportMissingImports=false +# pylint: disable=import-error,import-outside-toplevel +import pytest +from _appmap.test.helpers import DictIncluding + +pytestmark = [ + pytest.mark.appmap_enabled, +] + + +@pytest.fixture(autouse=True) +def setup(with_data_dir): # pylint: disable=unused-argument + # with_data_dir sets up sys.path so example_class can be imported + pass + + +def test_getter_instrumented(events): + from example_class import ExampleClass + + ec = ExampleClass() + + actual = ExampleClass.read_only.__doc__ + assert actual == "Read-only" + + assert ec.read_only == "read only" + + with pytest.raises(AttributeError, match=r".*(has no setter|can't set attribute).*"): + # E AttributeError: can't set attribute + + ec.read_only = "not allowed" + + with pytest.raises(AttributeError, match=r".*(has no deleter|can't delete attribute).*"): + del ec.read_only + + assert len(events) == 2 + assert events[0].to_dict() == DictIncluding( + { + "event": "call", + "defined_class": "example_class.ExampleClass", + "method_id": "read_only (get)", + } + ) + + +def test_accessible_instrumented(events): + from example_class import ExampleClass + + ec = ExampleClass() + assert ExampleClass.fully_accessible.__doc__ == "Fully-accessible" + + assert ec.fully_accessible == "fully accessible" + + ec.fully_accessible = "updated" + # Check the value of the attribute directly, to avoid extra events + assert ec._fully_accessible == "updated" # pylint: disable=protected-access + + del ec.fully_accessible + + # assert len(events) == 6 + assert events[0].to_dict() == DictIncluding( + { + "event": "call", + "defined_class": "example_class.ExampleClass", + "method_id": "fully_accessible (get)", + } + ) + + assert events[2].to_dict() == DictIncluding( + { + "event": "call", + "defined_class": "example_class.ExampleClass", + "method_id": "fully_accessible (set)", + } + ) + + assert events[4].to_dict() == DictIncluding( + { + "event": "call", + "defined_class": "example_class.ExampleClass", + "method_id": "fully_accessible (del)", + } + ) + + +def test_writable_instrumented(events): + from example_class import ExampleClass + + ec = ExampleClass() + assert ExampleClass.write_only.__doc__ == "Write-only" + + with pytest.raises(AttributeError, match=r".*(has no getter|unreadable attribute).*"): + _ = ec.write_only + + ec.write_only = "updated example" + + assert len(events) == 2 + assert events[0].to_dict() == DictIncluding( + { + "event": "call", + "defined_class": "example_class.ExampleClass", + "method_id": "set_write_only (set)", + } + ) diff --git a/_appmap/utils.py b/_appmap/utils.py index 7ac193a6..0b39e947 100644 --- a/_appmap/utils.py +++ b/_appmap/utils.py @@ -35,6 +35,10 @@ class FnType(IntFlag): CLASS = auto() INSTANCE = auto() MODULE = auto() + # auxtypes + GET = auto() + SET = auto() + DEL = auto() @staticmethod def classify(fn): From 660bcdc2f838d86b3425eafcc07773a380d9521a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 3 Jul 2024 15:08:30 +0000 Subject: [PATCH 055/113] chore(release): 2.1.0 [skip ci] # [2.1.0](https://github.com/getappmap/appmap-python/compare/v2.0.10...v2.1.0) (2024-07-03) ### Features * instrument properties ([d69b6e1](https://github.com/getappmap/appmap-python/commit/d69b6e1648bd647b91ca4f9ef75300af7e015bfb)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88aba07..7fd69e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [2.1.0](https://github.com/getappmap/appmap-python/compare/v2.0.10...v2.1.0) (2024-07-03) + + +### Features + +* instrument properties ([d69b6e1](https://github.com/getappmap/appmap-python/commit/d69b6e1648bd647b91ca4f9ef75300af7e015bfb)) + ## [2.0.10](https://github.com/getappmap/appmap-python/compare/v2.0.9...v2.0.10) (2024-06-21) diff --git a/pyproject.toml b/pyproject.toml index c1a369c7..7c70bfd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.0.10" +version = "2.1.0" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 1fc0ce632c920e91e994f901f50bed40a7f5bef4 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 5 Jul 2024 17:18:46 -0400 Subject: [PATCH 056/113] refactor: lint fixes --- _appmap/event.py | 3 ++- _appmap/recorder.py | 3 ++- pylintrc | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index a333fc78..38224660 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -94,7 +94,8 @@ def _describe_schema(name, val, depth, max_depth): if islist: elts = [(None, v) for v in val] schema_key = "items" - elif isdict: + else: + assert isdict elts = val.items() schema_key = "properties" diff --git a/_appmap/recorder.py b/_appmap/recorder.py index c79c898b..ba11680e 100644 --- a/_appmap/recorder.py +++ b/_appmap/recorder.py @@ -114,7 +114,8 @@ def stop_recording(cls): def check_time(cls, event_time): if _MAX_TIME is None: return - if event_time - cls.get_current()._start_time > _MAX_TIME: + delta = event_time - cls.get_current()._start_time # pylint: disable=protected-access + if delta > _MAX_TIME: raise AppMapSessionTooLong(f"Session exceeded {_MAX_TIME} seconds") @classmethod diff --git a/pylintrc b/pylintrc index b8959689..ecc361be 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,6 @@ [MAIN] # Specify a score threshold under which the program will exit with error. -fail-under=9.86 +fail-under=9.87 # Analyse import fallback blocks. This can be used to support both Python 2 and From 22eae792a2c8bc74ebb7b594361edc87ebc81f1e Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 8 Jul 2024 05:21:45 -0400 Subject: [PATCH 057/113] refactor: upgrade from disutils.dir_util to shutil --- _appmap/test/conftest.py | 4 ++-- _appmap/test/test_command.py | 4 ++-- _appmap/test/test_configuration.py | 22 +++++++++++----------- _appmap/test/test_recording.py | 11 +++++------ pylintrc | 2 +- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index 939f8eca..2dd0f2f9 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -2,7 +2,7 @@ import os import socket import sys -from distutils.dir_util import copy_tree +from shutil import copytree from functools import partial, partialmethod from pathlib import Path from typing import Any @@ -98,7 +98,7 @@ def git_directory_fixture(tmp_path_factory): @pytest.fixture(name="git") def tmp_git(git_directory, tmp_path): - copy_tree(git_directory, str(tmp_path)) + copytree(git_directory, str(tmp_path), dirs_exist_ok=True) return utils.git(cwd=tmp_path) diff --git a/_appmap/test/test_command.py b/_appmap/test/test_command.py index 6da6abc6..2f666982 100644 --- a/_appmap/test/test_command.py +++ b/_appmap/test/test_command.py @@ -1,6 +1,6 @@ import json import re -from distutils.dir_util import copy_tree +from shutil import copytree from importlib.metadata import version import pytest @@ -14,7 +14,7 @@ @pytest.fixture(name="_cmd_setup") def _cmd_setup(request, git, data_dir, monkeypatch): repo_root = git.cwd - copy_tree(data_dir / request.param, str(repo_root)) + copytree(data_dir / request.param, str(repo_root), dirs_exist_ok=True) monkeypatch.chdir(repo_root) # pylint: disable=protected-access diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index ecd73cb4..d5e2ed7a 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -2,7 +2,7 @@ # pylint: disable=missing-function-docstring from contextlib import contextmanager -from distutils.dir_util import copy_tree +from shutil import copytree from pathlib import Path from textwrap import dedent @@ -154,7 +154,7 @@ def check_default_config(self, expected_name): class TestDefaultConfig(DefaultHelpers): def test_created(self, git, data_dir, monkeypatch): repo_root = git.cwd - copy_tree(data_dir / "config", str(repo_root)) + copytree(data_dir / "config", str(repo_root), dirs_exist_ok=True) monkeypatch.chdir(repo_root) # pylint: disable=protected-access @@ -163,7 +163,7 @@ def test_created(self, git, data_dir, monkeypatch): self.check_default_config(repo_root.name) def test_created_outside_repo(self, data_dir, tmpdir, monkeypatch): - copy_tree(data_dir / "config", str(tmpdir)) + copytree(data_dir / "config", str(tmpdir), dirs_exist_ok=True) monkeypatch.chdir(tmpdir) # pylint: disable=protected-access @@ -180,7 +180,7 @@ def test_skipped_when_overridden(self): assert not appmap.enabled() def test_exclusions(self, data_dir, tmpdir, mocker, monkeypatch): - copy_tree(data_dir / "config-exclude", str(tmpdir)) + copytree(data_dir / "config-exclude", str(tmpdir), dirs_exist_ok=True) monkeypatch.chdir(tmpdir) mocker.patch( "_appmap.configuration._get_sys_prefix", @@ -193,7 +193,7 @@ def test_exclusions(self, data_dir, tmpdir, mocker, monkeypatch): def test_created_if_missing_and_enabled(self, git, data_dir, monkeypatch, tmpdir): repo_root = git.cwd - copy_tree(data_dir / "config", str(repo_root)) + copytree(data_dir / "config", str(repo_root), dirs_exist_ok=True) monkeypatch.chdir(repo_root) path = Path(repo_root / "appmap.yml") @@ -213,7 +213,7 @@ def test_created_if_missing_and_enabled(self, git, data_dir, monkeypatch, tmpdir def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch): repo_root = git.cwd - copy_tree(data_dir / "config", str(repo_root)) + copytree(data_dir / "config", str(repo_root), dirs_exist_ok=True) monkeypatch.chdir(repo_root) path = Path(repo_root / "appmap.yml") @@ -229,7 +229,7 @@ def test_not_created_if_missing_and_not_enabled(self, git, data_dir, monkeypatch class TestEmpty(DefaultHelpers): @pytest.fixture(autouse=True) def setup_config(self, data_dir, monkeypatch, tmpdir): - copy_tree(data_dir / "config", str(tmpdir)) + copytree(data_dir / "config", str(tmpdir), dirs_exist_ok=True) monkeypatch.chdir(tmpdir) @contextmanager @@ -269,7 +269,7 @@ class TestSearchConfig: # pylint: disable=too-many-arguments def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): - copy_tree(data_dir / "config-up", str(tmpdir)) + copytree(data_dir / "config-up", str(tmpdir), dirs_exist_ok=True) wd = tmpdir / "project" / "p1" monkeypatch.chdir(wd) @@ -279,8 +279,8 @@ def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): assert str(Env.current.output_dir).endswith(str(tmpdir / "tmp" / "appmap")) def _init_repo(self, data_dir, tmpdir, git_directory, repo_root, appmapdir): - copy_tree(data_dir / "config-up", str(tmpdir)) - copy_tree(git_directory, str(repo_root)) + copytree(data_dir / "config-up", str(tmpdir), dirs_exist_ok=True) + copytree(git_directory, str(repo_root), dirs_exist_ok=True) with open(appmapdir / "appmap.yml", "w+", encoding="utf-8") as f: f.writelines( dedent(""" @@ -325,7 +325,7 @@ def test_config_above_repo_root(self, data_dir, tmpdir, git_directory, monkeypat assert Env.current.enabled def test_config_not_found_in_path_hierarchy(self, data_dir, tmpdir, monkeypatch): - copy_tree(data_dir / "config-up", str(tmpdir)) + copytree(data_dir / "config-up", str(tmpdir), dirs_exist_ok=True) wd = tmpdir / "project" / "p1" monkeypatch.chdir(wd) diff --git a/_appmap/test/test_recording.py b/_appmap/test/test_recording.py index 1edbd153..6fcc763e 100644 --- a/_appmap/test/test_recording.py +++ b/_appmap/test/test_recording.py @@ -3,13 +3,12 @@ import json import os -from distutils.dir_util import copy_tree -from distutils.file_util import copy_file +from shutil import copy, copytree from threading import Thread +import appmap import pytest -import appmap from _appmap.event import Event from _appmap.recorder import Recorder, ThreadRecorder from _appmap.wrapt import FunctionWrapper @@ -193,9 +192,9 @@ def add_event(name): def test_process_recording(data_dir, shell, tmp_path): fixture = data_dir / "package1" tmp = tmp_path / "process" - copy_tree(fixture, str(tmp / "package1")) - copy_file(data_dir / "appmap.yml", str(tmp)) - copy_tree(data_dir / "flask" / "init", str(tmp / "init")) + copytree(fixture, str(tmp / "package1"), dirs_exist_ok=True) + copy(data_dir / "appmap.yml", str(tmp)) + copytree(data_dir / "flask" / "init", str(tmp / "init"), dirs_exist_ok=True) ret = shell.run( "python", diff --git a/pylintrc b/pylintrc index ecc361be..9bf0f510 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,6 @@ [MAIN] # Specify a score threshold under which the program will exit with error. -fail-under=9.87 +fail-under=9.94 # Analyse import fallback blocks. This can be used to support both Python 2 and From 842d973f23f79a94a754203fdf57b29460594f19 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sun, 7 Jul 2024 13:40:01 -0400 Subject: [PATCH 058/113] ci: add pytest-xdist Add xdist to speed up test runs some. --- _appmap/test/conftest.py | 14 +++++++++++--- _appmap/test/web_framework.py | 26 +++++++++++--------------- pyproject.toml | 2 ++ pytest.ini | 7 ++++--- tox.ini | 10 ++++++---- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index 2dd0f2f9..ff838243 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -14,7 +14,7 @@ import _appmap import appmap -from _appmap.test.web_framework import TEST_HOST, TEST_PORT +from _appmap.test.web_framework import TEST_HOST from appmap import generation from .. import utils @@ -198,14 +198,22 @@ def _starter(controldir, xprocess): return _starter +@pytest.fixture(name="server_port") +def server_port_fixture(worker_id): + if worker_id == "master": + offset = "0" + else: + offset = worker_id[2:] + return 8000 + int(offset) + @pytest.fixture(name="server_base") -def server_base_fixture(request): +def server_base_fixture(request, server_port): marker = request.node.get_closest_marker("server") debug = marker.kwargs.get("debug", False) server_env = os.environ.copy() server_env.update(marker.kwargs.get("env", {})) - info = ServerInfo(debug=debug, host=TEST_HOST, port=TEST_PORT, env=server_env) + info = ServerInfo(debug=debug, host=TEST_HOST, port=server_port, env=server_env) info.factory = partial(server_starter, info) return info diff --git a/_appmap/test/web_framework.py b/_appmap/test/web_framework.py index 699b4dfa..0729cace 100644 --- a/_appmap/test/web_framework.py +++ b/_appmap/test/web_framework.py @@ -5,6 +5,7 @@ import json import multiprocessing import os +import re import time import traceback from os.path import exists @@ -20,7 +21,6 @@ from .normalize import normalize_appmap TEST_HOST = "127.0.0.1" -TEST_PORT = 8000 _SR = SystemRandom() @@ -323,28 +323,25 @@ def test_can_record(self, data_dir, client): res = client.delete("/_appmap/record") assert res.status_code == 404 +@pytest.mark.xdist_group("group1") class _TestRecordRequests: """Common tests for per-requests recording (record requests.)""" @classmethod - def server_url(cls): - return f"http://{TEST_HOST}:{TEST_PORT}" - - @classmethod - def record_request_thread(cls): + def record_request_thread(cls, server_url): # I've seen occasional test failures, seemingly because the test servers can't handle the # barrage of requests. A tiny bit of delay still causes many, many concurrent requests, but # eliminates the failures. time.sleep(_SR.uniform(0, 0.1)) - return requests.get(cls.server_url() + "/test", timeout=30) + return requests.get(server_url + "/test", timeout=30) - def record_requests(self, record_remote): + def record_requests(self, record_remote, server_url): # pylint: disable=too-many-locals if record_remote: # when remote recording is enabled, this test also # verifies the global recorder doesn't save duplicate # events when per-request recording is enabled - response = requests.post(self.server_url() + "/_appmap/record", timeout=30) + response = requests.post(server_url + "/_appmap/record", timeout=30) assert response.status_code == 200 with concurrent.futures.ThreadPoolExecutor( @@ -354,7 +351,7 @@ def record_requests(self, record_remote): max_number_of_threads = 400 future_to_request_number = {} for n in range(max_number_of_threads): - future = executor.submit(self.record_request_thread) + future = executor.submit(self.record_request_thread, server_url) future_to_request_number[future] = n # wait for all threads to complete @@ -384,9 +381,8 @@ def record_requests(self, record_remote): appmap_file_name_basename_part = "_".join( appmap_file_name_basename.split("_")[2:] ) - assert ( - appmap_file_name_basename_part - == "http_127_0_0_1_8000_test.appmap.json" + assert re.match( + r"http_127_0_0_1_8[0-9]*_test.appmap.json", appmap_file_name_basename_part ) with open(appmap_file_name, encoding="utf-8") as f: @@ -414,12 +410,12 @@ def record_requests(self, record_remote): @pytest.mark.appmap_enabled @pytest.mark.server(debug=True) def test_record_requests_with_remote(self, server): - self.record_requests(server.debug) + self.record_requests(server.debug, server.url) @pytest.mark.appmap_enabled @pytest.mark.server(debug=False) def test_record_requests_without_remote(self, server): - self.record_requests(server.debug) + self.record_requests(server.debug, server.url) @pytest.mark.server(debug=False) def test_remote_disabled_in_prod(self, server): diff --git a/pyproject.toml b/pyproject.toml index 7c70bfd3..4ea6aec5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ fastapi = "^0.110.0" httpx = "^0.27.0" pytest-env = "^1.1.3" pytest-console-scripts = "^1.4.1" +pytest-xdist = "^3.6.1" +psutil = "^6.0.0" [build-system] requires = ["poetry-core>=1.1.0"] diff --git a/pytest.ini b/pytest.ini index 7d9408ce..6c14b6fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,9 +10,10 @@ markers = testpaths = _appmap/test pytester_example_dir = _appmap/test/data -# running in a subprocess ensures that environment variables are set -# correctly and no classes are loaded. -addopts = --runpytest subprocess --ignore vendor +# running in a subprocess ensures that environment variables are set correctly and no classes are +# loaded. Also, the remote-recording tests can't be run in parallel, so they're marked to run in the +# same load group and distribution is done by loadgroup. +addopts = --runpytest subprocess --ignore vendor --tb=short --dist loadgroup # We're stuck at pytest ~6.1.2. This warning got removed in a later # version. diff --git a/tox.ini b/tox.ini index a5ef8a44..e22265fd 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ deps= sqlalchemy >=2.0, <3.0 [testenv] +passenv = + PYTEST_XDIST_AUTO_NUM_WORKERS allowlist_externals = env bash @@ -26,10 +28,10 @@ deps= commands = poetry install -v - web: poetry run appmap-python {posargs:pytest} - django3: poetry run appmap-python pytest _appmap/test/test_django.py - flask2: poetry run appmap-python pytest _appmap/test/test_flask.py - sqlalchemy1: poetry run appmap-python pytest _appmap/test/test_sqlalchemy.py + web: poetry run appmap-python {posargs:pytest -n logical} + django3: poetry run appmap-python pytest -n logical _appmap/test/test_django.py + flask2: poetry run appmap-python pytest -n logical _appmap/test/test_flask.py + sqlalchemy1: poetry run appmap-python pytest -n logical _appmap/test/test_sqlalchemy.py [testenv:lint] skip_install = True From f970fb7828edc3e5d19aa03e48e9302077d58614 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 8 Jul 2024 09:34:33 -0400 Subject: [PATCH 059/113] fix: Flask events are ordered correctly Currently, if the config specifies that flask should be instrumented, an http_server_response event gets recorded between the call and return events for Flask.finalize_request. Also, a return event with an exception will get recorded between the call and return events for Flask.handle_user_exception. These changes hook finalize_request and handle_user_exception, and ensure that the events get ordered correctly. --- _appmap/env.py | 2 + _appmap/event.py | 19 +-- _appmap/test/conftest.py | 33 +++++ .../test/data/flask-instrumented/appmap.yml | 4 + .../test/data/flask-instrumented/flaskapp.py | 30 +++++ .../flask-instrumented/init/sitecustomize.py | 1 + .../test/data/flask-instrumented/test_app.py | 31 +++++ _appmap/test/data/flask/flaskapp.py | 13 +- _appmap/test/data/flask/test_app.py | 12 ++ _appmap/test/helpers.py | 22 ++++ _appmap/test/test_django.py | 3 - _appmap/test/test_flask.py | 116 ++++++++++++++++-- _appmap/test/test_test_frameworks.py | 40 +----- _appmap/web_framework.py | 37 ++++-- appmap/django.py | 14 ++- appmap/fastapi.py | 14 ++- appmap/flask.py | 73 +++++++---- 17 files changed, 363 insertions(+), 101 deletions(-) create mode 100644 _appmap/test/data/flask-instrumented/appmap.yml create mode 100644 _appmap/test/data/flask-instrumented/flaskapp.py create mode 100644 _appmap/test/data/flask-instrumented/init/sitecustomize.py create mode 100644 _appmap/test/data/flask-instrumented/test_app.py diff --git a/_appmap/env.py b/_appmap/env.py index b74683bc..29c182cd 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -39,6 +39,8 @@ def __init__(self, env=None, cwd=None): self.log_file_creation_failed = False self._configure_logging() + # This uses _APPMAP, rather than APPMAP, to control whether instrumentation is enabled. The + # tests use this split to make it easier to control recording. enabled = self._env.get("_APPMAP", "false") self._enabled = enabled is None or enabled.lower() != "false" diff --git a/_appmap/event.py b/_appmap/event.py index 38224660..82661152 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -480,18 +480,19 @@ class HttpResponseEvent(ReturnEvent): def __init__(self, status_code, headers=None, **kwargs): super().__init__(**kwargs) + self.response = {} + self.update(status_code, headers) - response = {"status_code": status_code} + def update(self, status_code, headers): + if status_code is not None: + self.response.update({"status_code": status_code}) if headers is not None: - response.update( - { - "mime_type": headers.get("Content-Type"), - "headers": none_if_empty(dict(headers)), - } - ) - - self.response = compact_dict(response) + if "Content-Type" in headers: + self.response.update({"mime_type": headers.get("Content-Type")}) + updated_headers = dict(headers) + if len(updated_headers) > 0: + self.response.update({"headers": updated_headers}) # pylint: disable=too-few-public-methods diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index ff838243..211688e1 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -217,3 +217,36 @@ def server_base_fixture(request, server_port): info = ServerInfo(debug=debug, host=TEST_HOST, port=server_port, env=server_env) info.factory = partial(server_starter, info) return info + +@pytest.fixture(name="testdir") +def testdir_fixture(request, data_dir, pytester, monkeypatch): + # We need to set environment variables to control how tests are run. This will only work + # properly if pytester runs pytest in a subprocess. + assert ( + pytester._method == "subprocess" # pylint:disable=protected-access + ), "must run pytest in a subprocess" + + # The init subdirectory contains a sitecustomize.py file that + # imports the appmap module. This simulates the way a real + # installation works, performing the same function as the the + # appmap.pth file that gets put in site-packages. + monkeypatch.setenv("PYTHONPATH", "init") + + # Make sure _APPMAP isn't in the environment, to test that recording-by-default is working as + # expected. Individual test cases may set it as necessary. + monkeypatch.delenv("_APPMAP", raising=False) + + marker = request.node.get_closest_marker("example_dir") + test_type = "unittest" if marker is None else marker.args[0] + pytester.copy_example(test_type) + + pytester.expected = data_dir / test_type / "expected" + pytester.test_type = test_type + + # this is so test_type can be overriden in test cases + def output_dir(): + return pytester.path / "tmp" / "appmap" / pytester.test_type + + pytester.output = output_dir + + return pytester diff --git a/_appmap/test/data/flask-instrumented/appmap.yml b/_appmap/test/data/flask-instrumented/appmap.yml new file mode 100644 index 00000000..ce1edabb --- /dev/null +++ b/_appmap/test/data/flask-instrumented/appmap.yml @@ -0,0 +1,4 @@ +name: FlaskTest +packages: +- path: flaskapp +- path: flask diff --git a/_appmap/test/data/flask-instrumented/flaskapp.py b/_appmap/test/data/flask-instrumented/flaskapp.py new file mode 100644 index 00000000..4547c81a --- /dev/null +++ b/_appmap/test/data/flask-instrumented/flaskapp.py @@ -0,0 +1,30 @@ +""" +Rudimentary Flask application for testing. +""" +# pylint: disable=missing-function-docstring + +import werkzeug +from appmap.flask import AppmapFlask +from flask import Flask, request + +app = Flask(__name__) +AppmapFlask(app).init_app() + + +@app.route("/") +def hello_world(): + return "Hello, World!" + +@app.route("/exception") +def raise_exception(): + raise Exception("An exception") + +@app.post("/do_post") +def do_post(): + _ = request.get_json() + return "Got post request" + + +@app.errorhandler(werkzeug.exceptions.BadRequest) +def handle_bad_request(e): + return "That's a bad request!", 400 \ No newline at end of file diff --git a/_appmap/test/data/flask-instrumented/init/sitecustomize.py b/_appmap/test/data/flask-instrumented/init/sitecustomize.py new file mode 100644 index 00000000..d1fe4fec --- /dev/null +++ b/_appmap/test/data/flask-instrumented/init/sitecustomize.py @@ -0,0 +1 @@ +import appmap diff --git a/_appmap/test/data/flask-instrumented/test_app.py b/_appmap/test/data/flask-instrumented/test_app.py new file mode 100644 index 00000000..82c5be53 --- /dev/null +++ b/_appmap/test/data/flask-instrumented/test_app.py @@ -0,0 +1,31 @@ +import pytest +from flaskapp import app + + +@pytest.fixture(name="client") +def test_client(): + with app.test_client() as c: # pylint: disable=no-member + yield c + + +def test_request(client): + response = client.get("/") + + assert response.status_code == 200 + +def test_exception(client): + response = client.get("/exception") + + assert response.status_code == 500 + +def test_not_found(client): + response = client.get("/not_found") + + assert response.status_code == 404 + + +def test_errorhandler(client): + response = client.post("/do_post", content_type="application/json") + + assert response.status_code == 400 + assert response.text == "That's a bad request!" diff --git a/_appmap/test/data/flask/flaskapp.py b/_appmap/test/data/flask/flaskapp.py index a2c819b9..69e2c800 100644 --- a/_appmap/test/data/flask/flaskapp.py +++ b/_appmap/test/data/flask/flaskapp.py @@ -6,8 +6,9 @@ """ # pylint: disable=missing-function-docstring -from flask import Flask, make_response +from flask import Flask, make_response, request from markupsafe import escape +import werkzeug app = Flask(__name__) @@ -50,3 +51,13 @@ def show_org_user_posts(org, username): @app.route("/exception") def raise_exception(): raise Exception("An exception") + +@app.post("/do_post") +def do_post(): + _ = request.get_json() + return "Got post request" + + +@app.errorhandler(werkzeug.exceptions.BadRequest) +def handle_bad_request(e): + return "That's a bad request!", 400 \ No newline at end of file diff --git a/_appmap/test/data/flask/test_app.py b/_appmap/test/data/flask/test_app.py index eed359d1..891801bb 100644 --- a/_appmap/test/data/flask/test_app.py +++ b/_appmap/test/data/flask/test_app.py @@ -12,3 +12,15 @@ def test_request(client): response = client.get("/") assert response.status_code == 200 + +def test_not_found(client): + response = client.get("/not_found") + + assert response.status_code == 404 + + +def test_errorhandler(client): + response = client.post("/do_post", content_type="application/json") + + assert response.status_code == 400 + assert response.text == "That's a bad request!" diff --git a/_appmap/test/helpers.py b/_appmap/test/helpers.py index 342c35e8..3e183ef3 100644 --- a/_appmap/test/helpers.py +++ b/_appmap/test/helpers.py @@ -1,6 +1,11 @@ """Test helpers""" +import importlib.metadata + +from packaging import version as pkg_version + + class DictIncluding(dict): """A dict that on comparison just checks whether the other dict includes all of its items. Any extra ones are ignored. @@ -26,3 +31,20 @@ def __eq__(self, other): if v is None: return False return True + + +def package_version(pkg): + return pkg_version.parse(importlib.metadata.version(pkg)) + + +def check_call_stack(events): + """Ensure that the call stack in events has balanced call and return events""" + stack = [] + for e in events: + if e.get("event") == "call": + stack.append(e) + elif e.get("event") == "return": + assert len(stack) > 0, "return without call" + call = stack.pop() + assert call.get("id") == e.get("parent_id") + assert len(stack) == 0, "leftover events" diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index 730c7f36..dec17809 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -171,9 +171,6 @@ def raise_on_call(*args): assert events[1].event == "return" assert events[1].parent_id == events[0].id - assert events[1].exceptions == [ - DictIncluding({"class": "builtins.RuntimeError", "message": "An error"}) - ] @pytest.mark.appmap_enabled(env={"APPMAP_RECORD_REQUESTS": "false"}) diff --git a/_appmap/test/test_flask.py b/_appmap/test/test_flask.py index 9bb20c50..8512d5f1 100644 --- a/_appmap/test/test_flask.py +++ b/_appmap/test/test_flask.py @@ -2,6 +2,7 @@ # pylint: disable=missing-function-docstring import importlib +import json import os from importlib.metadata import version from types import SimpleNamespace as NS @@ -13,7 +14,7 @@ from _appmap.metadata import Metadata from appmap.flask import AppmapFlask -from ..test.helpers import DictIncluding +from ..test.helpers import DictIncluding, check_call_stack, package_version from .web_framework import ( _TestFormCapture, _TestFormData, @@ -22,7 +23,6 @@ _TestRequestCapture, ) - class TestFormCapture(_TestFormCapture): pass @@ -77,18 +77,59 @@ def test_framework_metadata(client, events): # pylint: disable=unused-argument @pytest.mark.appmap_enabled(env={"APPMAP_RECORD_REQUESTS": "false"}) -def test_exception(client, events): # pylint: disable=unused-argument +def test_exception(client, events): with pytest.raises(Exception): client.get("/exception") assert events[0].http_server_request == DictIncluding( {"request_method": "GET", "path_info": "/exception", "protocol": "HTTP/1.1"} ) + + assert events[1].event == "return" + assert events[1].parent_id == events[0].id + assert events[1].http_server_response["status_code"] == 500 + + +@pytest.mark.appmap_enabled(env={"APPMAP_RECORD_REQUESTS": "false"}) +def test_not_found(client, events): + client.get("/not_found") + + assert events[0].http_server_request == DictIncluding( + {"request_method": "GET", "path_info": "/not_found", "protocol": "HTTP/1.1"} + ) + + assert events[1].event == "return" + assert events[1].parent_id == events[0].id + assert events[1].http_server_response["status_code"] == 404 + + +@pytest.mark.appmap_enabled(env={"APPMAP_RECORD_REQUESTS": "false"}) +def test_bad_request(client, events): + client.post("/test") + + assert events[0].http_server_request == DictIncluding( + {"request_method": "POST", "path_info": "/test", "protocol": "HTTP/1.1"} + ) + + assert events[1].event == "return" + assert events[1].parent_id == events[0].id + assert events[1].http_server_response["status_code"] == 405 + + +@pytest.mark.appmap_enabled(env={"APPMAP_RECORD_REQUESTS": "false"}) +def test_errorhandler(client, events): + response = client.post("/do_post", content_type="application/json") + + # Verify that the custom errorhandler was used + assert response.text == "That's a bad request!" + + assert events[0].http_server_request == DictIncluding( + {"request_method": "POST", "path_info": "/do_post", "protocol": "HTTP/1.1"} + ) + assert events[1].event == "return" assert events[1].parent_id == events[0].id - assert events[1].exceptions == [ - DictIncluding({"class": "builtins.Exception", "message": "An exception"}) - ] + assert events[1].http_server_response["status_code"] == 400 @pytest.mark.appmap_enabled @@ -140,7 +181,7 @@ def beforeEach(self, monkeypatch, pytester): def test_enabled(self, pytester): result = pytester.runpytest("-svv") - result.assert_outcomes(passed=1, failed=0, errors=0) + result.assert_outcomes(passed=3, failed=0, errors=0) appmap_file = ( pytester.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json" ) @@ -156,7 +197,7 @@ def test_disabled(self, pytester, monkeypatch): result = pytester.runpytest("-svv") - result.assert_outcomes(passed=1, failed=0, errors=0) + result.assert_outcomes(passed=3, failed=0, errors=0) assert not (pytester.path / "tmp" / "appmap").exists() def test_disabled_for_process(self, pytester, monkeypatch): @@ -164,8 +205,65 @@ def test_disabled_for_process(self, pytester, monkeypatch): result = pytester.runpytest("-svv") - result.assert_outcomes(passed=1, failed=0, errors=0) + result.assert_outcomes(passed=3, failed=0, errors=0) assert (pytester.path / "tmp" / "appmap" / "process").exists() assert not (pytester.path / "tmp" / "appmap" / "requests").exists() assert not (pytester.path / "tmp" / "appmap" / "pytest").exists() + +def verify_events(events): + def find(type): + return next(filter(lambda e: e[1].get(type) is not None, enumerate(events)), None) + + request = find("http_server_request") + assert request is not None + request_idx, request_event = request + + response = find("http_server_response") + assert response is not None + response_idx, response_event = response + + assert response_event.get("parent_id") == request_event.get("id") + + nested_events = events[request_idx + 1 : response_idx] + check_call_stack(nested_events) + + +@pytest.mark.example_dir("flask-instrumented") +class TestFlaskInstrumented: + + def test_all(self, testdir): + result = testdir.runpytest("-svv") + result.assert_outcomes(passed=4) + + def test_response(self, testdir): + result = testdir.runpytest("-svv", "-k", "test_request") + result.assert_outcomes(passed=1) + + appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json" + appmap = json.load(appmap_file.open()) + verify_events(appmap["events"]) + + def test_unhandled_exception(self, testdir): + result = testdir.runpytest("-svv", "-k", "test_exception") + result.assert_outcomes(passed=1) + + appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_exception.appmap.json" + appmap = json.load(appmap_file.open()) + verify_events(appmap["events"]) + + def test_default_exception(self, testdir): + result = testdir.runpytest("-svv", "-k", "test_not_found") + result.assert_outcomes(passed=1) + + appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_not_found.appmap.json" + appmap = json.load(appmap_file.open()) + verify_events(appmap["events"]) + + def test_errorhandler(self, testdir): + result = testdir.runpytest("-svv", "-k", "test_errorhandler") + result.assert_outcomes(passed=1) + + appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_errorhandler.appmap.json" + appmap = json.load(appmap_file.open()) + verify_events(appmap["events"]) diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 815e380b..1fc73ea7 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -7,12 +7,11 @@ import sys import types from abc import ABC, abstractmethod -from importlib.metadata import version as md_version from pathlib import Path import pytest -from packaging import version from _appmap import recording +from _appmap.test.helpers import package_version from ..test.helpers import DictIncluding from .normalize import normalize_appmap @@ -102,11 +101,10 @@ def run_tests(self, testdir): def test_enabled(self, testdir): self.run_tests(testdir) assert len(list(testdir.output().iterdir())) == 6 - numpy_version = version.parse(md_version("numpy")) + numpy_version = package_version("numpy") verify_expected_appmap(testdir, f"-numpy{numpy_version.major}") verify_expected_metadata(testdir) - @pytest.mark.example_dir("trial") class TestPytestRunnerTrial(_TestTestRunner): @classmethod @@ -158,40 +156,6 @@ def test_write_appmap(recorder_outdir): assert (recorder_outdir / expected_shortname).read_text().startswith('{"version"') -@pytest.fixture(name="testdir") -def fixture_runner_testdir(request, data_dir, pytester, monkeypatch): - # We need to set environment variables to control how tests are run. This will only work - # properly if pytester runs pytest in a subprocess. - assert ( - pytester._method == "subprocess" # pylint:disable=protected-access - ), "must run pytest in a subprocess" - - # The init subdirectory contains a sitecustomize.py file that - # imports the appmap module. This simulates the way a real - # installation works, performing the same function as the the - # appmap.pth file that gets put in site-packages. - monkeypatch.setenv("PYTHONPATH", "init") - - # Make sure APPMAP isn't the environment, to test that recording-by-default is working as - # expected. Individual test cases may set it as necessary. - monkeypatch.delenv("_APPMAP", raising=False) - - marker = request.node.get_closest_marker("example_dir") - test_type = "unittest" if marker is None else marker.args[0] - pytester.copy_example(test_type) - - pytester.expected = data_dir / test_type / "expected" - pytester.test_type = test_type - - # this is so test_type can be overriden in test cases - def output_dir(): - return pytester.path / "tmp" / "appmap" / pytester.test_type - - pytester.output = output_dir - - return pytester - - def verify_expected_appmap(testdir, suffix=""): appmap_json = list(testdir.output().glob("*test_hello_world.appmap.json")) assert len(appmap_json) == 1 # sanity check diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index aebf08a7..33e27e17 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -143,16 +143,28 @@ def before_request_main(self, rec, req: Any) -> Tuple[float, int]: raise NotImplementedError # pylint: disable=too-many-arguments - def after_request_main(self, rec, status, headers, start, call_event_id) -> None: + def after_request_main( + self, request_path, status, headers, start, call_event_id + ) -> Optional[HttpServerResponseEvent]: + if request_path == self.record_url: + return None - duration = time.monotonic() - start - return_event = HttpServerResponseEvent( - parent_id=call_event_id, - elapsed=duration, - status_code=status, - headers=headers, - ) - rec.add_event(return_event) + env = Env.current + if env.enables("requests") or env.enables("remote"): + rec = request_recorder.get() if env.enables("requests") else Recorder.get_global() + assert rec is not None + + duration = time.monotonic() - start + return_event = HttpServerResponseEvent( + parent_id=call_event_id, + elapsed=duration, + status_code=status, + headers=headers, + ) + rec.add_event(return_event) + return return_event + + return None def __init__(self, framework_name): self.record_url = "/_appmap/record" @@ -189,8 +201,7 @@ def after_request_hook( request_base_url, status, headers, - start, - call_event_id, + return_event, ) -> None: if request_path == self.record_url: return @@ -201,7 +212,7 @@ def after_request_hook( assert rec is not None try: - self.after_request_main(rec, status, headers, start, call_event_id) + return_event.update(status, headers) output_dir = Env.current.output_dir / "requests" create_appmap_file( @@ -221,7 +232,7 @@ def after_request_hook( rec = Recorder.get_global() assert rec is not None if rec.get_enabled(): - self.after_request_main(rec, status, headers, start, call_event_id) + return_event.update(status, headers) def on_exception(self, rec, start, call_event_id, exc_info): duration = time.monotonic() - start diff --git a/appmap/django.py b/appmap/django.py index 2e393c0d..08674946 100644 --- a/appmap/django.py +++ b/appmap/django.py @@ -224,16 +224,24 @@ def __call__(self, request): self.on_exception(rec, start, call_event_id, sys.exc_info()) raise - self.after_request_hook( + return_event = self.after_request_main( request.path_info, - request.method, - request.build_absolute_uri(), response.status_code, response.headers, start, call_event_id, ) + if return_event is not None: + self.after_request_hook( + request.path_info, + request.method, + request.build_absolute_uri(), + response.status_code, + response.headers, + return_event, + ) + return response def before_request_main(self, rec, req): diff --git a/appmap/fastapi.py b/appmap/fastapi.py index 170ae331..bfedba6a 100644 --- a/appmap/fastapi.py +++ b/appmap/fastapi.py @@ -117,15 +117,23 @@ async def _dispatch(self, request, call_next): parsed = request.url.components baseurl = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", "")) - self.after_request_hook( + return_event = self.after_request_main( request.url.path, - request.method, - baseurl, response.status_code, response.headers, start, call_event_id, ) + if return_event is not None: + self.after_request_hook( + request.url.path, + request.method, + baseurl, + response.status_code, + response.headers, + return_event, + ) + return response async def _parse_json(self, request): diff --git a/appmap/flask.py b/appmap/flask.py index ebc56df6..2faf0c84 100644 --- a/appmap/flask.py +++ b/appmap/flask.py @@ -1,10 +1,11 @@ import re import time from importlib.metadata import version +from types import SimpleNamespace import jinja2 -from flask import g, got_request_exception, request, request_finished, request_started -from flask.cli import ScriptInfo +from blinker import signal +from flask import g, request, request_finished, request_started from werkzeug.exceptions import BadRequest, UnsupportedMediaType from werkzeug.middleware.dispatcher import DispatcherMiddleware @@ -59,6 +60,9 @@ def request_params(req): NP_PARAMS = re.compile(r"", "{}") +_after_finalize = signal("_appmap_after_finalize") +_before_exception = signal("_appmap_before_exception") + class AppmapFlask(AppmapMiddleware): """ @@ -91,7 +95,7 @@ def init_app(self): request_started.connect(self.request_started, self.app, weak=False) request_finished.connect(self.request_finished, self.app, weak=False) - got_request_exception.connect(self.got_request_exception, self.app, weak=False) + _after_finalize.connect(self.after_finalize, sender=self.app, weak=False) setattr(self.app, REQUEST_ENABLED_ATTR, True) @@ -130,33 +134,37 @@ def before_request_main(self, rec, req): # current Context before signaling, which removes our ContextVar. # TODO: enhance AppmapMiddleware so it allows subclasses to specify how # the request recording should be stored. - g._appmap_recorder = rec # pylint: disable=protected-access - g._appmap_request_event = call_event # pylint: disable=protected-access - g._appmap_request_start = time.monotonic() # pylint: disable=protected-access + g.appmap_recorder = rec + g.appmap_request_event = call_event + g.appmap_request_start = time.monotonic() return None, None - def request_finished(self, _, response, **__): + def after_finalize(self, _, **__): if not self.should_record: - return response + return + + return_event = self.after_request_main( + request.path, + None, + None, + g.appmap_request_start, + g.appmap_request_event.id, + ) self.after_request_hook( request.path, request.method, request.base_url, - response.status_code, - response.headers, - g._appmap_request_start, # pylint: disable=protected-access - g._appmap_request_event.id, # pylint: disable=protected-access + g.appmap_response.status_code, + g.appmap_response.headers, + return_event, ) - return response - def got_request_exception(self, _, exception): - self.on_exception( - g._appmap_recorder, # pylint: disable=protected-access - g._appmap_request_start, # pylint: disable=protected-access - g._appmap_request_event.id, # pylint: disable=protected-access - (type(exception), exception, None), - ) + def request_finished(self, _, response, **__): + if not self.should_record: + return response + g.appmap_response = response + return response @patch_class(jinja2.Template) @@ -187,10 +195,31 @@ def install_extension(wrapped, _, args, kwargs): return app +def _finalize_request(wrapped, inst, args, kwargs): + if not Env.current.enabled or kwargs.get("from_error_handler"): + return wrapped(*args, **kwargs) + + ret = wrapped(*args, **kwargs) + _after_finalize.send(inst) + return ret + + +def _handle_user_exception(wrapped, inst, args, kwargs): + if not Env.current.enabled: + return wrapped(*args, **kwargs) + + try: + return wrapped(*args, **kwargs) + except Exception: # pylint: disable=broad-exception-caught + g.appmap_response = SimpleNamespace(status_code=500, headers={}) + _after_finalize.send(inst) + raise + if Env.current.enabled: # ScriptInfo.load_app is the function that's used by the Flask cli to load an app, no matter how # the app's module is specified (e.g. with the FLASK_APP env var, the `--app` flag, etc). Hook # it so it installs our extension on the app. - load_app = wrapt.wrap_function_wrapper("flask.cli", "ScriptInfo.load_app", install_extension) - ScriptInfo.load_app = load_app # type: ignore[method-assign] + wrapt.wrap_function_wrapper("flask.cli", "ScriptInfo.load_app", install_extension) + wrapt.wrap_function_wrapper("flask.app", "Flask.finalize_request", _finalize_request) + wrapt.wrap_function_wrapper("flask.app", "Flask.handle_user_exception", _handle_user_exception) From 23b52b507eedfb42a15e93a5e7ce1a5fc1d3edad Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 15 Jul 2024 13:12:23 -0400 Subject: [PATCH 060/113] fix: only instrument property functions once Make sure that the fget, fset, and fdel functions associated with an instrumented property only get wrapped once. --- _appmap/importer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/_appmap/importer.py b/_appmap/importer.py index 6cff6cdd..aca46a45 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -205,15 +205,17 @@ def instrument_functions(filterable, selected_functions=None): for prop_name, (prop, prop_fns) in properties.items(): instrumented_fns = {} for k, (fn, auxtype) in prop_fns.items(): - if fn is None: + if fn is None or getattr(fn, "_appmap_wrapped", None): continue filterableFn = FilterableFn(filterable, fn, fn, auxtype) new_fn = cls.instrument_function(fn.__name__, filterableFn, selected_functions) if new_fn != fn: new_fn = wrapt.FunctionWrapper(fn, new_fn) + setattr(new_fn, "_appmap_wrapped", True) instrumented_fns[k] = new_fn - instrumented_fns["doc"] = prop.__doc__ - setattr(filterable.obj, prop_name, property(**instrumented_fns)) + if len(instrumented_fns) > 0: + instrumented_fns["doc"] = prop.__doc__ + setattr(filterable.obj, prop_name, property(**instrumented_fns)) # Import Config here, to avoid circular top-level imports. from .configuration import Config # pylint: disable=import-outside-toplevel From b74513d14497332bd39d04561e45084286c92eec Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 15 Jul 2024 13:32:24 -0400 Subject: [PATCH 061/113] refactor: clean up the rest of the lint warnings Eliminate the rest of the current (i.e. not suppressed) pylint warnings. --- _appmap/test/test_django_simplelazyobject.py | 3 ++- _appmap/test/test_events.py | 8 ++++---- _appmap/test/test_fastapi.py | 6 +++++- _appmap/test/test_flask.py | 6 +++--- _appmap/test/test_params.py | 2 +- _appmap/test/test_recording.py | 2 +- pylintrc | 2 +- tox.ini | 1 + 8 files changed, 18 insertions(+), 12 deletions(-) diff --git a/_appmap/test/test_django_simplelazyobject.py b/_appmap/test/test_django_simplelazyobject.py index c5e31abe..f7c1776e 100644 --- a/_appmap/test/test_django_simplelazyobject.py +++ b/_appmap/test/test_django_simplelazyobject.py @@ -16,7 +16,8 @@ def test_recording_simplelazyobject_does_not_evaluate(): doesn't cause incorrect premature evaluation. """ with appmap.Recording(): - import appmap_testing.django_simplelazyobject as ecds # pylint: disable=import-outside-toplevel + import appmap_testing.django_simplelazyobject as ecds # pylint: disable=import-outside-toplevel, import-error + ecds.lazy() # if we're here and the exception wasn't thrown, we're good diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index 4dba7cd8..b5d5eac2 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -45,7 +45,7 @@ class TestEvents: def test_recursion_protection(self): r = appmap.Recording() with r: - from example_class import ExampleClass + from example_class import ExampleClass # pylint: disable=import-outside-toplevel ExampleClass().instance_method() @@ -56,7 +56,7 @@ def test_recursion_protection(self): def test_when_str_raises(self, mocker): r = appmap.Recording() with r: - from example_class import ExampleClass + from example_class import ExampleClass # pylint: disable=import-outside-toplevel param = mocker.Mock() param.__str__ = mocker.Mock(side_effect=Exception) @@ -72,7 +72,7 @@ def test_when_str_raises(self, mocker): def test_when_both_raise(self, mocker): r = appmap.Recording() with r: - from example_class import ExampleClass + from example_class import ExampleClass # pylint: disable=import-outside-toplevel param = mocker.Mock() param.__str__ = mocker.Mock(side_effect=Exception) @@ -88,7 +88,7 @@ def test_when_display_disabled(self, mocker): Env.current.set("APPMAP_DISPLAY_PARAMS", "false") r = appmap.Recording() with r: - from example_class import ExampleClass + from example_class import ExampleClass # pylint: disable=import-outside-toplevel param = mocker.MagicMock() diff --git a/_appmap/test/test_fastapi.py b/_appmap/test/test_fastapi.py index d6ab0b9f..7122508b 100644 --- a/_appmap/test/test_fastapi.py +++ b/_appmap/test/test_fastapi.py @@ -24,6 +24,10 @@ class TestRecordRequests(_TestRecordRequests): @pytest.mark.app(remote_enabled=True) class TestRemoteRecording(_TestRemoteRecording): + def __init__(self): + self.expected_thread_id = None + self.expected_content_type = None + def setup_method(self): self.expected_thread_id = 1 self.expected_content_type = "application/json" @@ -39,7 +43,7 @@ def fastapi_app(data_dir, monkeypatch, request): Env.current.set("APPMAP_CONFIG", data_dir / "fastapi" / "appmap.yml") - from fastapiapp import main # pyright: ignore[reportMissingImports] + from fastapiapp import main # pyright: ignore[reportMissingImports] pylint: disable=import-error,import-outside-toplevel importlib.reload(main) diff --git a/_appmap/test/test_flask.py b/_appmap/test/test_flask.py index 8512d5f1..bed1f303 100644 --- a/_appmap/test/test_flask.py +++ b/_appmap/test/test_flask.py @@ -14,7 +14,7 @@ from _appmap.metadata import Metadata from appmap.flask import AppmapFlask -from ..test.helpers import DictIncluding, check_call_stack, package_version +from ..test.helpers import DictIncluding, check_call_stack from .web_framework import ( _TestFormCapture, _TestFormData, @@ -212,8 +212,8 @@ def test_disabled_for_process(self, pytester, monkeypatch): assert not (pytester.path / "tmp" / "appmap" / "pytest").exists() def verify_events(events): - def find(type): - return next(filter(lambda e: e[1].get(type) is not None, enumerate(events)), None) + def find(event_type): + return next(filter(lambda e: e[1].get(event_type) is not None, enumerate(events)), None) request = find("http_server_request") assert request is not None diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index 7026755b..d49d9a4d 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -58,7 +58,7 @@ def params(self, request): of this fixture, unload it after. This ensures that each test sees a pristine version of the classes it contains. """ - from params import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error + from params import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error,import-outside-toplevel C, ) diff --git a/_appmap/test/test_recording.py b/_appmap/test/test_recording.py index 6fcc763e..d4656a4d 100644 --- a/_appmap/test/test_recording.py +++ b/_appmap/test/test_recording.py @@ -6,9 +6,9 @@ from shutil import copy, copytree from threading import Thread -import appmap import pytest +import appmap from _appmap.event import Event from _appmap.recorder import Recorder, ThreadRecorder from _appmap.wrapt import FunctionWrapper diff --git a/pylintrc b/pylintrc index 9bf0f510..2cf9f132 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,6 @@ [MAIN] # Specify a score threshold under which the program will exit with error. -fail-under=9.94 +fail-under=9.99 # Analyse import fallback blocks. This can be used to support both Python 2 and diff --git a/tox.ini b/tox.ini index e22265fd..d4e47a9f 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ skip_install = True deps = poetry {[web-deps]deps} + numpy >=2 commands = poetry install # It doesn't seem great to disable cyclic-import checking, but the imports From c24c89712003bb85c3ab50cfc8bad483e195536b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 15 Jul 2024 19:53:12 +0000 Subject: [PATCH 062/113] chore(release): 2.1.1 [skip ci] ## [2.1.1](https://github.com/getappmap/appmap-python/compare/v2.1.0...v2.1.1) (2024-07-15) ### Bug Fixes * Flask events are ordered correctly ([f970fb7](https://github.com/getappmap/appmap-python/commit/f970fb7828edc3e5d19aa03e48e9302077d58614)) * only instrument property functions once ([23b52b5](https://github.com/getappmap/appmap-python/commit/23b52b507eedfb42a15e93a5e7ce1a5fc1d3edad)) --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd69e28..a2388e86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [2.1.1](https://github.com/getappmap/appmap-python/compare/v2.1.0...v2.1.1) (2024-07-15) + + +### Bug Fixes + +* Flask events are ordered correctly ([f970fb7](https://github.com/getappmap/appmap-python/commit/f970fb7828edc3e5d19aa03e48e9302077d58614)) +* only instrument property functions once ([23b52b5](https://github.com/getappmap/appmap-python/commit/23b52b507eedfb42a15e93a5e7ce1a5fc1d3edad)) + # [2.1.0](https://github.com/getappmap/appmap-python/compare/v2.0.10...v2.1.0) (2024-07-03) diff --git a/pyproject.toml b/pyproject.toml index 4ea6aec5..f0304d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.0" +version = "2.1.1" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From c927f9cbbd809f683e8028505ef38bc3311fcf36 Mon Sep 17 00:00:00 2001 From: zermelo-wisen Date: Thu, 4 Jul 2024 12:27:33 +0300 Subject: [PATCH 063/113] fix: catch BaseException from instrumented code Catch all exceptions thrown from instrumented code by catching BaseException, rather than Exception. This ensures that the appropriate event gets added to the recording. This problem was originally found when running instrumented tests in pytest-dev/pytest, so the changes include a test copied from there. pytest's OutcomeException (and its subclasses, like Skipped) inherits from BaseException. pytest raises an OutcomeException internally to indicate how a test case finished. --- _appmap/instrument.py | 7 +++- _appmap/test/data/example_class.py | 3 ++ .../test/data/pytest-instrumented/appmap.yml | 12 +++++++ .../pytest-instrumented/init/sitecustomize.py | 1 + .../pytest-instrumented/test_instrumented.py | 16 +++++++++ _appmap/test/helpers.py | 18 ++++++++-- _appmap/test/test_events.py | 33 +++++++++++++++++++ _appmap/test/test_test_frameworks.py | 15 +++++++-- 8 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 _appmap/test/data/pytest-instrumented/appmap.yml create mode 100644 _appmap/test/data/pytest-instrumented/init/sitecustomize.py create mode 100644 _appmap/test/data/pytest-instrumented/test_instrumented.py diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 191e5ef9..0fb2a9e7 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -101,7 +101,12 @@ def call_instrumented(f, instance, args, kwargs): return ret except AppMapLimitExceeded: raise - except Exception: # noqa: E722 + # Some applications make use of exceptions that aren't descended from Exception. For example, + # pytest's OutcomeException, used to indicate the outcome of a test case, is a child of + # BaseException. + # + # We need to catch *any* exception raised, to ensure that we add the appropriate ExceptionEvent. + except BaseException: # noqa: E722 elapsed_time = time.time() - start_time Recorder.add_event( event.ExceptionEvent( diff --git a/_appmap/test/data/example_class.py b/_appmap/test/data/example_class.py index 46c124f5..a5bc9992 100644 --- a/_appmap/test/data/example_class.py +++ b/_appmap/test/data/example_class.py @@ -5,6 +5,7 @@ import time from functools import lru_cache, wraps +from typing import NoReturn import appmap @@ -155,6 +156,8 @@ def del_write_only(self): write_only = property(None, set_write_only, del_write_only, "Write-only") + def raise_base_exception(self) -> NoReturn: + raise BaseException("not derived from Exception") # pylint: disable=broad-exception-raised def modfunc(): return "Hello world!" diff --git a/_appmap/test/data/pytest-instrumented/appmap.yml b/_appmap/test/data/pytest-instrumented/appmap.yml new file mode 100644 index 00000000..8f2bb605 --- /dev/null +++ b/_appmap/test/data/pytest-instrumented/appmap.yml @@ -0,0 +1,12 @@ +name: Simple +packages: +- path: simple +- path: _pytest + exclude: + # - _py.path + - compat.safe_getattr + - config.PytestPluginManager + - fixtures.getfixturemarker + - config.argparsing + - config.Config.rootpath +- path: pytest \ No newline at end of file diff --git a/_appmap/test/data/pytest-instrumented/init/sitecustomize.py b/_appmap/test/data/pytest-instrumented/init/sitecustomize.py new file mode 100644 index 00000000..d1fe4fec --- /dev/null +++ b/_appmap/test/data/pytest-instrumented/init/sitecustomize.py @@ -0,0 +1 @@ +import appmap diff --git a/_appmap/test/data/pytest-instrumented/test_instrumented.py b/_appmap/test/data/pytest-instrumented/test_instrumented.py new file mode 100644 index 00000000..52af6a1a --- /dev/null +++ b/_appmap/test/data/pytest-instrumented/test_instrumented.py @@ -0,0 +1,16 @@ +import pytest + +# Copied from pytest-dev/pytest. When recorded, this test case will raise an OutcomeException +# (specifically _pytest.outcomes.Skipped). +def test_skipped(pytester): + pytester.makeconftest( + """ + import pytest + def pytest_ignore_collect(): + pytest.skip("intentional") + """ + ) + pytester.makepyfile("def test_hello(): pass") + result = pytester.runpytest_inprocess() + assert result.ret == pytest.ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines(["*1 skipped*"]) diff --git a/_appmap/test/helpers.py b/_appmap/test/helpers.py index 3e183ef3..ac07c428 100644 --- a/_appmap/test/helpers.py +++ b/_appmap/test/helpers.py @@ -44,7 +44,19 @@ def check_call_stack(events): if e.get("event") == "call": stack.append(e) elif e.get("event") == "return": - assert len(stack) > 0, "return without call" + assert len(stack) > 0, f"return without call, {e.get('id')}" call = stack.pop() - assert call.get("id") == e.get("parent_id") - assert len(stack) == 0, "leftover events" + assert call.get("id") == e.get( + "parent_id" + ), f"parent mismatch, {call.get('id')} != {e.get('parent_id')}" + assert len(stack) == 0, f"leftover events, {len(stack)}" + + +if __name__ == "__main__": + import json + from pathlib import Path + import sys + + with Path(sys.argv[1]).open(encoding="utf-8") as f: + appmap = json.load(f) + check_call_stack(appmap["events"]) diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index b5d5eac2..f57e0218 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -117,3 +117,36 @@ def test_describe_return_value_recursion_protection(self): assert [e.method_id for e in r.events if e.event == "call" and hasattr(e, "method_id")] == [ "return_self" ] + + # There should be an exception return event generated even when the raised exception is a + # BaseException. + def test_exception_event_with_base_exception(self): + r = appmap.Recording() + with r: + # pylint: disable=import-outside-toplevel + from example_class import ExampleClass + + try: + ExampleClass().raise_base_exception() + except BaseException: # pylint: disable=broad-exception-caught + pass + assert check_call_return_stack_order(r.events), "Unbalanced call stack" + + +def check_call_return_stack_order(events): + stack = [] + for e in events: + if e.event == "call": + stack.append(e) + elif e.event == "return": + if len(stack) > 0: + call = stack.pop() + if call.id != e.parent_id: + return False + else: + return False + if len(stack) == 0: + return True + + return False + diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 1fc73ea7..b847c1e3 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -10,10 +10,10 @@ from pathlib import Path import pytest + from _appmap import recording -from _appmap.test.helpers import package_version -from ..test.helpers import DictIncluding +from .helpers import DictIncluding, check_call_stack, package_version from .normalize import normalize_appmap @@ -155,6 +155,17 @@ def test_write_appmap(recorder_outdir): expected_shortname = longname[:235] + "-5d6e10d.appmap.json" assert (recorder_outdir / expected_shortname).read_text().startswith('{"version"') +@pytest.mark.example_dir("pytest-instrumented") +@pytest.mark.appmap_enabled +def test_pytest_instrumented(testdir): + result = testdir.runpytest("-svv", "-p", "pytester", "test_instrumented.py") + result.assert_outcomes(passed=1) + appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_skipped.appmap.json" + appmap = json.load(appmap_file.open()) + events = appmap["events"] + assert len(events) > 0 + check_call_stack(events) + def verify_expected_appmap(testdir, suffix=""): appmap_json = list(testdir.output().glob("*test_hello_world.appmap.json")) From c30529523be66cea2f9d0707200d71ae0277564d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 16 Jul 2024 08:36:53 +0000 Subject: [PATCH 064/113] chore(release): 2.1.2 [skip ci] ## [2.1.2](https://github.com/getappmap/appmap-python/compare/v2.1.1...v2.1.2) (2024-07-16) ### Bug Fixes * catch BaseException from instrumented code ([c927f9c](https://github.com/getappmap/appmap-python/commit/c927f9cbbd809f683e8028505ef38bc3311fcf36)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2388e86..72e9dc93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.1.2](https://github.com/getappmap/appmap-python/compare/v2.1.1...v2.1.2) (2024-07-16) + + +### Bug Fixes + +* catch BaseException from instrumented code ([c927f9c](https://github.com/getappmap/appmap-python/commit/c927f9cbbd809f683e8028505ef38bc3311fcf36)) + ## [2.1.1](https://github.com/getappmap/appmap-python/compare/v2.1.0...v2.1.1) (2024-07-15) diff --git a/pyproject.toml b/pyproject.toml index f0304d4e..d2c18357 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.1" +version = "2.1.2" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From feec761fefd5596c4fd7bde0cd9c3901e02791b3 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 15 Jul 2024 15:45:13 -0400 Subject: [PATCH 065/113] fix: show config packages on startup In addition to the path of config file, show the packages property, too. --- _appmap/configuration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index e83fd0ca..af954dea 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -5,6 +5,7 @@ import ast import importlib.metadata import inspect +import json import os import sys from os.path import realpath @@ -144,6 +145,9 @@ def __init__(self): if "labels" in self._config: self.labels.append(self._config["labels"]) + def __repr__(self): + return json.dumps(self._config["packages"]) + @property def name(self): return self._config["name"] @@ -483,6 +487,7 @@ def initialize(): # pylint: disable=protected-access c._load_config(show_warnings=True) logger.info("file: %s", c._file if c.file_present else "[no appmap.yml]") + logger.info("config: %r", c) logger.debug("package_functions: %s", c.package_functions) logger.info("env: %r", os.environ) os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" From 11b6307cf2bdbfae50f30d4f329e6ba3ac6f4035 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 15 Jul 2024 15:47:29 -0400 Subject: [PATCH 066/113] fix: add APPMAP_INSTRUMENT_PROPERTIES Add APPMAP_INSTRUMENT_PROPERTIES to control whether properties should be instrumented. --- _appmap/importer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_appmap/importer.py b/_appmap/importer.py index aca46a45..e077dda5 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -132,7 +132,7 @@ def is_member_func(m): if key.startswith("__"): continue static_value = inspect.getattr_static(cls, key) - if isinstance(static_value, property): + if Importer.instrument_properties and isinstance(static_value, property): properties[key] = ( static_value, { @@ -164,6 +164,9 @@ def initialize(cls): cls.filter_stack = [] cls.filter_chain = [] cls._skip_instrumenting = ("appmap", "_appmap") + cls.instrument_properties = ( + Env.current.get("APPMAP_INSTRUMENT_PROPERTIES", "true").lower() == "true" + ) @classmethod def use_filter(cls, filter_class): From 5cce0f0644eebf7d19bad5cda61726393cd7ba68 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 19 Jul 2024 05:05:48 -0400 Subject: [PATCH 067/113] fix: improve property handling Base the decision to instrument a property on the property name, rather than trying to use the name of the f{get,set,del} functions. This aligns them with the rest of the class's members, e.g. for exclusion. Also, ensure that those functions only get instrumented once. --- _appmap/configuration.py | 4 ++-- _appmap/importer.py | 27 ++++++++++++++++++--------- _appmap/instrument.py | 2 +- _appmap/test/test_params.py | 2 +- appmap/fastapi.py | 2 +- vendor/_appmap/wrapt/wrappers.py | 2 ++ 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index af954dea..870c2aec 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -423,8 +423,8 @@ def wrap(self, filterable): # appropriate. # rule = self.match(filterable) - wrapped = getattr(filterable.obj, "_appmap_wrapped", None) - if wrapped is None: + wrapped = getattr(filterable.obj, "_appmap_instrumented", None) + if not wrapped: logger.trace(" wrapping %s", filterable.fqname) Config.current.labels.apply(filterable) ret = instrument(filterable) diff --git a/_appmap/importer.py b/_appmap/importer.py index e077dda5..b188539b 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -42,8 +42,8 @@ class FilterableFn( ): __slots__ = () - def __new__(cls, scope, fn, static_fn, auxtype=None): - fqname = "%s.%s" % (scope.fqname, fn.__name__) + def __new__(cls, scope, fn_name, fn, static_fn, auxtype=None): # pylint: disable=too-many-arguments + fqname = "%s.%s" % (scope.fqname, fn_name) self = super(FilterableFn, cls).__new__(cls, scope.scope, fqname, fn, static_fn, auxtype) return self @@ -132,7 +132,9 @@ def is_member_func(m): if key.startswith("__"): continue static_value = inspect.getattr_static(cls, key) - if Importer.instrument_properties and isinstance(static_value, property): + # Don't use isinstance to check the type of static_value -- we don't want to invoke the + # descriptor protocol. + if Importer.instrument_properties and type(static_value) is property: # pylint: disable=unidiomatic-typecheck properties[key] = ( static_value, { @@ -194,27 +196,34 @@ def do_import(cls, *args, **kwargs): cls.filter_chain = reduce(lambda acc, e: e(acc), cls.filter_stack, NullFilter(None)) def instrument_functions(filterable, selected_functions=None): + # pylint: disable=too-many-locals logger.trace(" looking for members of %s", filterable.obj) functions, properties = get_members(filterable.obj) logger.trace(" functions %s", functions) for fn_name, static_fn, fn in functions: - filterableFn = FilterableFn(filterable, fn, static_fn) + filterableFn = FilterableFn(filterable, fn.__name__, fn, static_fn) new_fn = cls.instrument_function(fn_name, filterableFn, selected_functions) if new_fn != fn: - wrapt.wrap_function_wrapper(filterable.obj, fn_name, new_fn) + fw = wrapt.wrap_function_wrapper(filterable.obj, fn_name, new_fn) + fw._appmap_instrumented = True # pylint: disable=protected-access + # Now that we've instrumented all the functions, go through the properties and update # them for prop_name, (prop, prop_fns) in properties.items(): instrumented_fns = {} for k, (fn, auxtype) in prop_fns.items(): - if fn is None or getattr(fn, "_appmap_wrapped", None): + if fn is None: + continue + filterableFn = FilterableFn(filterable, prop_name, fn, fn, auxtype) + if getattr(fn, "_appmap_instrumented", None): continue - filterableFn = FilterableFn(filterable, fn, fn, auxtype) - new_fn = cls.instrument_function(fn.__name__, filterableFn, selected_functions) + new_fn = cls.instrument_function(prop_name, filterableFn, selected_functions) if new_fn != fn: new_fn = wrapt.FunctionWrapper(fn, new_fn) - setattr(new_fn, "_appmap_wrapped", True) + # Set _appmap_instrumented on the FunctionWrapper, not on the wrapped function + new_fn._appmap_instrumented = True # pylint: disable=protected-access + instrumented_fns[k] = new_fn if len(instrumented_fns) > 0: instrumented_fns["doc"] = prop.__doc__ diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 0fb2a9e7..79085372 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -138,5 +138,5 @@ def instrumented_fn(wrapped, instance, args, kwargs): return call_instrumented(f, instance, args, kwargs) ret = instrumented_fn - setattr(ret, "_appmap_wrapped", True) + setattr(ret, "_appmap_instrumented", True) return ret diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index d49d9a4d..3c079d55 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -44,7 +44,7 @@ def wrap_test_func(self, fnname): static_fn = inspect.getattr_static(C, fnname) fn = getattr(C, fnname) fc = FilterableCls(C) - ffn = FilterableFn(fc, fn, static_fn) + ffn = FilterableFn(fc, fn.__name__, fn, static_fn) wrapped = self.prepare(ffn) wrapt.wrap_function_wrapper(C, fnname, wrapped) diff --git a/appmap/fastapi.py b/appmap/fastapi.py index bfedba6a..da8acc44 100644 --- a/appmap/fastapi.py +++ b/appmap/fastapi.py @@ -34,7 +34,7 @@ def _add_api_route(wrapped, _, args, kwargs): fqn = utils.FqFnName(fn) scope = Filterable(fqn.scope, fqn.fqclass, None) - filterable_fn = FilterableFn(scope, fn, fn) + filterable_fn = FilterableFn(scope, fn.__name__, fn, fn) logger.debug("_add_api_route, fn: %s", filterable_fn.fqname) instrumented_fn = Importer.instrument_function(fqn.fn_name, filterable_fn) diff --git a/vendor/_appmap/wrapt/wrappers.py b/vendor/_appmap/wrapt/wrappers.py index f269bbcb..bbe9b0e5 100644 --- a/vendor/_appmap/wrapt/wrappers.py +++ b/vendor/_appmap/wrapt/wrappers.py @@ -516,6 +516,7 @@ class _FunctionWrapperBase(ObjectProxy): def __init__(self, wrapped, instance, wrapper, enabled=None, binding='function', parent=None): + "_appmap_instrumented", super(_FunctionWrapperBase, self).__init__(wrapped) object.__setattr__(self, '_self_instance', instance) @@ -524,6 +525,7 @@ def __init__(self, wrapped, instance, wrapper, enabled=None, object.__setattr__(self, '_self_binding', binding) object.__setattr__(self, '_self_parent', parent) object.__setattr__(self, '_bfws', list()) + object.__setattr__(self, "_appmap_instrumented", False) def __get__(self, instance, owner): # This method is actually doing double duty for both unbound and From 1847b0e7177327adc080854fbbec17b89166d516 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 19 Jul 2024 05:29:36 -0400 Subject: [PATCH 068/113] fix: try to avoid recording tests When creating the default config, ignore directories with names that match the regex .*test.*. This should avoid instrumenting the majority of test functions. For those that do still get instrumented, have the test framework integration disable the wrapt wrapped on it, thereby disabling recording. --- _appmap/configuration.py | 47 +++- .../test/data/pytest/appmap-no-test-cases.yml | 4 + _appmap/test/data/pytest/appmap.yml | 2 + .../pytest-numpy1-no-test-cases.appmap.json | 242 ++++++++++++++++++ .../pytest/expected/pytest-numpy1.appmap.json | 129 ++++++---- .../pytest-numpy2-no-test-cases.appmap.json | 242 ++++++++++++++++++ .../pytest/expected/pytest-numpy2.appmap.json | 129 ++++++---- .../expected/status_errored.metadata.json | 2 +- .../expected/status_failed.metadata.json | 2 +- .../expected/status_xfailed.metadata.json | 2 +- _appmap/test/data/pytest/tests/__init__.py | 0 .../data/pytest/{ => tests}/test_noappmap.py | 0 .../data/pytest/{ => tests}/test_simple.py | 0 .../test/data/trial/appmap-no-test-cases.yml | 4 + _appmap/test/data/trial/appmap.yml | 1 + .../expected/pytest-no-test-cases.appmap.json | 28 ++ .../data/unittest/appmap-no-test-cases.yml | 4 + _appmap/test/data/unittest/appmap.yml | 1 + .../unittest-no-test-cases.appmap.json | 148 +++++++++++ _appmap/test/test_configuration.py | 7 +- _appmap/test/test_events.py | 1 - _appmap/test/test_fastapi.py | 6 +- _appmap/test/test_test_frameworks.py | 25 +- _appmap/testing_framework.py | 25 +- _appmap/unittest.py | 101 ++------ appmap/pytest.py | 2 + vendor/_appmap/wrapt/wrappers.py | 29 ++- 27 files changed, 972 insertions(+), 211 deletions(-) create mode 100644 _appmap/test/data/pytest/appmap-no-test-cases.yml create mode 100644 _appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json create mode 100644 _appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json create mode 100644 _appmap/test/data/pytest/tests/__init__.py rename _appmap/test/data/pytest/{ => tests}/test_noappmap.py (100%) rename _appmap/test/data/pytest/{ => tests}/test_simple.py (100%) create mode 100644 _appmap/test/data/trial/appmap-no-test-cases.yml create mode 100644 _appmap/test/data/trial/expected/pytest-no-test-cases.appmap.json create mode 100644 _appmap/test/data/unittest/appmap-no-test-cases.yml create mode 100644 _appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json diff --git a/_appmap/configuration.py b/_appmap/configuration.py index 870c2aec..ed1965e3 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -7,12 +7,14 @@ import inspect import json import os +import re import sys from os.path import realpath from pathlib import Path from textwrap import dedent import yaml +from yaml import SafeLoader from yaml.parser import ParserError from _appmap.labels import LabelSet @@ -50,20 +52,19 @@ def _get_sys_prefix(): return realpath(sys.prefix) +_EXCLUDE_PATTERN = re.compile(r"\..*|node_modules|.*test.*|site-packages") + + def find_top_packages(rootdir): """ - Scan a directory tree for packages that should appear in the - default config file. + Scan a directory tree for packages that should appear in the default config file. - Examine directories in rootdir, to see if they contains an - __init__.py. If it does, add it to the list of packages and don't - scan any of its subdirectories. If it doesn't, scan its + Examine each directory in rootdir, to see if it contains an __init__.py. If it does, add it to + the list of packages and don't scan any of its subdirectories. If it doesn't, scan its subdirectories to find __init__.py. - Some directories are automatically excluded from the search: - * sys.prefix - * Hidden directories (i.e. those that start with a '.') - * node_modules + Directory traversal will stop at directories that match _EXCLUDE_PATTERN. Such a directory (and + its subdirectories) will not be added to the returned packages. For example, in a directory like this @@ -71,7 +72,7 @@ def find_top_packages(rootdir): LICENSE Makefile appveyor.yml docs/ src/ tests/ MANIFEST.in README.rst blog/ setup.py tddium.yml tox.ini - docs, src, tests, and blog will get scanned. + docs, src, blog will get scanned. tests will be ignored. Only src has a subdirectory containing an __init__.py: @@ -105,7 +106,7 @@ def find_top_packages(rootdir): packages = set() def excluded(d): - excluded = d == "node_modules" or d[0] == "." + excluded = _EXCLUDE_PATTERN.search(d) is not None if excluded: logger.trace("excluding dir %s", d) return excluded @@ -130,6 +131,19 @@ def excluded(d): class AppMapInvalidConfigException(Exception): pass +# We don't have any control over the PyYAML class hierarchy, so we can't control how many ancestors +# SafeLoader has.... +class _ConfigLoader(SafeLoader): # pylint: disable=too-many-ancestors + def construct_mapping(self, node, deep=False): + mapping = super().construct_mapping(node, deep=deep) + # Allow record_test_cases to be set using a string (in addition to allowing a boolean). + if "record_test_cases" in mapping: + val = mapping["record_test_cases"] + if isinstance(val, str): + mapping["record_test_cases"] = val.lower() == "true" + return mapping + + class Config(metaclass=SingletonMeta): """Singleton Config class""" @@ -156,11 +170,16 @@ def name(self): def packages(self): return self._config["packages"] + @property + def record_test_cases(self): + return self._config.get("record_test_cases", False) + @property def default(self): ret = { "name": self.default_name, "language": "python", + "record_test_cases": False, "packages": self.default_packages, } env = Env.current @@ -233,7 +252,7 @@ def _load_config(self, show_warnings=False): Env.current.enabled = False self.file_valid = False try: - self._config = yaml.safe_load(path.read_text(encoding="utf-8")) + self._config = yaml.load(path.read_text(encoding="utf-8"), Loader=_ConfigLoader) if not self._config: # It parsed, but was (effectively) empty. self._config = self.default @@ -334,7 +353,6 @@ def _check_path_value(self, value): except SyntaxError: return False - def startswith(prefix, sequence): """ Check if a sequence starts with the prefix. @@ -377,7 +395,8 @@ class DistMatcher(PathMatcher): def __init__(self, dist, *args, **kwargs): super().__init__(*args, **kwargs) self.dist = dist - self.files = [str(pp.locate()) for pp in importlib.metadata.files(dist)] + dist_files = importlib.metadata.files(dist) + self.files = [str(pp.locate()) for pp in dist_files] if dist_files is not None else [] def matches(self, filterable): try: diff --git a/_appmap/test/data/pytest/appmap-no-test-cases.yml b/_appmap/test/data/pytest/appmap-no-test-cases.yml new file mode 100644 index 00000000..4e0eb415 --- /dev/null +++ b/_appmap/test/data/pytest/appmap-no-test-cases.yml @@ -0,0 +1,4 @@ +name: Simple +record_test_cases: false +packages: +- path: simple diff --git a/_appmap/test/data/pytest/appmap.yml b/_appmap/test/data/pytest/appmap.yml index 2d20878f..4eaae12e 100644 --- a/_appmap/test/data/pytest/appmap.yml +++ b/_appmap/test/data/pytest/appmap.yml @@ -1,3 +1,5 @@ name: Simple +record_test_cases: true packages: - path: simple +- path: tests \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json new file mode 100644 index 00000000..bc711efa --- /dev/null +++ b/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json @@ -0,0 +1,242 @@ +{ + "version": "1.9", + "metadata": { + "language": { + "name": "python" + }, + "client": { + "name": "appmap", + "url": "https://github.com/applandinc/appmap-python" + }, + "app": "Simple", + "recorder": { + "name": "pytest", + "type": "tests" + }, + "source_location": "tests/test_simple.py:5", + "name": "hello world", + "feature": "Hello world", + "test_status": "succeeded" + }, + "events": [ + { + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple.py", + "lineno": 8, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 1, + "event": "call", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 2, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello'" + }, + "parent_id": 2, + "id": 3, + "event": "return", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 4, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'world!'" + }, + "parent_id": 4, + "id": 5, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello world!'" + }, + "parent_id": 1, + "id": 6, + "event": "return", + "thread_id": 1 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 7, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "show_numpy_dict", + "path": "simple.py", + "lineno": 11 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [ + { + "kind": "req", + "value": "{0: 'zero', 1: 'one'}", + "name": "d", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + } + ], + "id": 8, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "get_numpy_dict", + "path": "simple.py", + "lineno": 18 + }, + { + "return_value": { + "value": "{0: 'zero', 1: 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 8, + "id": 9, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "{0: 'zero', 1: 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 7, + "id": 10, + "event": "return", + "thread_id": 1 + } + ], + "classMap": [ + { + "name": "simple", + "type": "package", + "children": [ + { + "name": "Simple", + "type": "class", + "children": [ + { + "name": "get_numpy_dict", + "type": "function", + "location": "simple.py:18", + "static": false + }, + { + "name": "hello", + "type": "function", + "location": "simple.py:2", + "static": false + }, + { + "name": "hello_world", + "type": "function", + "location": "simple.py:8", + "static": false + }, + { + "name": "show_numpy_dict", + "type": "function", + "location": "simple.py:11", + "static": false + }, + { + "name": "world", + "type": "function", + "location": "simple.py:5", + "static": false + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json index cd3ef53c..6a90f1bc 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json @@ -8,95 +8,106 @@ "name": "appmap", "url": "https://github.com/applandinc/appmap-python" }, + "source_location": "tests/test_simple.py:5", + "name": "hello world", + "feature": "Hello world", "app": "Simple", "recorder": { "name": "pytest", "type": "tests" }, - "source_location": "test_simple.py:5", - "name": "hello world", - "feature": "Hello world", "test_status": "succeeded" }, "events": [ { - "defined_class": "simple.Simple", - "method_id": "hello_world", - "path": "simple.py", - "lineno": 8, + "static": true, + "parameters": [], + "id": 1, + "event": "call", + "thread_id": 1, + "defined_class": "tests.test_simple", + "method_id": "test_hello_world", + "path": "tests/test_simple.py", + "lineno": 6 + }, + { "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 1, + "id": 2, "event": "call", - "thread_id": 1 - }, - { + "thread_id": 1, "defined_class": "simple.Simple", - "method_id": "hello", + "method_id": "hello_world", "path": "simple.py", - "lineno": 2, + "lineno": 8 + }, + { "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 2, + "id": 3, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello'" + "value": "'Hello'", + "class": "builtins.str" }, - "parent_id": 2, - "id": 3, + "parent_id": 3, + "id": 4, "event": "return", "thread_id": 1 }, { - "defined_class": "simple.Simple", - "method_id": "world", - "path": "simple.py", - "lineno": 5, "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 4, + "id": 5, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5 }, { "return_value": { - "class": "builtins.str", - "value": "'world!'" + "value": "'world!'", + "class": "builtins.str" }, - "parent_id": 4, - "id": 5, + "parent_id": 5, + "id": 6, "event": "return", "thread_id": 1 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello world!'" + "value": "'Hello world!'", + "class": "builtins.str" }, - "parent_id": 1, - "id": 6, + "parent_id": 2, + "id": 7, "event": "return", "thread_id": 1 }, @@ -109,7 +120,7 @@ "class": "simple.Simple" }, "parameters": [], - "id": 7, + "id": 8, "event": "call", "thread_id": 1, "defined_class": "simple.Simple", @@ -144,7 +155,7 @@ "size": 2 } ], - "id": 8, + "id": 9, "event": "call", "thread_id": 1, "defined_class": "simple.Simple", @@ -168,8 +179,8 @@ ], "size": 2 }, - "parent_id": 8, - "id": 9, + "parent_id": 9, + "id": 10, "event": "return", "thread_id": 1 }, @@ -189,8 +200,18 @@ ], "size": 2 }, - "parent_id": 7, - "id": 10, + "parent_id": 8, + "id": 11, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "None", + "class": "builtins.NoneType" + }, + "parent_id": 1, + "id": 12, "event": "return", "thread_id": 1 } @@ -237,6 +258,24 @@ ] } ] + }, + { + "name": "tests", + "type": "package", + "children": [ + { + "name": "test_simple", + "type": "class", + "children": [ + { + "name": "test_hello_world", + "type": "function", + "location": "tests/test_simple.py:6", + "static": true + } + ] + } + ] } ] } \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json new file mode 100644 index 00000000..b6d96002 --- /dev/null +++ b/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json @@ -0,0 +1,242 @@ +{ + "version": "1.9", + "metadata": { + "language": { + "name": "python" + }, + "client": { + "name": "appmap", + "url": "https://github.com/applandinc/appmap-python" + }, + "app": "Simple", + "recorder": { + "name": "pytest", + "type": "tests" + }, + "source_location": "tests/test_simple.py:5", + "name": "hello world", + "feature": "Hello world", + "test_status": "succeeded" + }, + "events": [ + { + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple.py", + "lineno": 8, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 1, + "event": "call", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 2, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello'" + }, + "parent_id": 2, + "id": 3, + "event": "return", + "thread_id": 1 + }, + { + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5, + "static": false, + "receiver": { + "class": "simple.Simple", + "kind": "req", + "name": "self", + "value": "" + }, + "parameters": [], + "id": 4, + "event": "call", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'world!'" + }, + "parent_id": 4, + "id": 5, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "class": "builtins.str", + "value": "'Hello world!'" + }, + "parent_id": 1, + "id": 6, + "event": "return", + "thread_id": 1 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 7, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "show_numpy_dict", + "path": "simple.py", + "lineno": 11 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [ + { + "kind": "req", + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "name": "d", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + } + ], + "id": 8, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "get_numpy_dict", + "path": "simple.py", + "lineno": 18 + }, + { + "return_value": { + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 8, + "id": 9, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "{np.int64(0): 'zero', np.int64(1): 'one'}", + "class": "builtins.dict", + "properties": [ + { + "name": "0", + "class": "builtins.str" + }, + { + "name": "1", + "class": "builtins.str" + } + ], + "size": 2 + }, + "parent_id": 7, + "id": 10, + "event": "return", + "thread_id": 1 + } + ], + "classMap": [ + { + "name": "simple", + "type": "package", + "children": [ + { + "name": "Simple", + "type": "class", + "children": [ + { + "name": "get_numpy_dict", + "type": "function", + "location": "simple.py:18", + "static": false + }, + { + "name": "hello", + "type": "function", + "location": "simple.py:2", + "static": false + }, + { + "name": "hello_world", + "type": "function", + "location": "simple.py:8", + "static": false + }, + { + "name": "show_numpy_dict", + "type": "function", + "location": "simple.py:11", + "static": false + }, + { + "name": "world", + "type": "function", + "location": "simple.py:5", + "static": false + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json index 0f12e30c..8d6436b4 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json @@ -8,95 +8,106 @@ "name": "appmap", "url": "https://github.com/applandinc/appmap-python" }, + "source_location": "tests/test_simple.py:5", + "name": "hello world", + "feature": "Hello world", "app": "Simple", "recorder": { "name": "pytest", "type": "tests" }, - "source_location": "test_simple.py:5", - "name": "hello world", - "feature": "Hello world", "test_status": "succeeded" }, "events": [ { - "defined_class": "simple.Simple", - "method_id": "hello_world", - "path": "simple.py", - "lineno": 8, + "static": true, + "parameters": [], + "id": 1, + "event": "call", + "thread_id": 1, + "defined_class": "tests.test_simple", + "method_id": "test_hello_world", + "path": "tests/test_simple.py", + "lineno": 6 + }, + { "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 1, + "id": 2, "event": "call", - "thread_id": 1 - }, - { + "thread_id": 1, "defined_class": "simple.Simple", - "method_id": "hello", + "method_id": "hello_world", "path": "simple.py", - "lineno": 2, + "lineno": 8 + }, + { "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 2, + "id": 3, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple.py", + "lineno": 2 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello'" + "value": "'Hello'", + "class": "builtins.str" }, - "parent_id": 2, - "id": 3, + "parent_id": 3, + "id": 4, "event": "return", "thread_id": 1 }, { - "defined_class": "simple.Simple", - "method_id": "world", - "path": "simple.py", - "lineno": 5, "static": false, "receiver": { - "class": "simple.Simple", "kind": "req", + "value": "", "name": "self", - "value": "" + "class": "simple.Simple" }, "parameters": [], - "id": 4, + "id": 5, "event": "call", - "thread_id": 1 + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple.py", + "lineno": 5 }, { "return_value": { - "class": "builtins.str", - "value": "'world!'" + "value": "'world!'", + "class": "builtins.str" }, - "parent_id": 4, - "id": 5, + "parent_id": 5, + "id": 6, "event": "return", "thread_id": 1 }, { "return_value": { - "class": "builtins.str", - "value": "'Hello world!'" + "value": "'Hello world!'", + "class": "builtins.str" }, - "parent_id": 1, - "id": 6, + "parent_id": 2, + "id": 7, "event": "return", "thread_id": 1 }, @@ -109,7 +120,7 @@ "class": "simple.Simple" }, "parameters": [], - "id": 7, + "id": 8, "event": "call", "thread_id": 1, "defined_class": "simple.Simple", @@ -144,7 +155,7 @@ "size": 2 } ], - "id": 8, + "id": 9, "event": "call", "thread_id": 1, "defined_class": "simple.Simple", @@ -168,8 +179,8 @@ ], "size": 2 }, - "parent_id": 8, - "id": 9, + "parent_id": 9, + "id": 10, "event": "return", "thread_id": 1 }, @@ -189,8 +200,18 @@ ], "size": 2 }, - "parent_id": 7, - "id": 10, + "parent_id": 8, + "id": 11, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "None", + "class": "builtins.NoneType" + }, + "parent_id": 1, + "id": 12, "event": "return", "thread_id": 1 } @@ -237,6 +258,24 @@ ] } ] + }, + { + "name": "tests", + "type": "package", + "children": [ + { + "name": "test_simple", + "type": "class", + "children": [ + { + "name": "test_hello_world", + "type": "function", + "location": "tests/test_simple.py:6", + "static": true + } + ] + } + ] } ] } \ No newline at end of file diff --git a/_appmap/test/data/pytest/expected/status_errored.metadata.json b/_appmap/test/data/pytest/expected/status_errored.metadata.json index 1c9d0f21..d2862df1 100644 --- a/_appmap/test/data/pytest/expected/status_errored.metadata.json +++ b/_appmap/test/data/pytest/expected/status_errored.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "RuntimeError: test error", - "location": "test_simple.py:30" + "location": "tests/test_simple.py:30" }, "exception": { "class": "RuntimeError", diff --git a/_appmap/test/data/pytest/expected/status_failed.metadata.json b/_appmap/test/data/pytest/expected/status_failed.metadata.json index cca17c0d..27955766 100644 --- a/_appmap/test/data/pytest/expected/status_failed.metadata.json +++ b/_appmap/test/data/pytest/expected/status_failed.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "AssertionError: assert False", - "location": "test_simple.py:16" + "location": "tests/test_simple.py:16" }, "exception": { "class": "AssertionError", diff --git a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json index 56494885..6f26ad59 100644 --- a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json +++ b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json @@ -2,7 +2,7 @@ "test_status": "failed", "test_failure": { "message": "AssertionError: assert False", - "location": "test_simple.py:21" + "location": "tests/test_simple.py:21" }, "exception": { "class": "AssertionError", diff --git a/_appmap/test/data/pytest/tests/__init__.py b/_appmap/test/data/pytest/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/_appmap/test/data/pytest/test_noappmap.py b/_appmap/test/data/pytest/tests/test_noappmap.py similarity index 100% rename from _appmap/test/data/pytest/test_noappmap.py rename to _appmap/test/data/pytest/tests/test_noappmap.py diff --git a/_appmap/test/data/pytest/test_simple.py b/_appmap/test/data/pytest/tests/test_simple.py similarity index 100% rename from _appmap/test/data/pytest/test_simple.py rename to _appmap/test/data/pytest/tests/test_simple.py diff --git a/_appmap/test/data/trial/appmap-no-test-cases.yml b/_appmap/test/data/trial/appmap-no-test-cases.yml new file mode 100644 index 00000000..595717ee --- /dev/null +++ b/_appmap/test/data/trial/appmap-no-test-cases.yml @@ -0,0 +1,4 @@ +name: deferred +record_test_cases: "false" +packages: +- path: test diff --git a/_appmap/test/data/trial/appmap.yml b/_appmap/test/data/trial/appmap.yml index 8dcecd82..ffa9f3da 100644 --- a/_appmap/test/data/trial/appmap.yml +++ b/_appmap/test/data/trial/appmap.yml @@ -1,3 +1,4 @@ name: deferred +record_test_cases: "true" packages: - path: test diff --git a/_appmap/test/data/trial/expected/pytest-no-test-cases.appmap.json b/_appmap/test/data/trial/expected/pytest-no-test-cases.appmap.json new file mode 100644 index 00000000..3bf51b05 --- /dev/null +++ b/_appmap/test/data/trial/expected/pytest-no-test-cases.appmap.json @@ -0,0 +1,28 @@ +{ + "version": "1.9", + "metadata": { + "language": { + "name": "python" + }, + "client": { + "name": "appmap", + "url": "https://github.com/applandinc/appmap-python" + }, + "feature_group": "Deferred", + "recording": { + "defined_class": "test.test_deferred.TestDeferred", + "method_id": "test_hello_world" + }, + "source_location": "test/test_deferred.py:7", + "name": "Deferred hello world", + "feature": "Hello world", + "app": "deferred", + "recorder": { + "name": "pytest", + "type": "tests" + }, + "test_status": "succeeded" + }, + "events": [], + "classMap": [] +} \ No newline at end of file diff --git a/_appmap/test/data/unittest/appmap-no-test-cases.yml b/_appmap/test/data/unittest/appmap-no-test-cases.yml new file mode 100644 index 00000000..4e0eb415 --- /dev/null +++ b/_appmap/test/data/unittest/appmap-no-test-cases.yml @@ -0,0 +1,4 @@ +name: Simple +record_test_cases: false +packages: +- path: simple diff --git a/_appmap/test/data/unittest/appmap.yml b/_appmap/test/data/unittest/appmap.yml index 2d20878f..817f8cf9 100644 --- a/_appmap/test/data/unittest/appmap.yml +++ b/_appmap/test/data/unittest/appmap.yml @@ -1,3 +1,4 @@ name: Simple +record_test_cases: true packages: - path: simple diff --git a/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json b/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json new file mode 100644 index 00000000..2880ba33 --- /dev/null +++ b/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json @@ -0,0 +1,148 @@ +{ + "version": "1.9", + "metadata": { + "language": { + "name": "python" + }, + "client": { + "name": "appmap", + "url": "https://github.com/applandinc/appmap-python" + }, + "feature_group": "Unit test test", + "recording": { + "defined_class": "simple.test_simple.UnitTestTest", + "method_id": "test_hello_world" + }, + "source_location": "simple/test_simple.py:14", + "name": "Unit test test hello world", + "feature": "Hello world", + "app": "Simple", + "recorder": { + "name": "unittest", + "type": "tests" + }, + "test_status": "succeeded" + }, + "events": [ + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [ + { + "kind": "req", + "value": "'!'", + "name": "bang", + "class": "builtins.str" + } + ], + "id": 1, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello_world", + "path": "simple/__init__.py", + "lineno": 8 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 2, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "hello", + "path": "simple/__init__.py", + "lineno": 2 + }, + { + "return_value": { + "value": "'Hello'", + "class": "builtins.str" + }, + "parent_id": 2, + "id": 3, + "event": "return", + "thread_id": 1 + }, + { + "static": false, + "receiver": { + "kind": "req", + "value": "", + "name": "self", + "class": "simple.Simple" + }, + "parameters": [], + "id": 4, + "event": "call", + "thread_id": 1, + "defined_class": "simple.Simple", + "method_id": "world", + "path": "simple/__init__.py", + "lineno": 5 + }, + { + "return_value": { + "value": "'world'", + "class": "builtins.str" + }, + "parent_id": 4, + "id": 5, + "event": "return", + "thread_id": 1 + }, + { + "return_value": { + "value": "'Hello world!'", + "class": "builtins.str" + }, + "parent_id": 1, + "id": 6, + "event": "return", + "thread_id": 1 + } + ], + "classMap": [ + { + "name": "simple", + "type": "package", + "children": [ + { + "name": "Simple", + "type": "class", + "children": [ + { + "name": "hello", + "type": "function", + "location": "simple/__init__.py:2", + "static": false + }, + { + "name": "hello_world", + "type": "function", + "location": "simple/__init__.py:8", + "static": false + }, + { + "name": "world", + "type": "function", + "location": "simple/__init__.py:5", + "static": false + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index d5e2ed7a..a1492e4a 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -139,8 +139,10 @@ def test_empty_path(self, data_dir, caplog): class DefaultHelpers: def check_default_packages(self, actual_packages): + # Project directory has a "test" subdirectory, so actual_packages may have it (indicating a + # bug in the way directories are excluded). pkgs = [p["path"] for p in actual_packages if p["path"] in ("package", "test")] - assert ["package", "test"] == sorted(pkgs) + assert ["package"] == sorted(pkgs) def check_default_config(self, expected_name): assert appmap.enabled() @@ -149,6 +151,7 @@ def check_default_config(self, expected_name): assert default_config.name == expected_name self.check_default_packages(default_config.packages) assert default_config.default["appmap_dir"] == "tmp/appmap" + assert default_config.default["record_test_cases"] is False class TestDefaultConfig(DefaultHelpers): @@ -249,7 +252,7 @@ def test_empty(self, tmpdir): def test_missing_name(self, tmpdir): with self.incomplete_config() as f: - print('packages: [{"path": "package"}, {"path": "test"}]', file=f) + print('packages: [{"path": "package"}]', file=f) _appmap.initialize( cwd=tmpdir, env={"APPMAP_CONFIG": "appmap-incomplete.yml"}, diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index f57e0218..fba942fd 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -149,4 +149,3 @@ def check_call_return_stack_order(events): return True return False - diff --git a/_appmap/test/test_fastapi.py b/_appmap/test/test_fastapi.py index 7122508b..9a39eb93 100644 --- a/_appmap/test/test_fastapi.py +++ b/_appmap/test/test_fastapi.py @@ -24,11 +24,9 @@ class TestRecordRequests(_TestRecordRequests): @pytest.mark.app(remote_enabled=True) class TestRemoteRecording(_TestRemoteRecording): - def __init__(self): - self.expected_thread_id = None - self.expected_content_type = None - def setup_method(self): + # Can't add __init__, pytest won't collect test classes that have one + # pylint: disable=attribute-defined-outside-init self.expected_thread_id = 1 self.expected_content_type = "application/json" diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index b847c1e3..49467a95 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -67,6 +67,14 @@ def test_enabled(self, testdir): verify_expected_appmap(testdir) verify_expected_metadata(testdir) + def test_enabled_no_test_cases(self, testdir, monkeypatch): + monkeypatch.setenv("APPMAP_CONFIG", "appmap-no-test-cases.yml") + + self.run_tests(testdir) + + assert len(list(testdir.output().iterdir())) == 7 + verify_expected_appmap(testdir, "-no-test-cases") + verify_expected_metadata(testdir) class TestPytestRunnerUnittest(_TestTestRunner): @classmethod @@ -105,6 +113,16 @@ def test_enabled(self, testdir): verify_expected_appmap(testdir, f"-numpy{numpy_version.major}") verify_expected_metadata(testdir) + def test_enabled_no_test_cases(self, testdir, monkeypatch): + monkeypatch.setenv("APPMAP_CONFIG", "appmap-no-test-cases.yml") + + self.run_tests(testdir) + assert len(list(testdir.output().iterdir())) == 6 + numpy_version = package_version("numpy") + verify_expected_appmap(testdir, f"-numpy{numpy_version.major}-no-test-cases") + verify_expected_metadata(testdir) + + @pytest.mark.example_dir("trial") class TestPytestRunnerTrial(_TestTestRunner): @classmethod @@ -122,10 +140,15 @@ def run_tests(self, testdir): # unclean. result.assert_outcomes(xfailed=1) - def test_pytest_trial(self, testdir): + def test_enabled(self, testdir): self.run_tests(testdir) verify_expected_appmap(testdir) + def test_enabled_no_test_cases(self, testdir, monkeypatch): + monkeypatch.setenv("APPMAP_CONFIG", "appmap-no-test-cases.yml") + self.run_tests(testdir) + verify_expected_appmap(testdir, "-no-test-cases") + EMPTY_APPMAP = types.SimpleNamespace(events=[]) diff --git a/_appmap/testing_framework.py b/_appmap/testing_framework.py index eeffe45c..9a676be1 100644 --- a/_appmap/testing_framework.py +++ b/_appmap/testing_framework.py @@ -8,7 +8,8 @@ import inflection -from _appmap import configuration, env, recording +from _appmap import env, recording +from _appmap.configuration import Config from _appmap.recording import Recording from _appmap.utils import fqname, root_relative_path @@ -104,15 +105,13 @@ def record(self, klass, method, **kwds): item = FuncItem(klass, method, **kwds) metadata = item.metadata - metadata.update( - { - "app": configuration.Config.current.name, - "recorder": { - "name": self.name, - "type": self.recorder_type, - }, - } - ) + metadata.update({ + "app": Config.current.name, + "recorder": { + "name": self.name, + "type": self.recorder_type, + }, + }) rec = Recording() environ = env.Env.current @@ -174,3 +173,9 @@ def failure_location(exn: Exception) -> str: if relative: break return loc + + +def disable_test_case(fn): + record_test_cases = Config.current.record_test_cases + if not record_test_cases and hasattr(fn, "_self_enabled"): # it's instrumented + fn._self_enabled = False # pylint: disable=protected-access diff --git a/_appmap/unittest.py b/_appmap/unittest.py index d2a2bd7f..eb79c9a0 100644 --- a/_appmap/unittest.py +++ b/_appmap/unittest.py @@ -1,7 +1,3 @@ -import sys -import unittest -from contextlib import contextmanager - from _appmap import noappmap, testing_framework, wrapt from _appmap.env import Env from _appmap.utils import get_function_location @@ -13,72 +9,31 @@ def _get_test_location(cls, method_name): fn = getattr(cls, method_name) return get_function_location(fn) - -if sys.version_info[1] < 8: - # Prior to 3.8, unittest called the test case's test method directly, which left us without an - # opportunity to hook it. So, instead, instrument unittest.case._Outcome.testPartExecutor, a - # method used to run test cases. `isTest` will be True when the part is the actual test method, - # False when it's setUp or teardown. - @wrapt.patch_function_wrapper("unittest.case", "_Outcome.testPartExecutor") - @contextmanager - def testPartExecutor(wrapped, _, args, kwargs): - def _args(test_case, *_, isTest=False, **__): - return (test_case, isTest) - - test_case, is_test = _args(*args, **kwargs) - already_recording = getattr(test_case, "_appmap_pytest_recording", None) - # fmt: off - if ( - (not is_test) - or isinstance(test_case, unittest.case._SubTest) # pylint: disable=protected-access - or already_recording - ): - # fmt: on - with wrapped(*args, **kwargs): - yield - return - - method_name = test_case.id().split(".")[-1] - location = _get_test_location(test_case.__class__, method_name) - with _session.record( - test_case.__class__, method_name, location=location - ) as metadata: - if metadata: - with wrapped( - *args, **kwargs - ), testing_framework.collect_result_metadata(metadata): - yield - else: - # session.record may return None - yield - -else: - # We need to disable request recording in TestCase._callSetUp too - # in order to prevent creation of a request recording besides test - # recording when requests are made inside setUp method. - # This edge case can be observed in this test in django project: - # $ APPMAP=TRUE ./runtests.py auth_tests.test_views.ChangelistTests.test_user_change_email - #  (ChangelistTests.setUp makes a request) - @wrapt.patch_function_wrapper("unittest.case", "TestCase._callSetUp") - def callSetUp(wrapped, test_case, args, kwargs): # pylint: disable=unused-argument - with Env.current.disabled("requests"): - wrapped(*args, **kwargs) - - # As of 3.8, unittest.case.TestCase now calls the test's method indirectly, through - # TestCase._callTestMethod. Hook that to manage a recording session. - @wrapt.patch_function_wrapper("unittest.case", "TestCase._callTestMethod") - def callTestMethod(wrapped, test_case, args, kwargs): - already_recording = getattr(test_case, "_appmap_pytest_recording", None) - - test_method_name = test_case._testMethodName # pylint: disable=protected-access - test_method = getattr(test_case, test_method_name) - if already_recording or noappmap.disables(test_method, test_case.__class__): - wrapped(*args, **kwargs) - return - - method_name = test_case.id().split(".")[-1] - location = _get_test_location(test_case.__class__, method_name) - with _session.record(test_case.__class__, method_name, location=location) as metadata: - if metadata: - with testing_framework.collect_result_metadata(metadata): - wrapped(*args, **kwargs) +# We need to disable request recording in TestCase._callSetUp. This prevents creation of a request +# recording calls when requests made inside setUp method. +# +# This edge case can be observed in this test in django project: +# $ APPMAP=TRUE ./runtests.py auth_tests.test_views.ChangelistTests.test_user_change_email +# (ChangelistTests.setUp makes a request) +@wrapt.patch_function_wrapper("unittest.case", "TestCase._callSetUp") +def callSetUp(wrapped, _, args, kwargs): + with Env.current.disabled("requests"): + wrapped(*args, **kwargs) + +@wrapt.patch_function_wrapper("unittest.case", "TestCase._callTestMethod") +def callTestMethod(wrapped, test_case, _, kwargs): + already_recording = getattr(test_case, "_appmap_pytest_recording", None) + + test_method_name = test_case._testMethodName # pylint: disable=protected-access + test_method = getattr(test_case, test_method_name) + if already_recording or noappmap.disables(test_method, test_case.__class__): + wrapped(test_method, **kwargs) + return + + method_name = test_case.id().split(".")[-1] + location = _get_test_location(test_case.__class__, method_name) + testing_framework.disable_test_case(test_method) + with _session.record(test_case.__class__, method_name, location=location) as metadata: + if metadata: + with testing_framework.collect_result_metadata(metadata): + wrapped(test_method, **kwargs) diff --git a/appmap/pytest.py b/appmap/pytest.py index 8c555b52..67d3cb4d 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -58,6 +58,7 @@ def pytest_runtest_call(item): True, ) if not noappmap.disables(item.obj, item.cls): + testing_framework.disable_test_case(item.obj) item.obj = recorded_testcase(item)(item.obj) @pytest.hookimpl(hookwrapper=True) @@ -76,6 +77,7 @@ def pytest_pyfunc_call(pyfuncitem): method_id=pyfuncitem.originalname, location=pyfuncitem.location, ) as metadata: + testing_framework.disable_test_case(pyfuncitem.obj) result = yield try: with testing_framework.collect_result_metadata(metadata): diff --git a/vendor/_appmap/wrapt/wrappers.py b/vendor/_appmap/wrapt/wrappers.py index bbe9b0e5..31739da0 100644 --- a/vendor/_appmap/wrapt/wrappers.py +++ b/vendor/_appmap/wrapt/wrappers.py @@ -509,22 +509,25 @@ def _unpack_self(self, *args): return self.__wrapped__(*_args, **_kwargs) class _FunctionWrapperBase(ObjectProxy): - - __slots__ = ('_self_instance', '_self_wrapper', '_self_enabled', - '_self_binding', '_self_parent', '_bfws') - - def __init__(self, wrapped, instance, wrapper, enabled=None, - binding='function', parent=None): - + __slots__ = ( + "_self_instance", + "_self_wrapper", + "_self_enabled", + "_self_binding", + "_self_parent", + "_bfws", "_appmap_instrumented", + ) + + def __init__(self, wrapped, instance, wrapper, enabled=None, binding="function", parent=None): super(_FunctionWrapperBase, self).__init__(wrapped) - object.__setattr__(self, '_self_instance', instance) - object.__setattr__(self, '_self_wrapper', wrapper) - object.__setattr__(self, '_self_enabled', enabled) - object.__setattr__(self, '_self_binding', binding) - object.__setattr__(self, '_self_parent', parent) - object.__setattr__(self, '_bfws', list()) + object.__setattr__(self, "_self_instance", instance) + object.__setattr__(self, "_self_wrapper", wrapper) + object.__setattr__(self, "_self_enabled", enabled) + object.__setattr__(self, "_self_binding", binding) + object.__setattr__(self, "_self_parent", parent) + object.__setattr__(self, "_bfws", list()) object.__setattr__(self, "_appmap_instrumented", False) def __get__(self, instance, owner): From ac94204d9bd35bf238865a7bf44cea039f8282fb Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 19 Jul 2024 05:33:01 -0400 Subject: [PATCH 069/113] fix: add ruff Add ruff to dev dependencies, along with an example config. At some point in the future, we may want to switch to using it, rather than pylint. --- .gitignore | 2 +- pyproject.toml | 1 + ruff.toml.example | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 ruff.toml.example diff --git a/.gitignore b/.gitignore index 274346f8..c6f0f746 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ htmlcov/ /.tox /node_modules - +/ruff.toml \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d2c18357..33a7be7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ pytest-env = "^1.1.3" pytest-console-scripts = "^1.4.1" pytest-xdist = "^3.6.1" psutil = "^6.0.0" +ruff = "^0.5.3" [build-system] requires = ["poetry-core>=1.1.0"] diff --git a/ruff.toml.example b/ruff.toml.example new file mode 100644 index 00000000..ef74a1df --- /dev/null +++ b/ruff.toml.example @@ -0,0 +1,5 @@ +line-length = 100 +extend-exclude = ["sitecustomize.py"] + +[lint.isort] +known-first-party = ['appmap', '_appmap'] \ No newline at end of file From 7538fa7ee811a32b398d492a5140ef180c80160b Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 26 Jul 2024 13:43:14 -0400 Subject: [PATCH 070/113] test: pin incremental The latest update to incremental (which twisted depends on), appears to be broken. Pin to the previous version. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 33a7be7d..0c5dde22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ packaging = ">=19.0" [tool.poetry.group.dev.dependencies] Twisted = "^22.4.0" +incremental = "<24.7.0" asgiref = "^3.7.2" black = "^24.2.0" coverage = "^5.3" From 6fcad4abcc323b2dafd6e96e7b674bfa8ac1f88a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 26 Jul 2024 19:49:10 +0000 Subject: [PATCH 071/113] chore(release): 2.1.3 [skip ci] ## [2.1.3](https://github.com/getappmap/appmap-python/compare/v2.1.2...v2.1.3) (2024-07-26) ### Bug Fixes * add APPMAP_INSTRUMENT_PROPERTIES ([11b6307](https://github.com/getappmap/appmap-python/commit/11b6307cf2bdbfae50f30d4f329e6ba3ac6f4035)) * add ruff ([ac94204](https://github.com/getappmap/appmap-python/commit/ac94204d9bd35bf238865a7bf44cea039f8282fb)) * improve property handling ([5cce0f0](https://github.com/getappmap/appmap-python/commit/5cce0f0644eebf7d19bad5cda61726393cd7ba68)) * show config packages on startup ([feec761](https://github.com/getappmap/appmap-python/commit/feec761fefd5596c4fd7bde0cd9c3901e02791b3)) * try to avoid recording tests ([1847b0e](https://github.com/getappmap/appmap-python/commit/1847b0e7177327adc080854fbbec17b89166d516)) --- CHANGELOG.md | 11 +++++++++++ pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72e9dc93..31b04b67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## [2.1.3](https://github.com/getappmap/appmap-python/compare/v2.1.2...v2.1.3) (2024-07-26) + + +### Bug Fixes + +* add APPMAP_INSTRUMENT_PROPERTIES ([11b6307](https://github.com/getappmap/appmap-python/commit/11b6307cf2bdbfae50f30d4f329e6ba3ac6f4035)) +* add ruff ([ac94204](https://github.com/getappmap/appmap-python/commit/ac94204d9bd35bf238865a7bf44cea039f8282fb)) +* improve property handling ([5cce0f0](https://github.com/getappmap/appmap-python/commit/5cce0f0644eebf7d19bad5cda61726393cd7ba68)) +* show config packages on startup ([feec761](https://github.com/getappmap/appmap-python/commit/feec761fefd5596c4fd7bde0cd9c3901e02791b3)) +* try to avoid recording tests ([1847b0e](https://github.com/getappmap/appmap-python/commit/1847b0e7177327adc080854fbbec17b89166d516)) + ## [2.1.2](https://github.com/getappmap/appmap-python/compare/v2.1.1...v2.1.2) (2024-07-16) diff --git a/pyproject.toml b/pyproject.toml index 0c5dde22..53667b35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.2" +version = "2.1.3" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From a2803003a57b1aabc87cadc755786613e2709ff2 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Fri, 26 Jul 2024 17:02:35 -0400 Subject: [PATCH 072/113] fix: disable property instrumentation by default Disable property instrumentation until it works properly for Django. --- _appmap/importer.py | 2 +- _appmap/test/test_properties.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/_appmap/importer.py b/_appmap/importer.py index b188539b..5ee79fc4 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -167,7 +167,7 @@ def initialize(cls): cls.filter_chain = [] cls._skip_instrumenting = ("appmap", "_appmap") cls.instrument_properties = ( - Env.current.get("APPMAP_INSTRUMENT_PROPERTIES", "true").lower() == "true" + Env.current.get("APPMAP_INSTRUMENT_PROPERTIES", "false").lower() == "true" ) @classmethod diff --git a/_appmap/test/test_properties.py b/_appmap/test/test_properties.py index c22ae029..d1c098e6 100644 --- a/_appmap/test/test_properties.py +++ b/_appmap/test/test_properties.py @@ -5,10 +5,7 @@ import pytest from _appmap.test.helpers import DictIncluding -pytestmark = [ - pytest.mark.appmap_enabled, -] - +pytestmark = pytest.mark.skip(reason="property instrumentation is broken in Django") @pytest.fixture(autouse=True) def setup(with_data_dir): # pylint: disable=unused-argument From f092a77f2f5538a5d892474a95bae8d4a675a89c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 26 Jul 2024 22:28:07 +0000 Subject: [PATCH 073/113] chore(release): 2.1.4 [skip ci] ## [2.1.4](https://github.com/getappmap/appmap-python/compare/v2.1.3...v2.1.4) (2024-07-26) ### Bug Fixes * disable property instrumentation by default ([a280300](https://github.com/getappmap/appmap-python/commit/a2803003a57b1aabc87cadc755786613e2709ff2)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31b04b67..0332f999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.1.4](https://github.com/getappmap/appmap-python/compare/v2.1.3...v2.1.4) (2024-07-26) + + +### Bug Fixes + +* disable property instrumentation by default ([a280300](https://github.com/getappmap/appmap-python/commit/a2803003a57b1aabc87cadc755786613e2709ff2)) + ## [2.1.3](https://github.com/getappmap/appmap-python/compare/v2.1.2...v2.1.3) (2024-07-26) diff --git a/pyproject.toml b/pyproject.toml index 53667b35..2810fe4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.3" +version = "2.1.4" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 7b3119a4fdf60e19a28a279d4ba6afe379925e14 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Sun, 28 Jul 2024 16:36:12 -0400 Subject: [PATCH 074/113] fix: reenable instrumentation of properties Have APPMAP_INSTRUMENT_PROPERTIES default to true once again. Special-case properties that have operator.attrgetter as their getter. Django uses this pattern extensively, and this special-case fixes some failures when trying to generate AppMap data for Django tests. --- _appmap/event.py | 28 ++++++- _appmap/importer.py | 7 +- _appmap/instrument.py | 2 +- _appmap/test/data/appmap.yml | 1 + _appmap/test/data/example_class.py | 49 ------------ _appmap/test/data/properties_class.py | 56 +++++++++++++ _appmap/test/test_params.py | 3 +- _appmap/test/test_properties.py | 111 ++++++++++++++------------ _appmap/utils.py | 9 +-- appmap/fastapi.py | 2 +- pylintrc | 5 +- 11 files changed, 154 insertions(+), 119 deletions(-) create mode 100644 _appmap/test/data/properties_class.py diff --git a/_appmap/event.py b/_appmap/event.py index 82661152..a61a5a04 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -1,6 +1,7 @@ # pylint: disable=missing-module-docstring,missing-class-docstring,missing-function-docstring import inspect import logging +import operator import threading from functools import lru_cache, partial from inspect import Parameter, Signature @@ -180,18 +181,19 @@ class CallEvent(Event): __slots__ = ["_fn", "_fqfn", "static", "receiver", "parameters", "labels", "auxtype"] @staticmethod - def make(fn, fntype): + def make(filterable): """ Return a factory for creating new CallEvents based on introspecting the given function. """ # Delete the labels so the app doesn't see them. + fn = filterable.obj labels = getattr(fn, "_appmap_labels", None) if labels: del fn._appmap_labels - return partial(CallEvent, fn, fntype, labels=labels) + return partial(CallEvent, filterable, labels=labels) @staticmethod def make_params(filterable): @@ -312,10 +314,28 @@ def comment(self): comment = inspect.getcomments(self._fn) return comment - def __init__(self, fn, fntype, parameters, labels): + def __init__(self, filterable, parameters, labels): super().__init__("call") + fn = filterable.obj self._fn = fn - self._fqfn = FqFnName(fn) + if type(fn) is not operator.attrgetter: + # fn is a regular function + modname = fn.__module__ + qualname = fn.__qualname__ + elif (parts := filterable.fqname.split(".")) and len(parts) > 2: + # fn is an attrgetter, which will is being used as the getter for a property. If + # filterable.fqname has enough components to be the fully-qualified name of a class + # member, set the module name and qualname based on those components. + modname = ".".join(parts[:-2]) + qualname = ".".join(parts[-2:]) + else: + # The two previous cases should handle all known possibilities, but don't crash if + # somethign else sneaks in. + modname = "unknown" + qualname = "unknown" + self._fqfn = FqFnName(modname, qualname) + + fntype = filterable.fntype self.static = fntype in FnType.STATIC | FnType.CLASS | FnType.MODULE self.receiver = None if fntype in FnType.CLASS | FnType.INSTANCE: diff --git a/_appmap/importer.py b/_appmap/importer.py index 5ee79fc4..a98d275e 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -134,7 +134,7 @@ def is_member_func(m): static_value = inspect.getattr_static(cls, key) # Don't use isinstance to check the type of static_value -- we don't want to invoke the # descriptor protocol. - if Importer.instrument_properties and type(static_value) is property: # pylint: disable=unidiomatic-typecheck + if Importer.instrument_properties and type(static_value) is property: properties[key] = ( static_value, { @@ -167,7 +167,7 @@ def initialize(cls): cls.filter_chain = [] cls._skip_instrumenting = ("appmap", "_appmap") cls.instrument_properties = ( - Env.current.get("APPMAP_INSTRUMENT_PROPERTIES", "false").lower() == "true" + Env.current.get("APPMAP_INSTRUMENT_PROPERTIES", "true").lower() == "true" ) @classmethod @@ -221,7 +221,8 @@ def instrument_functions(filterable, selected_functions=None): new_fn = cls.instrument_function(prop_name, filterableFn, selected_functions) if new_fn != fn: new_fn = wrapt.FunctionWrapper(fn, new_fn) - # Set _appmap_instrumented on the FunctionWrapper, not on the wrapped function + # Set _appmap_instrumented on the FunctionWrapper, not on the wrapped + # function. new_fn._appmap_instrumented = True # pylint: disable=protected-access instrumented_fns[k] = new_fn diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 79085372..a21a2c8d 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -121,7 +121,7 @@ def instrument(filterable): logger.debug("hooking %s", filterable.fqname) fn = filterable.obj - make_call_event = event.CallEvent.make(fn, filterable.fntype) + make_call_event = event.CallEvent.make(filterable) params = CallEvent.make_params(filterable) # django depends on being able to find the cache_clear attribute diff --git a/_appmap/test/data/appmap.yml b/_appmap/test/data/appmap.yml index 65d41b0c..97a4a91f 100644 --- a/_appmap/test/data/appmap.yml +++ b/_appmap/test/data/appmap.yml @@ -4,6 +4,7 @@ packages: - path: example_class.Super shallow: true - path: example_class +- path: properties_class - path: appmap_testing - path: package1 - dist: PyYAML diff --git a/_appmap/test/data/example_class.py b/_appmap/test/data/example_class.py index a5bc9992..4d7c2c93 100644 --- a/_appmap/test/data/example_class.py +++ b/_appmap/test/data/example_class.py @@ -114,55 +114,6 @@ def with_comment(self): def return_self(self): return self - def __init__(self): - self._read_only = "read only" - self._fully_accessible = "fully accessible" - self._undecorated = "undecorated" - - @property - def read_only(self): - """Read-only""" - return self._read_only - - @property - def fully_accessible(self): - """Fully-accessible""" - return self._fully_accessible - - @fully_accessible.setter - def fully_accessible(self, v): - self._fully_accessible = v - - @fully_accessible.deleter - def fully_accessible(self): - del self._fully_accessible - - def get_undecorated(self): - return self._undecorated - - def set_undecorated(self, value): - self._undecorated = value - - def delete_undecorated(self): - del self._undecorated - - undecorated_property = property(get_undecorated, set_undecorated, delete_undecorated) - - def set_write_only(self, v): - self._write_only = v - - def del_write_only(self): - del self._write_only - - write_only = property(None, set_write_only, del_write_only, "Write-only") - - def raise_base_exception(self) -> NoReturn: - raise BaseException("not derived from Exception") # pylint: disable=broad-exception-raised def modfunc(): return "Hello world!" - -if __name__ == "__main__": - ec = ExampleClass() - ec.fully_accessible = "updated" - print(ec.fully_accessible) \ No newline at end of file diff --git a/_appmap/test/data/properties_class.py b/_appmap/test/data/properties_class.py new file mode 100644 index 00000000..ab81f898 --- /dev/null +++ b/_appmap/test/data/properties_class.py @@ -0,0 +1,56 @@ +from functools import cached_property +import operator +from typing import NoReturn + + +class PropertiesClass: + def __init__(self): + self._read_only = "read only" + self._fully_accessible = "fully accessible" + self._undecorated = "undecorated" + + @property + def read_only(self): + """Read-only""" + return self._read_only + + @property + def fully_accessible(self): + """Fully-accessible""" + return self._fully_accessible + + @fully_accessible.setter + def fully_accessible(self, v): + self._fully_accessible = v + + @fully_accessible.deleter + def fully_accessible(self): + del self._fully_accessible + + def get_undecorated(self): + return self._undecorated + + def set_undecorated(self, value): + self._undecorated = value + + def delete_undecorated(self): + del self._undecorated + + undecorated_property = property(get_undecorated, set_undecorated, delete_undecorated) + + def set_write_only(self, v): + self._write_only = v + + def del_write_only(self): + del self._write_only + + write_only = property(None, set_write_only, del_write_only, "Write-only") + + def raise_base_exception(self) -> NoReturn: + raise BaseException("not derived from Exception") # pylint: disable=broad-exception-raised + + @cached_property + def cached_read_only(self): + return self._read_only + + operator_read_only = property(operator.attrgetter("cached_read_only")) diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index 3c079d55..26669861 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -28,8 +28,7 @@ def __init__(self, C): @classmethod def prepare(cls, ffn): - fn = ffn.obj - make_call_event = CallEvent.make(fn, ffn.fntype) + make_call_event = CallEvent.make(ffn) params = CallEvent.make_params(ffn) def wrapped_fn(_, instance, args, kwargs): diff --git a/_appmap/test/test_properties.py b/_appmap/test/test_properties.py index d1c098e6..565d0566 100644 --- a/_appmap/test/test_properties.py +++ b/_appmap/test/test_properties.py @@ -5,47 +5,43 @@ import pytest from _appmap.test.helpers import DictIncluding -pytestmark = pytest.mark.skip(reason="property instrumentation is broken in Django") +pytestmark = pytest.mark.appmap_enabled @pytest.fixture(autouse=True) def setup(with_data_dir): # pylint: disable=unused-argument - # with_data_dir sets up sys.path so example_class can be imported + # with_data_dir sets up sys.path so properties_class can be imported pass def test_getter_instrumented(events): - from example_class import ExampleClass + from properties_class import PropertiesClass - ec = ExampleClass() + ec = PropertiesClass() - actual = ExampleClass.read_only.__doc__ + actual = PropertiesClass.read_only.__doc__ assert actual == "Read-only" assert ec.read_only == "read only" with pytest.raises(AttributeError, match=r".*(has no setter|can't set attribute).*"): - # E AttributeError: can't set attribute - ec.read_only = "not allowed" with pytest.raises(AttributeError, match=r".*(has no deleter|can't delete attribute).*"): del ec.read_only assert len(events) == 2 - assert events[0].to_dict() == DictIncluding( - { - "event": "call", - "defined_class": "example_class.ExampleClass", - "method_id": "read_only (get)", - } - ) + assert events[0].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "read_only (get)", + }) def test_accessible_instrumented(events): - from example_class import ExampleClass + from properties_class import PropertiesClass - ec = ExampleClass() - assert ExampleClass.fully_accessible.__doc__ == "Fully-accessible" + ec = PropertiesClass() + assert PropertiesClass.fully_accessible.__doc__ == "Fully-accessible" assert ec.fully_accessible == "fully accessible" @@ -55,37 +51,31 @@ def test_accessible_instrumented(events): del ec.fully_accessible - # assert len(events) == 6 - assert events[0].to_dict() == DictIncluding( - { - "event": "call", - "defined_class": "example_class.ExampleClass", - "method_id": "fully_accessible (get)", - } - ) - - assert events[2].to_dict() == DictIncluding( - { - "event": "call", - "defined_class": "example_class.ExampleClass", - "method_id": "fully_accessible (set)", - } - ) - - assert events[4].to_dict() == DictIncluding( - { - "event": "call", - "defined_class": "example_class.ExampleClass", - "method_id": "fully_accessible (del)", - } - ) + assert len(events) == 6 + assert events[0].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "fully_accessible (get)", + }) + + assert events[2].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "fully_accessible (set)", + }) + + assert events[4].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "fully_accessible (del)", + }) def test_writable_instrumented(events): - from example_class import ExampleClass + from properties_class import PropertiesClass - ec = ExampleClass() - assert ExampleClass.write_only.__doc__ == "Write-only" + ec = PropertiesClass() + assert PropertiesClass.write_only.__doc__ == "Write-only" with pytest.raises(AttributeError, match=r".*(has no getter|unreadable attribute).*"): _ = ec.write_only @@ -93,10 +83,29 @@ def test_writable_instrumented(events): ec.write_only = "updated example" assert len(events) == 2 - assert events[0].to_dict() == DictIncluding( - { - "event": "call", - "defined_class": "example_class.ExampleClass", - "method_id": "set_write_only (set)", - } - ) + assert events[0].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "set_write_only (set)", + }) + + +def test_operator_attrgetter(events): + from properties_class import PropertiesClass + + ec = PropertiesClass() + + assert ec.operator_read_only == "read only" + + with pytest.raises(AttributeError, match=r".*(has no setter|can't set attribute).*"): + ec.operator_read_only = "not allowed" + + with pytest.raises(AttributeError, match=r".*(has no deleter|can't delete attribute).*"): + del ec.operator_read_only + + assert len(events) == 2 + assert events[0].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class.PropertiesClass", + "method_id": "operator_read_only (get)", + }) diff --git a/_appmap/utils.py b/_appmap/utils.py index 0b39e947..06909ba6 100644 --- a/_appmap/utils.py +++ b/_appmap/utils.py @@ -8,7 +8,6 @@ from contextvars import ContextVar from enum import Enum, IntFlag, auto from pathlib import Path -from typing import Any, Callable from .env import Env @@ -79,10 +78,8 @@ class FqFnName: FqFnName makes it easy to reference the parts of the fully-qualified name of a callable. """ - def __init__(self, fn: Callable[..., Any]): - - self._modname = fn.__module__ - qualname = fn.__qualname__ + def __init__(self, modname, qualname): + self._modname = modname if "." in qualname: self._scope = Scope.CLASS self._class_name, self._fn_name = qualname.rsplit(".", 1) @@ -112,8 +109,6 @@ def fqfn(self): def fn_name(self): return self._fn_name -FqFnName(fqname) - def root_relative_path(path): """Returns the path relative to the current root_dir. diff --git a/appmap/fastapi.py b/appmap/fastapi.py index da8acc44..97aa1b7b 100644 --- a/appmap/fastapi.py +++ b/appmap/fastapi.py @@ -31,7 +31,7 @@ def _add_api_route(wrapped, _, args, kwargs): fn = args[1] - fqn = utils.FqFnName(fn) + fqn = utils.FqFnName(fn.__module__, fn.__qualname__) scope = Filterable(fqn.scope, fqn.fqclass, None) filterable_fn = FilterableFn(scope, fn.__name__, fn, fn) diff --git a/pylintrc b/pylintrc index 2cf9f132..808bf36e 100644 --- a/pylintrc +++ b/pylintrc @@ -416,7 +416,10 @@ confidence=HIGH, # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=raw-checker-failed, +# Disable unidiomatic-typecheck. Using isinstance() invokes the descriptor protocol, which can have +# side effects. Using type() avoids this. +disable=unidiomatic-typecheck, + raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, From a708b5d5c16a62ce85da7d59b6f398a5847d9c45 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 31 Jul 2024 07:56:31 -0400 Subject: [PATCH 075/113] fixup! fix: reenable instrumentation of properties --- _appmap/event.py | 42 +++++++++++++++++---------- _appmap/importer.py | 6 ++-- _appmap/test/data/properties_class.py | 17 ++++++++++- _appmap/test/test_properties.py | 37 +++++++++++++++++++++++ 4 files changed, 83 insertions(+), 19 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index a61a5a04..0bb254cf 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -6,6 +6,7 @@ from functools import lru_cache, partial from inspect import Parameter, Signature from itertools import chain +import types from .env import Env from .recorder import Recorder @@ -175,6 +176,31 @@ def to_dict(self, value): ret.update(describe_value(self.name, value)) return ret +def _get_name_parts(filterable): + """ + Return the module name and qualname for filterable.obj. + + If filterable.obj is an operator.attrgetter that we've determined is associated with a property, + compute the names from the fqname of the filterable. If it's anything else, try to get its + __module__ and __qualname__, falling back to default values if they're not available. + """ + fn = filterable.obj + assert callable(fn), f"{filterable} doesn't have a callable obj" + + if ( + type(fn) is operator.attrgetter + and (parts := filterable.fqname.split(".")) + and len(parts) > 2 + ): + # filterable.fqname was set when we identified this filterable as a property + modname = ".".join(parts[:-2]) + qualname = ".".join(parts[-2:]) + else: + modname = getattr(fn, "__module__", "unknown") + qualname = getattr(fn, "__qualname__", None) + if qualname is None: + qualname = getattr(fn.__class__, "__name__", "unknown") + return modname, qualname class CallEvent(Event): # pylint: disable=method-cache-max-size-none @@ -318,21 +344,7 @@ def __init__(self, filterable, parameters, labels): super().__init__("call") fn = filterable.obj self._fn = fn - if type(fn) is not operator.attrgetter: - # fn is a regular function - modname = fn.__module__ - qualname = fn.__qualname__ - elif (parts := filterable.fqname.split(".")) and len(parts) > 2: - # fn is an attrgetter, which will is being used as the getter for a property. If - # filterable.fqname has enough components to be the fully-qualified name of a class - # member, set the module name and qualname based on those components. - modname = ".".join(parts[:-2]) - qualname = ".".join(parts[-2:]) - else: - # The two previous cases should handle all known possibilities, but don't crash if - # somethign else sneaks in. - modname = "unknown" - qualname = "unknown" + modname, qualname = _get_name_parts(filterable) self._fqfn = FqFnName(modname, qualname) fntype = filterable.fntype diff --git a/_appmap/importer.py b/_appmap/importer.py index a98d275e..5323bb80 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from collections import namedtuple from collections.abc import MutableSequence -from functools import reduce +from functools import partial, reduce from _appmap import wrapt @@ -147,7 +147,7 @@ def is_member_func(m): if not is_member_func(static_value): continue value = getattr(cls, key) - if value.__module__ != modname: + if (m := getattr(value, "__module__", None)) and (m is None or m != modname): continue functions.append((key, static_value, value)) @@ -202,7 +202,7 @@ def instrument_functions(filterable, selected_functions=None): logger.trace(" functions %s", functions) for fn_name, static_fn, fn in functions: - filterableFn = FilterableFn(filterable, fn.__name__, fn, static_fn) + filterableFn = FilterableFn(filterable, fn_name, fn, static_fn) new_fn = cls.instrument_function(fn_name, filterableFn, selected_functions) if new_fn != fn: fw = wrapt.wrap_function_wrapper(filterable.obj, fn_name, new_fn) diff --git a/_appmap/test/data/properties_class.py b/_appmap/test/data/properties_class.py index ab81f898..e6a3f119 100644 --- a/_appmap/test/data/properties_class.py +++ b/_appmap/test/data/properties_class.py @@ -1,8 +1,12 @@ -from functools import cached_property +from functools import cached_property, partial import operator from typing import NoReturn +def free_read_only(self): + return self._read_only +def free_func(): + return "hello world" class PropertiesClass: def __init__(self): self._read_only = "read only" @@ -54,3 +58,14 @@ def cached_read_only(self): return self._read_only operator_read_only = property(operator.attrgetter("cached_read_only")) + + tastes = {"bacon": "yum"} + + def __getitem__(self, key): + return self.tastes[key] + + taste = property(operator.itemgetter("bacon")) + + free_read_only_prop = property(free_read_only) + + static_partial_method = staticmethod(partial(free_func)) diff --git a/_appmap/test/test_properties.py b/_appmap/test/test_properties.py index 565d0566..7733769d 100644 --- a/_appmap/test/test_properties.py +++ b/_appmap/test/test_properties.py @@ -109,3 +109,40 @@ def test_operator_attrgetter(events): "defined_class": "properties_class.PropertiesClass", "method_id": "operator_read_only (get)", }) + +def test_operator_itemgetter(events): + from properties_class import PropertiesClass + + ec = PropertiesClass() + assert ec.taste == "yum" + assert len(events) == 2 + assert events[0].to_dict() == DictIncluding({ + "event": "call", + # operator.itemgetter.__module__ isn't available before 3.10 + # "defined_class": "operator", + "method_id": "itemgetter (get)", + }) + + +def test_free_function(events): + from properties_class import PropertiesClass + + ec = PropertiesClass() + assert ec.free_read_only_prop == "read only" + assert len(events) == 2 + assert events[0].to_dict() == DictIncluding({ + "event": "call", + "defined_class": "properties_class", + "method_id": "free_read_only (get)", + }) + + +@pytest.mark.xfail( + raises=AssertionError, + reason="needs fix for https://github.com/getappmap/appmap-python/issues/365", +) +def test_functools_partial(events): + from properties_class import PropertiesClass + + PropertiesClass.static_partial_method() + assert len(events) > 0 \ No newline at end of file From 416966bbd2c22b9b21cf493eb1c40b7d56155fd1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 5 Aug 2024 13:25:03 +0000 Subject: [PATCH 076/113] chore(release): 2.1.5 [skip ci] ## [2.1.5](https://github.com/getappmap/appmap-python/compare/v2.1.4...v2.1.5) (2024-08-05) ### Bug Fixes * reenable instrumentation of properties ([7b3119a](https://github.com/getappmap/appmap-python/commit/7b3119a4fdf60e19a28a279d4ba6afe379925e14)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0332f999..595de932 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.1.5](https://github.com/getappmap/appmap-python/compare/v2.1.4...v2.1.5) (2024-08-05) + + +### Bug Fixes + +* reenable instrumentation of properties ([7b3119a](https://github.com/getappmap/appmap-python/commit/7b3119a4fdf60e19a28a279d4ba6afe379925e14)) + ## [2.1.4](https://github.com/getappmap/appmap-python/compare/v2.1.3...v2.1.4) (2024-07-26) diff --git a/pyproject.toml b/pyproject.toml index 2810fe4f..f40d0acb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.4" +version = "2.1.5" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From f23e75848f4696a6892103acef60fb99b3911e20 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 8 Aug 2024 15:31:57 -0400 Subject: [PATCH 077/113] ci: Add planning with Navie AI --- .github/workflows/plan.yml | 27 +++++++++++++++++++++++++++ .gitignore | 4 +++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/plan.yml diff --git a/.github/workflows/plan.yml b/.github/workflows/plan.yml new file mode 100644 index 00000000..f03796ec --- /dev/null +++ b/.github/workflows/plan.yml @@ -0,0 +1,27 @@ +name: Plan issue with Navie + +on: + issues: + types: [opened, edited, reopened, labeled, unlabeled] + +permissions: + contents: read + issues: write + +jobs: + plan: + if: contains(github.event.issue.labels.*.name, 'navie-plan') + runs-on: ubuntu-latest + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Plan with Navie + uses: getappmap/navie-editor/plan@main + with: + issue_id: ${{ github.event.issue.number }} + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index c6f0f746..4473e002 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ htmlcov/ /.tox /node_modules -/ruff.toml \ No newline at end of file +/ruff.toml + +appmap.log From 3561e3b13a1f004b793073a9f9f7e1345b192fa1 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Thu, 1 Aug 2024 08:20:43 -0400 Subject: [PATCH 078/113] fix: make wrapt function objects pickleable Add pickling protocol support to FunctionWrapper and BoundFunctionWrapper. --- _appmap/test/test_recording.py | 43 ++++++++++++++-------- vendor/_appmap/wrapt/wrappers.py | 61 +++++++++++++++++++++----------- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/_appmap/test/test_recording.py b/_appmap/test/test_recording.py index d4656a4d..bf065cad 100644 --- a/_appmap/test/test_recording.py +++ b/_appmap/test/test_recording.py @@ -6,9 +6,10 @@ from shutil import copy, copytree from threading import Thread +import appmap import pytest -import appmap +from _appmap.configuration import Config from _appmap.event import Event from _appmap.recorder import Recorder, ThreadRecorder from _appmap.wrapt import FunctionWrapper @@ -16,6 +17,17 @@ from .normalize import normalize_appmap, remove_line_numbers +def _call_modfunc(q): + r = appmap.Recording() + with r: + f = q.get() + f() + events = r.events + assert len(events) == 2 + assert events[0].event == "call" + assert events[0].method_id == "modfunc" + + @pytest.mark.appmap_enabled @pytest.mark.usefixtures("with_data_dir") class TestRecordingWhenEnabled: @@ -51,7 +63,8 @@ def test_recording_clears(self): ExampleClass, ) - with appmap.Recording(): + rec = appmap.Recording() + with rec: ExampleClass.static_method() # fresh recording shouldn't contain previous traces @@ -117,21 +130,23 @@ def test_can_deepcopy_function(self): f1 = deepcopy(modfunc) f1() - def test_can_pickle(self): - import pickle + def test_can_pickle(self, monkeypatch): + # Make sure subprocesses see whatever config is set for us. + monkeypatch.setenv("APPMAP_CONFIG", str(Config.current._file)) # pylint: disable=protected-access - from example_class import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error - modfunc, + from multiprocessing import Process, Queue + + from example_class import ( + modfunc, # pyright: ignore[reportMissingImports] pylint: disable=import-error ) - rec = appmap.Recording() - with rec: - assert isinstance(modfunc, FunctionWrapper) - f = pickle.loads(pickle.dumps(modfunc)) - f() - evt = rec.events[-2] - assert evt.event == "call" - assert evt.method_id == "modfunc" + assert isinstance(modfunc, FunctionWrapper), "modfunc isn't instrumented?" + + q = Queue() + q.put(modfunc) + p = Process(target=_call_modfunc, args=(q,)) + p.start() + p.join() @pytest.mark.appmap_enabled diff --git a/vendor/_appmap/wrapt/wrappers.py b/vendor/_appmap/wrapt/wrappers.py index 31739da0..a8f61178 100644 --- a/vendor/_appmap/wrapt/wrappers.py +++ b/vendor/_appmap/wrapt/wrappers.py @@ -457,11 +457,8 @@ def __reduce__(self): raise NotImplementedError( 'object proxy must define __reduce_ex__()') - # Return the qualname of the wrapped function instead of a tuple. This allows an instance of - # subclasses to be pickled as the function it wraps. This seems to be adequate for generating - # AppMaps. - def __reduce_ex__(self, protocol): - return self.__wrapped__.__qualname__ + def __reduce_ex__(self): + raise NotImplementedError("object proxy must define __reduce_ex__()") class CallableObjectProxy(ObjectProxy): @@ -508,6 +505,20 @@ def _unpack_self(self, *args): return self.__wrapped__(*_args, **_kwargs) +def _unpickle_functionwrapper(modname, qualname): + """ + Given the module name and qualname of a function, return a FunctionWrapper instance for it. This + simply imports the module, then fetches the appropriate function. Provided AppMap + instrumentation has been configured correctly when unpickling, the attribute for the function + will be a FunctionWrapper. If it hasn't been configured correctly, the attribute will simply be + the original function. (This means the application will function correctly, but no events will + get generated when the function is called.) + """ + _, _, original = resolve_path(modname, qualname) + + return original + + class _FunctionWrapperBase(ObjectProxy): __slots__ = ( "_self_instance", @@ -656,6 +667,20 @@ def __subclasscheck__(self, subclass): else: return issubclass(subclass, self.__wrapped__) + # Implement this here, rather than in ObjectProxy, because _unpickle_functionwrapper will only + # create new instances of subclasses of _FunctionWrapperBase via the agent's import hooks. + def __reduce_ex__(self, _): + modname = self.__wrapped__.__module__ + qualname = getattr(self.__wrapped__, "__qualname__", None) + if qualname is None: + qualname = self.__wrapped__.__name__ + + return ( + _unpickle_functionwrapper, + (modname, qualname), + ) + + class BoundFunctionWrapper(_FunctionWrapperBase): def __new__(cls, *args, **kwargs): @@ -738,24 +763,18 @@ def _unpack_self(self, *args): return self._self_wrapper(self.__wrapped__, instance, args, kwargs) -class FunctionWrapper(_FunctionWrapperBase): + def __getattribute__(self, name): + if name == "__func__": + # The __func__ attribute of a bound method is the unbound method. The corresponding + # attribute of a BoundFunctionWrapper is the associated FunctionWrapper (saved in + # _self_parent when the BFW is created). + return self._self_parent - __bound_function_wrapper__ = BoundFunctionWrapper + return super().__getattribute__(name) - # The code here is pretty complicated (see the comment below), and it's not completely clear to - # me whether it actually keeps any state. If it does, __reduce_ex__ needs to return a tuple so a - # new FunctionWrapper will be created. If it doesn't, then __reduce_ex__ can simply return a - # string, which would cause deepcopy to return the original FunctionWrapper. - # - - # def __reduce_ex__(self, protocol): - # return self.__wrapped__.__qualname__ - - # return FunctionWrapper, ( - # self.__wrapped__, - # self._self_wrapper, - # self._self_enabled, - # ) + +class FunctionWrapper(_FunctionWrapperBase): + __bound_function_wrapper__ = BoundFunctionWrapper def __init__(self, wrapped, wrapper, enabled=None): # What it is we are wrapping here could be anything. We need to From fd0cd2dbf011da5be276f69b183ca941b967273a Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 7 Aug 2024 17:23:40 -0400 Subject: [PATCH 079/113] refactor: lint fixes --- _appmap/event.py | 1 - _appmap/importer.py | 2 +- _appmap/instrument.py | 1 - _appmap/recording.py | 1 + _appmap/test/test_properties.py | 2 +- _appmap/test/test_recording.py | 26 +++++++++++++------------- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index 0bb254cf..f9c7c490 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -6,7 +6,6 @@ from functools import lru_cache, partial from inspect import Parameter, Signature from itertools import chain -import types from .env import Env from .recorder import Recorder diff --git a/_appmap/importer.py b/_appmap/importer.py index 5323bb80..28a6c5a5 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from collections import namedtuple from collections.abc import MutableSequence -from functools import partial, reduce +from functools import reduce from _appmap import wrapt diff --git a/_appmap/instrument.py b/_appmap/instrument.py index a21a2c8d..179e425c 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -119,7 +119,6 @@ def call_instrumented(f, instance, args, kwargs): def instrument(filterable): """return an instrumented function""" logger.debug("hooking %s", filterable.fqname) - fn = filterable.obj make_call_event = event.CallEvent.make(filterable) params = CallEvent.make_params(filterable) diff --git a/_appmap/recording.py b/_appmap/recording.py index dafe9c1f..dadc958b 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -62,6 +62,7 @@ class NoopRecording: def __init__(self, exit_hook=None): self.exit_hook = exit_hook + self.events = [] def start(self): pass diff --git a/_appmap/test/test_properties.py b/_appmap/test/test_properties.py index 7733769d..9377358b 100644 --- a/_appmap/test/test_properties.py +++ b/_appmap/test/test_properties.py @@ -145,4 +145,4 @@ def test_functools_partial(events): from properties_class import PropertiesClass PropertiesClass.static_partial_method() - assert len(events) > 0 \ No newline at end of file + assert len(events) > 0 diff --git a/_appmap/test/test_recording.py b/_appmap/test/test_recording.py index bf065cad..d7749d36 100644 --- a/_appmap/test/test_recording.py +++ b/_appmap/test/test_recording.py @@ -6,9 +6,9 @@ from shutil import copy, copytree from threading import Thread -import appmap import pytest +import appmap from _appmap.configuration import Config from _appmap.event import Event from _appmap.recorder import Recorder, ThreadRecorder @@ -59,9 +59,9 @@ def test_recording_works(self, with_data_dir): ), f"expected path {expected_path}" def test_recording_clears(self): - from example_class import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error - ExampleClass, - ) + # pylint: disable=import-error + from example_class import ExampleClass # pyright: ignore[reportMissingImports] + # pylint: enable=import-error rec = appmap.Recording() with rec: @@ -81,9 +81,9 @@ def test_recording_clears(self): assert rec.events[2].method_id == "instance_method" def test_recording_shallow(self): - from example_class import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error - ExampleClass, - ) + # pylint: disable=import-error + from example_class import ExampleClass # pyright: ignore[reportMissingImports] + # pylint: enable=import-error rec = appmap.Recording() with rec: @@ -95,9 +95,9 @@ def test_recording_shallow(self): assert len(rec.events) == 8 def test_recording_wrapped(self): - from example_class import ( # pyright: ignore[reportMissingImports] pylint: disable=import-error - ExampleClass, - ) + # pylint: disable=import-error + from example_class import ExampleClass # pyright: ignore[reportMissingImports] + # pylint: enable=import-error rec = appmap.Recording() with rec: @@ -136,9 +136,9 @@ def test_can_pickle(self, monkeypatch): from multiprocessing import Process, Queue - from example_class import ( - modfunc, # pyright: ignore[reportMissingImports] pylint: disable=import-error - ) + # pylint: disable=import-error + from example_class import modfunc # pyright: ignore[reportMissingImports] + # pylint: enable=import-error assert isinstance(modfunc, FunctionWrapper), "modfunc isn't instrumented?" From ea0918cf4e952a9e1ab4a48253ec16e4484b78a0 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Mon, 12 Aug 2024 07:40:34 -0400 Subject: [PATCH 080/113] fix: generate AppMap data from django tests Django tests, i.e. those that inherit from django.test.TestCase, can be run with pytest, using the pytest-django plugin. This changes insure that such tests generate test recordings (and don't generate request recordings). --- _appmap/test/data/django/djangoapp/settings.py | 13 +++++++++++++ _appmap/test/data/django/test/test_request.py | 8 ++++++++ _appmap/test/test_django.py | 6 +++--- _appmap/test/test_test_frameworks.py | 14 ++++++++++++++ appmap/pytest.py | 9 ++++++++- 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 _appmap/test/data/django/test/test_request.py diff --git a/_appmap/test/data/django/djangoapp/settings.py b/_appmap/test/data/django/djangoapp/settings.py index 63042670..cf66d2c6 100644 --- a/_appmap/test/data/django/djangoapp/settings.py +++ b/_appmap/test/data/django/djangoapp/settings.py @@ -1,3 +1,9 @@ +from pathlib import Path + + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + # If the SECRET_KEY isn't defined we get the misleading error message # CommandError: You must set settings.ALLOWED_HOSTS if DEBUG is False. SECRET_KEY = "3*+d^_kjnr2gz)4q2m(&&^%$p4fj5dk3%lz4pl3g4m-%6!gf&)" @@ -10,3 +16,10 @@ # Turn off deprecation warning USE_TZ = True + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } +} diff --git a/_appmap/test/data/django/test/test_request.py b/_appmap/test/data/django/test/test_request.py new file mode 100644 index 00000000..ce08d5df --- /dev/null +++ b/_appmap/test/data/django/test/test_request.py @@ -0,0 +1,8 @@ +from django.test import TestCase +from django.test import Client + + +class TestRequest(TestCase): + def test_request_test(self): + resp = self.client.get("/test") + assert resp.status_code == 200 diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index dec17809..5547fb9c 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -198,7 +198,7 @@ def test_enabled(self, pytester): # To really check middleware reset, the tests must run in order, # so disable randomly. result = pytester.runpytest("-svv", "-p", "no:randomly") - result.assert_outcomes(passed=5, failed=0, errors=0) + result.assert_outcomes(passed=6, failed=0, errors=0) # Look for the http_server_request event in test_app's appmap. If # middleware reset is broken, it won't be there. appmap_file = pytester.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json" @@ -212,7 +212,7 @@ def test_enabled(self, pytester): def test_disabled(self, pytester, monkeypatch): monkeypatch.setenv("_APPMAP", "false") result = pytester.runpytest("-svv", "-p", "no:randomly", "-k", "test_request") - result.assert_outcomes(passed=2, failed=0, errors=0) + result.assert_outcomes(passed=3, failed=0, errors=0) assert not (pytester.path / "tmp").exists() def test_disabled_for_process(self, pytester, monkeypatch): @@ -222,7 +222,7 @@ def test_disabled_for_process(self, pytester, monkeypatch): # There are two tests for remote recording. They should both fail, # because process recording should disable remote recording. - result.assert_outcomes(passed=3, failed=2, errors=0) + result.assert_outcomes(passed=4, failed=2, errors=0) assert (pytester.path / "tmp" / "appmap" / "process").exists() assert not (pytester.path / "tmp" / "appmap" / "requests").exists() diff --git a/_appmap/test/test_test_frameworks.py b/_appmap/test/test_test_frameworks.py index 49467a95..a687a22e 100644 --- a/_appmap/test/test_test_frameworks.py +++ b/_appmap/test/test_test_frameworks.py @@ -122,6 +122,20 @@ def test_enabled_no_test_cases(self, testdir, monkeypatch): verify_expected_appmap(testdir, f"-numpy{numpy_version.major}-no-test-cases") verify_expected_metadata(testdir) +@pytest.mark.example_dir("django") +def test_pytest_django(testdir): + result = testdir.runpytest("-svv", "-k", "test_request_test") + result.assert_outcomes(passed=1) + # django.test.TestCase is a subclass of unittest.TestCase, so recorder type is unittest + assert ( + testdir.path + / "tmp" + / "appmap" + / "unittest" + / "test_test_request_TestRequest_test_request_test.appmap.json" + ).exists() + assert not (testdir.path / "tmp" / "appmap" / "requests").exists() + @pytest.mark.example_dir("trial") class TestPytestRunnerTrial(_TestTestRunner): diff --git a/appmap/pytest.py b/appmap/pytest.py index 67d3cb4d..61b655f4 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -1,6 +1,13 @@ from importlib.metadata import version import pytest +try: + from pytest_django.django_compat import is_django_unittest +except ImportError: + + def is_django_unittest(item): + return False + from _appmap import noappmap, testing_framework, wrapt from _appmap.env import Env @@ -51,7 +58,7 @@ def pytest_runtest_call(item): # running the test case. (This nesting of function calls is # verified by the expected appmap in the test for a unittest # TestCase run by pytest.) - if hasattr(item, "_testcase"): + if hasattr(item, "_testcase") and not is_django_unittest(item): setattr( item._testcase, # pylint: disable=protected-access "_appmap_pytest_recording", From ed05e077272b7cebd2d6098d9011f54848164e39 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 13 Aug 2024 21:18:34 +0000 Subject: [PATCH 081/113] chore(release): 2.1.6 [skip ci] ## [2.1.6](https://github.com/getappmap/appmap-python/compare/v2.1.5...v2.1.6) (2024-08-13) ### Bug Fixes * generate AppMap data from django tests ([ea0918c](https://github.com/getappmap/appmap-python/commit/ea0918cf4e952a9e1ab4a48253ec16e4484b78a0)) * make wrapt function objects pickleable ([3561e3b](https://github.com/getappmap/appmap-python/commit/3561e3b13a1f004b793073a9f9f7e1345b192fa1)) --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 595de932..2825fce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [2.1.6](https://github.com/getappmap/appmap-python/compare/v2.1.5...v2.1.6) (2024-08-13) + + +### Bug Fixes + +* generate AppMap data from django tests ([ea0918c](https://github.com/getappmap/appmap-python/commit/ea0918cf4e952a9e1ab4a48253ec16e4484b78a0)) +* make wrapt function objects pickleable ([3561e3b](https://github.com/getappmap/appmap-python/commit/3561e3b13a1f004b793073a9f9f7e1345b192fa1)) + ## [2.1.5](https://github.com/getappmap/appmap-python/compare/v2.1.4...v2.1.5) (2024-08-05) diff --git a/pyproject.toml b/pyproject.toml index f40d0acb..6f2879ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.5" +version = "2.1.6" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 91f136445c16bcb55912549d8512bea7732d8218 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 14 Aug 2024 06:18:21 -0400 Subject: [PATCH 082/113] fix: disable parameter rendering by default By default, parameters will no longer be rendered as strings by default. To see the values of parameters, APPMAP_DISPLAY_PARAMS must be set to "true". --- README.md | 7 +++---- _appmap/env.py | 16 ++++++++-------- _appmap/test/conftest.py | 2 ++ _appmap/test/test_events.py | 3 +-- appmap/__init__.py | 7 ++++++- tox.ini | 6 ++---- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index cf878433..a73aa470 100644 --- a/README.md +++ b/README.md @@ -90,16 +90,15 @@ Note that you must install the dependencies contained in [requirements-dev.txt](requirements-dev.txt) before running tests. See the explanation in [pyproject.toml](pyproject.toml) for details. -Additionally, the tests currently require that you set `APPMAP=true`. You can -either run `pytest` with `appmap-python` (see [tox.ini](tox.ini)), or you can explicitly -set the environment variable. +Additionally, the tests currently require that you set `APPMAP=true` and +`APPMAP_DISPLAY_PARAMS=true`. [pytest](https://docs.pytest.org/en/stable/) for testing: ``` % cd appmap-python % pip install -r requirements-test.txt -% poetry run pytest +% APPMAP=true APPMAP_DISPLAY_PARAMS=true poetry run pytest ``` ### tox diff --git a/_appmap/env.py b/_appmap/env.py index 29c182cd..639e23e8 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -39,13 +39,13 @@ def __init__(self, env=None, cwd=None): self.log_file_creation_failed = False self._configure_logging() - # This uses _APPMAP, rather than APPMAP, to control whether instrumentation is enabled. The - # tests use this split to make it easier to control recording. - enabled = self._env.get("_APPMAP", "false") - self._enabled = enabled is None or enabled.lower() != "false" - - self._root_dir = str(self._cwd) + "/" - self._root_dir_len = len(self._root_dir) + # This uses the underscore-decorated, rather than the undecorated variants, to control + # whether these settings are enabled. The tests use this split to make it easier to control + # them. + enabled = self._env.get("_APPMAP", None) + self._enabled = enabled is not None and enabled.lower() != "false" + display_params = self._env.get("_APPMAP_DISPLAY_PARAMS", None) + self._display_params = display_params is not None and display_params.lower() != "false" logger = logging.getLogger(__name__) # The user shouldn't set APPMAP_OUTPUT_DIR, but some tests depend on being able to use it. @@ -131,7 +131,7 @@ def is_appmap_repo(self): @property def display_params(self): - return self.get("APPMAP_DISPLAY_PARAMS", "true").lower() == "true" + return self._display_params def getLogger(self, name) -> trace_logger.TraceLogger: return cast(trace_logger.TraceLogger, logging.getLogger(name)) diff --git a/_appmap/test/conftest.py b/_appmap/test/conftest.py index 211688e1..4c4358b3 100644 --- a/_appmap/test/conftest.py +++ b/_appmap/test/conftest.py @@ -68,6 +68,8 @@ def pytest_runtest_setup(item): elif appmap_enabled is None: env.pop("_APPMAP", None) + env["_APPMAP_DISPLAY_PARAMS"] = env.get("APPMAP_DISPLAY_PARAMS", "true") + _appmap.initialize(env=env) # pylint: disable=protected-access # Some tests want yaml instrumented, others don't. diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index fba942fd..bef98577 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -9,7 +9,6 @@ import pytest import appmap -from _appmap.env import Env from _appmap.event import _EventIds @@ -84,8 +83,8 @@ def test_when_both_raise(self, mocker): actual_value = r.events[0].parameters[0]["value"] assert re.fullmatch(expected_re, actual_value) + @pytest.mark.appmap_enabled(env={"APPMAP_DISPLAY_PARAMS": "false"}) def test_when_display_disabled(self, mocker): - Env.current.set("APPMAP_DISPLAY_PARAMS", "false") r = appmap.Recording() with r: from example_class import ExampleClass # pylint: disable=import-outside-toplevel diff --git a/appmap/__init__.py b/appmap/__init__.py index b5750595..a60e1af4 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -11,8 +11,11 @@ _recording_exported = False if _enabled is None or _enabled.upper() == "TRUE": if _enabled is not None: - # Use setdefault so tests can manage _APPMAP as necessary + # Use setdefault so tests can manage settings as necessary os.environ.setdefault("_APPMAP", _enabled) + _display_params = os.environ.get("APPMAP_DISPLAY_PARAMS", "false") + os.environ.setdefault("_APPMAP_DISPLAY_PARAMS", _display_params) + from _appmap import generation # noqa: F401 from _appmap.env import Env # noqa: F401 from _appmap.importer import instrument_module # noqa: F401 @@ -52,8 +55,10 @@ def enabled(): return Env.current.enabled else: os.environ.pop("_APPMAP", None) + os.environ.pop("_APPMAP_DISPLAY_PARAMS", None) else: os.environ.setdefault("_APPMAP", "false") + os.environ.setdefault("_APPMAP_DISPLAY_PARAMS", "false") if not _recording_exported: # Client code that imports appmap.Recording should run correctly diff --git a/tox.ini b/tox.ini index d4e47a9f..383107f5 100644 --- a/tox.ini +++ b/tox.ini @@ -13,10 +13,8 @@ deps= [testenv] passenv = PYTEST_XDIST_AUTO_NUM_WORKERS -allowlist_externals = - env - bash - +setenv = + APPMAP_DISPLAY_PARAMS=true deps= poetry web: {[web-deps]deps} From 5122d7659722663cecf4d203883a48d136e19618 Mon Sep 17 00:00:00 2001 From: Alan Potter Date: Wed, 14 Aug 2024 06:25:00 -0400 Subject: [PATCH 083/113] fix: cache Env.root_dir, is_appmap_repo Cache the value computed for root_dir. Note that there are many tests that expect to set root dir (by passing cwd to __init__), which show that the cached value is managed correctly. Also, cache is_appmap_repo. Its value is never reset. --- _appmap/env.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_appmap/env.py b/_appmap/env.py index 639e23e8..aa61b233 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -1,5 +1,6 @@ """Initialize from the environment""" +from functools import cached_property import logging import logging.config import os @@ -69,13 +70,13 @@ def get(self, name, default=None): def delete(self, name): del self._env[name] - @property + @cached_property def root_dir(self): - return self._root_dir + return str(self._cwd) + "/" - @property + @cached_property def root_dir_len(self): - return self._root_dir_len + return len(self.root_dir) @property def output_dir(self): @@ -123,7 +124,7 @@ def disabled(self, recording_method: str): if value: self.set(key, value) - @property + @cached_property def is_appmap_repo(self): return os.path.exists("appmap/__init__.py") and os.path.exists( "_appmap/__init__.py" From 8a602f4ba13965ca07204725e0a6a16ab2f9f886 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 15 Aug 2024 01:33:44 +0000 Subject: [PATCH 084/113] chore(release): 2.1.7 [skip ci] ## [2.1.7](https://github.com/getappmap/appmap-python/compare/v2.1.6...v2.1.7) (2024-08-15) ### Bug Fixes * cache Env.root_dir, is_appmap_repo ([5122d76](https://github.com/getappmap/appmap-python/commit/5122d7659722663cecf4d203883a48d136e19618)) * disable parameter rendering by default ([91f1364](https://github.com/getappmap/appmap-python/commit/91f136445c16bcb55912549d8512bea7732d8218)) --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2825fce6..7f1ebcc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [2.1.7](https://github.com/getappmap/appmap-python/compare/v2.1.6...v2.1.7) (2024-08-15) + + +### Bug Fixes + +* cache Env.root_dir, is_appmap_repo ([5122d76](https://github.com/getappmap/appmap-python/commit/5122d7659722663cecf4d203883a48d136e19618)) +* disable parameter rendering by default ([91f1364](https://github.com/getappmap/appmap-python/commit/91f136445c16bcb55912549d8512bea7732d8218)) + ## [2.1.6](https://github.com/getappmap/appmap-python/compare/v2.1.5...v2.1.6) (2024-08-13) diff --git a/pyproject.toml b/pyproject.toml index 6f2879ba..be7df41a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.6" +version = "2.1.7" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 0347af18d69f5f3be6a8c0789400bacd3fff42b9 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 12 Nov 2024 16:50:07 -0500 Subject: [PATCH 085/113] fix: Prevent process recordings from clobbering one another AppMap data output via process recordings now contains the process ID in the file name to prevent multiple processes from quitting within the same second and overwriting each others data. --- _appmap/recording.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_appmap/recording.py b/_appmap/recording.py index dadc958b..8513cbff 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -115,7 +115,9 @@ def save_at_exit(): nonlocal r r.stop() now = datetime.now(timezone.utc) - appmap_name = now.isoformat(timespec="seconds").replace("+00:00", "Z") + iso_time = now.isoformat(timespec="seconds").replace("+00:00", "Z") + process_id = os.getpid() + appmap_name = f"{iso_time}_{process_id}" recorder_type = "process" metadata = { "name": appmap_name, From 6c0bd35b584268420799da7eeccbab5a4d8d9216 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 13 Nov 2024 14:59:20 +0000 Subject: [PATCH 086/113] chore(release): 2.1.8 [skip ci] ## [2.1.8](https://github.com/getappmap/appmap-python/compare/v2.1.7...v2.1.8) (2024-11-13) ### Bug Fixes * Prevent process recordings from clobbering one another ([0347af1](https://github.com/getappmap/appmap-python/commit/0347af18d69f5f3be6a8c0789400bacd3fff42b9)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1ebcc4..e9d876d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.1.8](https://github.com/getappmap/appmap-python/compare/v2.1.7...v2.1.8) (2024-11-13) + + +### Bug Fixes + +* Prevent process recordings from clobbering one another ([0347af1](https://github.com/getappmap/appmap-python/commit/0347af18d69f5f3be6a8c0789400bacd3fff42b9)) + ## [2.1.7](https://github.com/getappmap/appmap-python/compare/v2.1.6...v2.1.7) (2024-08-15) diff --git a/pyproject.toml b/pyproject.toml index be7df41a..e0b54baf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.7" +version = "2.1.8" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From 3472d585a0a0453d1db3e31dfe4a0db3b9d7acb1 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Tue, 17 Dec 2024 10:33:55 -0500 Subject: [PATCH 087/113] ci: Update tox configuration to run Django5 Attaching notes on the run, including test errors. We should also see these in CI --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 383107f5..aacd1379 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = true # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{8,9,10,11,12}-{web,django3,flask2,sqlalchemy1},lint +envlist = py3{8,9,10,11,12}-{web,django3,django4,django5,flask2,sqlalchemy1},lint [web-deps] deps= @@ -22,12 +22,16 @@ deps= py3{9,10,11,12}: numpy >=2 flask2: Flask >= 2.0, <3.0 django3: Django >=3.2, <4.0 + django4: Django >=4.0, <5.0 + django5: Django >=5.0, <6.0 sqlalchemy1: sqlalchemy >=1.4.11, <2.0 commands = poetry install -v web: poetry run appmap-python {posargs:pytest -n logical} django3: poetry run appmap-python pytest -n logical _appmap/test/test_django.py + django4: poetry run appmap-python pytest -n logical _appmap/test/test_django.py + django5: poetry run appmap-python pytest -n logical _appmap/test/test_django.py flask2: poetry run appmap-python pytest -n logical _appmap/test/test_flask.py sqlalchemy1: poetry run appmap-python pytest -n logical _appmap/test/test_sqlalchemy.py From e9c404404f3b213f0304a73bc47b0bf98fbfc47b Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Wed, 18 Dec 2024 09:25:55 -0500 Subject: [PATCH 088/113] ci: Limit django5 tests to Python >= 3.10 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index aacd1379..a92da61a 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = true # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{8,9,10,11,12}-{web,django3,django4,django5,flask2,sqlalchemy1},lint +envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1},lint [web-deps] deps= From b783958541451d3c2db89c3ea6b7e04825d3fef9 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 13:44:43 +0200 Subject: [PATCH 089/113] CI on Github: enable main workflow on ci/* branches for debugging --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d2fd055..4de8a51e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,9 +1,11 @@ name: Build on: - pull_request: + pull_request: # to master schedule: - cron: "0 0 * * 0" - + push: + branches: # CI debugging + - "ci/**" jobs: build: runs-on: ${{ matrix.os }} From 9f520804f3c1239289160c3419ec9f1b6d01fcb7 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 16:43:24 +0200 Subject: [PATCH 090/113] CI on Github: separate linter/test, shared setup * tox.ini: tox-gh-actions integration * tox.ini: lint removed from envlist, default scope is test-only * workflows: test job added (matrixed) * workflows: lint job is separated, no-matrix, non-blocking on failure * workflows: reusable env lifting steps factored out into ./github/actions --- .github/actions/setup/action.yml | 46 ++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 43 +++++++++++++++++------------ tox.ini | 15 +++++++++-- 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..48e761d3 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,46 @@ +name: Setup base (python, pip cache, tox) +inputs: + python: + description: "Python version to use" + required: true + type: string +runs: + using: "composite" + steps: + - name: pip cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + key: ${{ runner.os }}-pip-${{ inputs.python }} + + - name: Cargo cache + uses: actions/cache/@v4 + with: + path: "~/.cargo" + key: ${{ runner.os }}-cargo + + - name: Poetry cache + uses: actions/cache/@v4 + with: + path: "~/.cache/pypoetry" + key: ${{ runner.os }}-poetry-${{ inputs.python }} + restore-keys: | + ${{ runner.os }}-poetry- + + - uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python }} + + - name: upgrade pip and install tox + shell: bash + run: | + python -m pip -q install --upgrade pip "setuptools==65.6.2" + pip -q install "tox<4" tox-gh-actions + + - name: install Rust and Poetry + shell: bash + run : | + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal + source "$HOME/.cargo/env" + pip -q install poetry>=1.2.0 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4de8a51e..a8d80463 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,27 +7,36 @@ on: branches: # CI debugging - "ci/**" jobs: - build: - runs-on: ${{ matrix.os }} + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + with: + python: 3.12 + - name: Lint + id: lint + run: tox -e lint + continue-on-error: true + - name: Emit warning if lint failed + if: ${{ steps.lint.outcome != 'success' }} + run: echo "::warning::Linter failure suppressed (continue-on-error=true)" + test: strategy: fail-fast: false matrix: os: [ ubuntu-latest ] - python: ["3.12"] - include: - - python: "3.12" - tox_env: "lint" + python: + - "3.12" + #- "3.11" + #- "3.10" + #- "3.9.14" + #- "3.8" + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup with: - python-version: ${{ matrix.python }} - - name: Install tox - run: | - python -m pip install --upgrade pip setuptools - pip install tox + python: ${{ matrix.python }} - name: Test - run: | - tox -e ${{ matrix.tox_env }} - + run: tox diff --git a/tox.ini b/tox.ini index a92da61a..be81c9d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,19 @@ [tox] isolated_build = true + # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1},lint + +# Default envlist is only for matrix testing. Linter and vendoring should be called explicitly +envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1} + +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 [web-deps] deps= @@ -54,4 +65,4 @@ deps = vendoring commands = poetry run vendoring {posargs:sync} # We don't need the .pyi files vendoring generates - python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' \ No newline at end of file + python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' From bd92ac8dc463855f3f13cc91b7833c6bd03c4afe Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 20:30:21 +0200 Subject: [PATCH 091/113] CI on Github: smoketest --- .github/actions/dockerhub-login/action.yml | 8 ++++++++ .github/actions/setup/action.yml | 5 +++++ .github/workflows/main.yml | 20 ++++++++++++++++++-- ci/run_tests.sh | 4 +++- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .github/actions/dockerhub-login/action.yml diff --git a/.github/actions/dockerhub-login/action.yml b/.github/actions/dockerhub-login/action.yml new file mode 100644 index 00000000..6b519ce0 --- /dev/null +++ b/.github/actions/dockerhub-login/action.yml @@ -0,0 +1,8 @@ +name: login to Dockerhub (to prevent image pull trottling) +runs: + using: composite + steps: + - name: docker login + run: | + docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" + shell: bash diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 48e761d3..daba8580 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,6 +4,10 @@ inputs: description: "Python version to use" required: true type: string + default: 3.12 +outputs: + 'python-version': + value: ${{ steps.python.outputs.python-version }} runs: using: "composite" steps: @@ -29,6 +33,7 @@ runs: ${{ runner.os }}-poetry- - uses: actions/setup-python@v6 + id: python with: python-version: ${{ inputs.python }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a8d80463..f3cbef94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,8 +12,6 @@ jobs: steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup - with: - python: 3.12 - name: Lint id: lint run: tox -e lint @@ -40,3 +38,21 @@ jobs: python: ${{ matrix.python }} - name: Test run: tox + smoketest: + runs-on: ubuntu-latest + needs: [ 'lint','test' ] + steps: + - uses: actions/checkout@v5 + - name: dockerhub login (for seamless docker pulling) + uses: ./.github/actions/dockerhub-login + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + continue-on-error: true + - id: setup + uses: ./.github/actions/setup + + - run: poetry build + - run: ci/run_tests.sh + env: + SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} diff --git a/ci/run_tests.sh b/ci/run_tests.sh index c78d5415..977d9f17 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} + set -x t=$([ -t 0 ] && echo 't') docker run -q -i${t} --rm\ @@ -7,4 +9,4 @@ docker run -q -i${t} --rm\ -v $PWD/ci:/ci\ -w /tmp\ -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ - python:3.11 bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" + $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" From 9fdbf146277df1e441e8d2d87156284b4fbe2acf Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Wed, 1 Oct 2025 01:28:35 +0200 Subject: [PATCH 092/113] CI on Github: release --- .github/actions/dockerhub-login/action.yml | 2 +- .../actions/setup-semantic-release/action.yml | 18 +++++++++++++++ .github/workflows/main.yml | 23 +++++++++++++++++++ .releaserc.yml | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 .github/actions/setup-semantic-release/action.yml diff --git a/.github/actions/dockerhub-login/action.yml b/.github/actions/dockerhub-login/action.yml index 6b519ce0..6e07493b 100644 --- a/.github/actions/dockerhub-login/action.yml +++ b/.github/actions/dockerhub-login/action.yml @@ -4,5 +4,5 @@ runs: steps: - name: docker login run: | - docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" || echo "::warning::docker-login failed, ignoring" shell: bash diff --git a/.github/actions/setup-semantic-release/action.yml b/.github/actions/setup-semantic-release/action.yml new file mode 100644 index 00000000..455ef854 --- /dev/null +++ b/.github/actions/setup-semantic-release/action.yml @@ -0,0 +1,18 @@ +name: setup semantic-release with plugins +runs: + using: composite + steps: + - uses: actions/setup-node@v5 + id: setup-node + - uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-npm-${{ steps.setup-node.node-version }} + - shell: bash + run: | + npm i -g \ + semantic-release \ + @semantic-release/exec \ + @semantic-release/git \ + @semantic-release/changelog \ + @google/semantic-release-replace-plugin diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3cbef94..a5eff90e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,7 @@ jobs: - name: Test run: tox smoketest: + if: false runs-on: ubuntu-latest needs: [ 'lint','test' ] steps: @@ -56,3 +57,25 @@ jobs: - run: ci/run_tests.sh env: SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} + release: + if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') ) + #needs: ['smoketest','lint','test'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-semantic-release # node+semantic-release + - uses: ./.github/actions/setup # poetry + - name: configure poetry repos + run: | + poetry config repositories.pypi https://upload.pypi.org/legacy/ + poetry config repositories.testpypi https://test.pypi.org/legacy/ + - run: semantic-release --branches ${{ github.ref_name }} + env: + GIT_AUTHOR_NAME: appland-release + GIT_AUTHOR_EMAIL: release@app.land + GIT_COMMITTER_NAME: appland-release + GIT_COMMITTER_EMAIL: release@app.land + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} + POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} diff --git a/.releaserc.yml b/.releaserc.yml index 22f1d3c7..b82a7e45 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -18,4 +18,4 @@ plugins: - CHANGELOG.md - pyproject.toml - - '@semantic-release/exec' - - publishCmd: poetry publish --build + - publishCmd: "poetry publish --build -r <%= process.env.PYPI_PUBLISH_REPO ? process.env.PYPI_PUBLISH_REPO : 'pypi' %>" From 503c6d5e7c44b607c12f81fd7b6ac8692bfcbc67 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Wed, 1 Oct 2025 04:45:00 +0200 Subject: [PATCH 093/113] CI on Github: final rules, disabled Travis, remarks --- .github/workflows/main.yml | 44 +++++++++++++++++++++++++------------- .travis.yml | 5 ++++- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5eff90e..c995e4ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,20 @@ -name: Build +name: CI & Release + +# Requires secrets: +# DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD -- optional (for seamless pulls) +# GITHUB_TOKEN -- implicitly injected +# POETRY_PYPI_TOKEN_PYPI -- Pypi token, required for release +# POETRY_PYPI_TOKEN_TESTPYPI -- testpypi token, for CI debugging + on: - pull_request: # to master - schedule: - - cron: "0 0 * * 0" - push: - branches: # CI debugging - - "ci/**" + # NOTE: Release job includes extra guardrails (see job condition below) + # Without them, only linting and testing run + pull_request: + push: + branches: + - "master" + - "ci/**" # CI debugging branches + #- "v*.*" # optional: support for independent major release branches (e.g., 2.x, 3.x) jobs: lint: runs-on: ubuntu-latest @@ -26,10 +35,10 @@ jobs: os: [ ubuntu-latest ] python: - "3.12" - #- "3.11" - #- "3.10" - #- "3.9.14" - #- "3.8" + - "3.11" + - "3.10" + - "3.9.14" + - "3.8" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 @@ -39,7 +48,6 @@ jobs: - name: Test run: tox smoketest: - if: false runs-on: ubuntu-latest needs: [ 'lint','test' ] steps: @@ -58,8 +66,14 @@ jobs: env: SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} release: - if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') ) - #needs: ['smoketest','lint','test'] + + # NOTE: release is allowed only on selected branches + # NOTE: publishing target (prod or testpypi) is derived as PYPI_PUBLISH_REPO below + # full example: + # if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') || startsWith(github.ref_name,'v') ) + if: github.ref_name == 'master' + + needs: ['smoketest','lint','test'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -76,6 +90,6 @@ jobs: GIT_COMMITTER_NAME: appland-release GIT_COMMITTER_EMAIL: release@app.land GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} + PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} diff --git a/.travis.yml b/.travis.yml index d54311ce..78a77e56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ python: - "3.9.14" - "3.8" +# Travis CI disabled in October 2025; this file is kept temporary for reference and possibility of rollback; can be deleted safely after migration +if: false + # https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807 -if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ +#if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ before_install: | curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal From 2e801eef8c821dd24e6df06690b73d43d10fb682 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 11 Nov 2025 21:20:14 +0100 Subject: [PATCH 094/113] chore(ci): refactor workflows, enable github releases and define branch policies under .releaserc, publish to pypi via trusted publishing mechanism --- .github/actions/refetch-artifacts/action.yml | 17 ++ .../actions/setup-semantic-release/action.yml | 1 + .github/workflows/lint-and-test.yml | 41 +++++ .github/workflows/main.yml | 95 ------------ .github/workflows/release.yml | 146 ++++++++++++++++++ .releaserc.yml | 30 +++- ci/run_tests.sh | 12 -- ci/scripts/build_with_poetry.sh | 19 +++ ...tifacts_if_distribution_name_is_altered.sh | 29 ++++ ci/scripts/run_tests.sh | 16 ++ ci/{ => tests/data}/readonly-mount-appmap.log | 0 ci/{ => tests}/smoketest.sh | 10 +- ci/{ => tests}/test_pipenv.sh | 0 ci/{ => tests}/test_poetry.sh | 0 14 files changed, 306 insertions(+), 110 deletions(-) create mode 100644 .github/actions/refetch-artifacts/action.yml create mode 100644 .github/workflows/lint-and-test.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/release.yml delete mode 100755 ci/run_tests.sh create mode 100755 ci/scripts/build_with_poetry.sh create mode 100755 ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh create mode 100755 ci/scripts/run_tests.sh rename ci/{ => tests/data}/readonly-mount-appmap.log (100%) rename ci/{ => tests}/smoketest.sh (88%) rename ci/{ => tests}/test_pipenv.sh (100%) rename ci/{ => tests}/test_poetry.sh (100%) diff --git a/.github/actions/refetch-artifacts/action.yml b/.github/actions/refetch-artifacts/action.yml new file mode 100644 index 00000000..82f52567 --- /dev/null +++ b/.github/actions/refetch-artifacts/action.yml @@ -0,0 +1,17 @@ +name: Refetch artifacts +runs: + using: "composite" + steps: + - name: download wheel.zip + uses: actions/download-artifact@v4 + with: + name: wheel + path: ./dist + - name: download sdist.zip + uses: actions/download-artifact@v4 + with: + name: sdist + path: ./dist + - name: inspect + shell: bash + run: ls dist/ diff --git a/.github/actions/setup-semantic-release/action.yml b/.github/actions/setup-semantic-release/action.yml index 455ef854..ceb4e22f 100644 --- a/.github/actions/setup-semantic-release/action.yml +++ b/.github/actions/setup-semantic-release/action.yml @@ -14,5 +14,6 @@ runs: semantic-release \ @semantic-release/exec \ @semantic-release/git \ + @semantic-release/github \ @semantic-release/changelog \ @google/semantic-release-replace-plugin diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml new file mode 100644 index 00000000..e2fe0415 --- /dev/null +++ b/.github/workflows/lint-and-test.yml @@ -0,0 +1,41 @@ +name: Lint and test +on: + pull_request: + push: + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- 'feature/**' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + - name: Lint + id: lint + run: tox -e lint + continue-on-error: true + - name: Emit warning if lint failed + if: ${{ steps.lint.outcome != 'success' }} + run: echo "::warning::Linter failure suppressed (continue-on-error=true)" + test: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + python: + - "3.12" + - "3.11" + - "3.10" + - "3.9.14" + - "3.8" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + with: + python: ${{ matrix.python }} + - name: Test + run: tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index c995e4ed..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: CI & Release - -# Requires secrets: -# DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD -- optional (for seamless pulls) -# GITHUB_TOKEN -- implicitly injected -# POETRY_PYPI_TOKEN_PYPI -- Pypi token, required for release -# POETRY_PYPI_TOKEN_TESTPYPI -- testpypi token, for CI debugging - -on: - # NOTE: Release job includes extra guardrails (see job condition below) - # Without them, only linting and testing run - pull_request: - push: - branches: - - "master" - - "ci/**" # CI debugging branches - #- "v*.*" # optional: support for independent major release branches (e.g., 2.x, 3.x) -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup - - name: Lint - id: lint - run: tox -e lint - continue-on-error: true - - name: Emit warning if lint failed - if: ${{ steps.lint.outcome != 'success' }} - run: echo "::warning::Linter failure suppressed (continue-on-error=true)" - test: - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python: - - "3.12" - - "3.11" - - "3.10" - - "3.9.14" - - "3.8" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup - with: - python: ${{ matrix.python }} - - name: Test - run: tox - smoketest: - runs-on: ubuntu-latest - needs: [ 'lint','test' ] - steps: - - uses: actions/checkout@v5 - - name: dockerhub login (for seamless docker pulling) - uses: ./.github/actions/dockerhub-login - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} - continue-on-error: true - - id: setup - uses: ./.github/actions/setup - - - run: poetry build - - run: ci/run_tests.sh - env: - SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} - release: - - # NOTE: release is allowed only on selected branches - # NOTE: publishing target (prod or testpypi) is derived as PYPI_PUBLISH_REPO below - # full example: - # if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') || startsWith(github.ref_name,'v') ) - if: github.ref_name == 'master' - - needs: ['smoketest','lint','test'] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup-semantic-release # node+semantic-release - - uses: ./.github/actions/setup # poetry - - name: configure poetry repos - run: | - poetry config repositories.pypi https://upload.pypi.org/legacy/ - poetry config repositories.testpypi https://test.pypi.org/legacy/ - - run: semantic-release --branches ${{ github.ref_name }} - env: - GIT_AUTHOR_NAME: appland-release - GIT_AUTHOR_EMAIL: release@app.land - GIT_COMMITTER_NAME: appland-release - GIT_COMMITTER_EMAIL: release@app.land - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} - POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} - PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..2928b1af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,146 @@ +name: Release + +on: + workflow_run: # would only fire after file is merged to master + workflows: ["Lint and test"] + types: + - completed + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- develop # can emit -dev releases but we do not want to + workflow_dispatch: + inputs: + dry_run: + description: "Run in dry-run mode (no publish)" + required: false + default: "true" + push: # only temporary, until this file lands on master (see above) + branches: + - 'ci/**' + +# MUSTHAVE: Trusted publisher access for both repos. +# NOTE: according to docs, 'test' repo accounts are ephemeral and can be wiped at any time +# NOTE: 'test' accs are not that ephmeperal -- losing access to sandbox account (2FA issue) effectively locked us out of project; good test for workarounds though +# NOTE: as a part of regaining-control scenario we may use distinct project names in pyroject.toml (e.g. appmap-dev, appmap-ng) +env: + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + pypi_project: appmap + #testpypi_project: appmap-dev # workaround for lost-access scenario + testpypi_project: appmapcitest + +jobs: + + setup: + runs-on: ubuntu-latest + outputs: + distribution_name: ${{ steps.configure.outputs.distribution_name }} + publish_to: ${{ steps.configure.outputs.publish_to }} + publish_env: ${{ steps.configure.outputs.publish_env }} + steps: + - id: configure + shell: bash + run: | + case "${{ github.ref_name }}" in + ci/*) + echo "publish_env=testpypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + ;; + master) + echo "publish_env=pypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + *) + echo "publish_env=SKIP" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + esac + + release: + runs-on: ubuntu-latest + needs: setup + if: github.event_name == 'workflow_dispatch' || (github.event_name=='push' && startsWith(github.ref_name,'ci/') ) || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'ci/') ) ) + permissions: + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-semantic-release # node+semantic-release + - uses: ./.github/actions/setup # poetry + - id: semantic-release # branch policies defined in .releaserc + env: + GIT_AUTHOR_NAME: appland-release + GIT_AUTHOR_EMAIL: release@app.land + GIT_COMMITTER_NAME: appland-release + GIT_COMMITTER_EMAIL: release@app.land + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + run: | + if [ "$DRY_RUN" = "true" ]; then + semantic-release --dry-run + else + semantic-release + fi + + - name: Upload wheel + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + - name: Upload sdist + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + outputs: # not reused in fact + release_tag: ${{ steps.semantic-release.outputs.next_release_tag }} + + smoketest: + runs-on: ubuntu-latest + needs: ['setup', 'release'] + if: github.event.inputs.dry_run!='true' + continue-on-error: ${{ needs.setup.outputs.distribution_name!='appmap' }} # altered names won't work anyway + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + - name: dockerhub login (for seamless docker pulling) + uses: ./.github/actions/dockerhub-login + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + continue-on-error: true + - run: ci/scripts/run_tests.sh + env: + SMOKETEST_DOCKER_IMAGE: python:3.12-slim + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + + # as a workaround to ownership issues (lost access to project) + publish: + name: publish package on PyPI + needs: ['setup', 'release','smoketest'] + if: (( github.event.inputs.dry_run != 'true' ) && ( (needs.setup.outputs.publish_env == 'pypi') || (needs.setup.outputs.publish_env == 'testpypi') ) ) + runs-on: ubuntu-latest + environment: + name: ${{ needs.setup.outputs.publish_env }} + url: ${{ needs.setup.outputs.publish_to }} + permissions: + id-token: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + + - name: Publish to PyPI + if: needs.setup.outputs.publish_env=='pypi' + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: Publish to TestPyPI + if: needs.setup.outputs.publish_env=='testpypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ # trailing slash matters! diff --git a/.releaserc.yml b/.releaserc.yml index b82a7e45..b5efcfc6 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -1,3 +1,18 @@ +# Allowed number of prerelease rules: 1..3 +# While semantic-release allows globs, they must be combined with `prerelease: true` and suffix is derived from name than. It conflicts with PEP440 +# PEP440 version rules (not compatible with SemVer): [N!]N(.N)*[{a|b|rc}N][.postN][.devN] +# Consequences: +# - prerelease branches must be explicitly specified, no asterisks +# - prerelease parameter should be one of: a,b,rc,dev,post +# - translation from SemVer prerelease notation to PEP440 is tone in 'replacements' section +branches: # only branches listed here will create releases + - master + - name: ci/trusted_publishing_test + prerelease: dev + #- name: develop + # prerelease: dev + #- name: feature/* + # prerelease: true # will use branch name as suffix plugins: - '@semantic-release/commit-analyzer' - '@semantic-release/release-notes-generator' @@ -13,9 +28,22 @@ plugins: hasChanged: true numMatches: 1 numReplacements: 1 +- - '@google/semantic-release-replace-plugin' # optional SemVer -> PEP440 coercion + - replacements: + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-dev.1" -> "1.2.3.dev1") + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(dev|post)\\.(\\d+)"' + to: 'version = "\\1.\\2\\3"' + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-rc.10" -> "1.2.3rc10" ) + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(a|b|rc)\\.(\\d+)"' + to: 'version = "\\1\\2\\3"' - - '@semantic-release/git' - assets: - CHANGELOG.md - pyproject.toml - - '@semantic-release/exec' - - publishCmd: "poetry publish --build -r <%= process.env.PYPI_PUBLISH_REPO ? process.env.PYPI_PUBLISH_REPO : 'pypi' %>" + - prepareCmd: | + /bin/bash ./ci/scripts/build_with_poetry.sh +- - '@semantic-release/github': + - assets: + - dist/*.whl + - dist/*.tar.gz diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index 977d9f17..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} - -set -x -t=$([ -t 0 ] && echo 't') -docker run -q -i${t} --rm\ - -v $PWD/dist:/dist -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ - -v $PWD/ci:/ci\ - -w /tmp\ - -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ - $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" diff --git a/ci/scripts/build_with_poetry.sh b/ci/scripts/build_with_poetry.sh new file mode 100755 index 00000000..53c89791 --- /dev/null +++ b/ci/scripts/build_with_poetry.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ -z "$DISTRIBUTION_NAME" ] || [ "$DISTRIBUTION_NAME" = "appmap" ] ; then + exec poetry build $* +fi + +echo "Altering distribution name to $DISTRIBUTION_NAME" + +cp -v pyproject.toml /tmp/pyproject.bak +sed -i -e "s/^name = \".*\"/name = \"${DISTRIBUTION_NAME}\"/" pyproject.toml +grep -n 'name = "' pyproject.toml + +poetry build $* + +echo "Not patching artifacts with Provides-Dist, they won't work anyway (this flow is solely for publishing test)" +cp -v /tmp/pyproject.bak pyproject.toml diff --git a/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh new file mode 100755 index 00000000..acee04ff --- /dev/null +++ b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -o pipefail + +artifacts=$* +injection_string="Provides-Dist: appmap" +if [ -n "$artifacts" ] && [ -n "$DISTRIBUTION_NAME" ] && [ "$DISTRIBUTION_NAME" != "appmap" ]; then + echo "Altered distribution name detected, injecting '$injection_string' into artifacts: $artifacts" + for artifact in $artifacts ; do + TMP=$(mktemp -d) + ARTIFACT_PATH="$(realpath ${artifact})" + if [[ $artifact == *.whl ]]; then + unzip -q "$ARTIFACT_PATH" -d "$TMP" + DISTINFO=$(find "$TMP" -type d -name "*.dist-info") + echo "$injection_string" >> "$DISTINFO/METADATA" + (cd "$TMP" && zip -qr "$ARTIFACT_PATH" .) + else + tar -xzf "$ARTIFACT_PATH" -C "$TMP" + PKG_INFO_FILE=$(find "$TMP" -type f -name "PKG-INFO") + echo "$injection_string" >> "$PKG_INFO_FILE" + + # Get the top-level directory to repack correctly + PKGDIR=$(find "$TMP" -mindepth 1 -maxdepth 1 -type d) + (cd "$TMP" && tar -czf "$ARTIFACT_PATH" "$(basename "$PKGDIR")") + fi + echo "($injection_string): patched $ARTIFACT_PATH" + rm -rf "$TMP" + done +fi diff --git a/ci/scripts/run_tests.sh b/ci/scripts/run_tests.sh new file mode 100755 index 00000000..d780a187 --- /dev/null +++ b/ci/scripts/run_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} +DISTRIBUTION_NAME=${DISTRIBUTION_NAME:-appmap} + +set -x +t=$([ -t 0 ] && echo 't') +docker run -q -i${t} --rm \ + -v $PWD/dist:/dist \ + -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ + -v $PWD/ci/tests:/ci/tests\ + -v $PWD/.git:/tmp/.git:ro\ + -v $PWD/ci/tests/data/readonly-mount-appmap.log:/tmp/appmap.log:ro\ + -w /tmp\ + -e DISTRIBUTION_NAME \ + $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/tests/smoketest.sh; /ci/tests/test_pipenv.sh; /ci/tests/test_poetry.sh}" diff --git a/ci/readonly-mount-appmap.log b/ci/tests/data/readonly-mount-appmap.log similarity index 100% rename from ci/readonly-mount-appmap.log rename to ci/tests/data/readonly-mount-appmap.log diff --git a/ci/smoketest.sh b/ci/tests/smoketest.sh similarity index 88% rename from ci/smoketest.sh rename to ci/tests/smoketest.sh index 3d0eeb13..cd0abade 100755 --- a/ci/smoketest.sh +++ b/ci/tests/smoketest.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash + test_recording_when_appmap_not_true() { cat < test_client.py @@ -36,8 +37,13 @@ EOF } set -ex + +# now appmap requires git +apt-get update -qq \ + && apt-get install -y --no-install-recommends git + pip -q install -U pip pytest "flask>=2,<3" python-decouple -pip -q install /dist/appmap-*-py3-none-any.whl +pip -q install /dist/${DISTRIBUTION_NAME//-/_}-*-py3-none-any.whl cp -R /_appmap/test/data/unittest/simple ./. @@ -66,4 +72,4 @@ else exit 1 fi -test_log_file_not_writable \ No newline at end of file +test_log_file_not_writable diff --git a/ci/test_pipenv.sh b/ci/tests/test_pipenv.sh similarity index 100% rename from ci/test_pipenv.sh rename to ci/tests/test_pipenv.sh diff --git a/ci/test_poetry.sh b/ci/tests/test_poetry.sh similarity index 100% rename from ci/test_poetry.sh rename to ci/tests/test_poetry.sh From c15eb7c0c57fb90bb4b2b71151f86cf7404c4ffa Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Mon, 24 Nov 2025 19:40:16 +0100 Subject: [PATCH 095/113] chore(ci): remove travis.yml --- .travis.yml | 63 ----------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 78a77e56..00000000 --- a/.travis.yml +++ /dev/null @@ -1,63 +0,0 @@ -os: linux -dist: jammy -language: python -python: -- "3.12" -- "3.11" -- "3.10" -- "3.9.14" -- "3.8" - -# Travis CI disabled in October 2025; this file is kept temporary for reference and possibility of rollback; can be deleted safely after migration -if: false - -# https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807 -#if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ - -before_install: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal - source "$HOME/.cargo/env" - pip -q install --upgrade pip 'setuptools==65.6.2' 'poetry>=1.2.0' - -install: pip -q install --upgrade "tox < 4" tox-travis -script: tox - -cache: - cargo: true - pip: true - directories: - - $TRAVIS_BUILD_DIR/.tox/ - - $HOME/.cache/pypoetry - -jobs: - include: - - stage: smoke test - services: - - docker - script: - - pip -q install poetry - - poetry build - - echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - - ci/run_tests.sh - - stage: release - if: branch = master - script: skip - before_deploy: - - pip -q install poetry - - nvm install lts/* - - npm i -g - semantic-release - @semantic-release/exec - @semantic-release/git - @semantic-release/changelog - @google/semantic-release-replace-plugin - # Note publishing this way requires the PyPI credentials to be - # present in the environment. Travis doesn't currently support - # providing environment variables to deploy providers through - # the build config (i.e. in this file). So, they must be - # provided through the build settings instead. - deploy: - - provider: script - script: semantic-release - on: - branch: master From 8037796056bca0aae0030c8e538e658de374a9a2 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Mon, 1 Dec 2025 16:10:07 +0100 Subject: [PATCH 096/113] chore(ci): add validation job to Lint-and-Test workflow, as umbrella/rollup check of matrix tests status --- .github/workflows/lint-and-test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index e2fe0415..f0ad33d8 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -39,3 +39,17 @@ jobs: python: ${{ matrix.python }} - name: Test run: tox + validation: + name: Validation + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - name: Validate matrix test success + run: | + # Check the status of the 'test' job (which includes all matrix variations) + if [ "${{ needs.test.result }}" != "success" ]; then + echo "One or more matrix test jobs failed." + exit 1 + fi + echo "All matrix test jobs passed." From 40912b95d095ac0a5181c739124261609dbc8b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Wed, 3 Dec 2025 23:54:41 +0100 Subject: [PATCH 097/113] refactor(lint): Resolve pylint too-many-positional-arguments This commit addresses and resolves several `too-many-positional-arguments` and `unused-argument` linting errors reported by pylint. The changes primarily involve adding `too-many-positional-arguments` to existing `pylint: disable` directives where the function signatures are fixed by external frameworks (Django, SQLAlchemy) or by internal design requirements (e.g., `__new__` methods, `HttpServerRequestEvent` init). Additionally, an `unused-argument` was fixed by renaming a parameter with a leading underscore to indicate its intentional non-use. These modifications ensure that the codebase adheres to the pylint standards without altering the intended functionality or API compatibility. --- _appmap/event.py | 2 +- _appmap/importer.py | 4 +++- _appmap/test/test_configuration.py | 2 +- _appmap/test/test_django.py | 2 +- _appmap/web_framework.py | 6 +++--- appmap/django.py | 4 +++- appmap/pytest.py | 2 +- appmap/sqlalchemy.py | 4 ++-- 8 files changed, 15 insertions(+), 11 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index f9c7c490..0094412e 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -446,7 +446,7 @@ class HttpServerRequestEvent(MessageEvent): __slots__ = ["http_server_request"] - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__( self, request_method, diff --git a/_appmap/importer.py b/_appmap/importer.py index 28a6c5a5..4493057c 100644 --- a/_appmap/importer.py +++ b/_appmap/importer.py @@ -42,7 +42,9 @@ class FilterableFn( ): __slots__ = () - def __new__(cls, scope, fn_name, fn, static_fn, auxtype=None): # pylint: disable=too-many-arguments + def __new__( + cls, scope, fn_name, fn, static_fn, auxtype=None + ): # pylint: disable=too-many-arguments,too-many-positional-arguments fqname = "%s.%s" % (scope.fqname, fn_name) self = super(FilterableFn, cls).__new__(cls, scope.scope, fqname, fn, static_fn, auxtype) return self diff --git a/_appmap/test/test_configuration.py b/_appmap/test/test_configuration.py index a1492e4a..aaeb1add 100644 --- a/_appmap/test/test_configuration.py +++ b/_appmap/test/test_configuration.py @@ -269,7 +269,7 @@ def test_missing_packages(self, tmpdir): self.check_default_config(Path(tmpdir).name) class TestSearchConfig: - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def test_config_in_parent_folder(self, data_dir, tmpdir, monkeypatch): copytree(data_dir / "config-up", str(tmpdir), dirs_exist_ok=True) diff --git a/_appmap/test/test_django.py b/_appmap/test/test_django.py index 5547fb9c..072b5628 100644 --- a/_appmap/test/test_django.py +++ b/_appmap/test/test_django.py @@ -98,7 +98,7 @@ def test_template(events): class ClientAdaptor(django.test.Client): """Adaptor for the client request parameters used in .web_framework tests.""" - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def generic( self, method, diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index 33e27e17..c6edb39b 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -102,7 +102,7 @@ def name_hash(namepart): return sha256(os.fsencode(namepart)).hexdigest() -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments def create_appmap_file( output_dir, request_method, @@ -142,7 +142,7 @@ def before_request_main(self, rec, req: Any) -> Tuple[float, int]: """Specify the main operations to be performed by a request is processed.""" raise NotImplementedError - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def after_request_main( self, request_path, status, headers, start, call_event_id ) -> Optional[HttpServerResponseEvent]: @@ -193,7 +193,7 @@ def before_request_hook(self, request) -> Tuple[Optional[Recorder], float, int]: return rec, start, call_event_id - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def after_request_hook( self, request_path, diff --git a/appmap/django.py b/appmap/django.py index 08674946..420b9bbc 100644 --- a/appmap/django.py +++ b/appmap/django.py @@ -59,7 +59,9 @@ def __init__(self): self.recorder = Recorder.get_current() # This signature is correct, the implementation confuses pylint: - def __call__(self, execute, sql, params, many, context): # pylint: disable=too-many-arguments + def __call__( + self, execute, sql, params, many, context + ): # pylint: disable=too-many-arguments,too-many-positional-arguments start = time.monotonic() try: return execute(sql, params, many, context) diff --git a/appmap/pytest.py b/appmap/pytest.py index 61b655f4..0d20ff8e 100644 --- a/appmap/pytest.py +++ b/appmap/pytest.py @@ -5,7 +5,7 @@ from pytest_django.django_compat import is_django_unittest except ImportError: - def is_django_unittest(item): + def is_django_unittest(_item): return False diff --git a/appmap/sqlalchemy.py b/appmap/sqlalchemy.py index 2e4c382f..66c30d22 100644 --- a/appmap/sqlalchemy.py +++ b/appmap/sqlalchemy.py @@ -13,7 +13,7 @@ @event.listens_for(Engine, "before_cursor_execute") -# pylint: disable=too-many-arguments,unused-argument +# pylint: disable=too-many-arguments,unused-argument,too-many-positional-arguments def capture_sql_call(conn, cursor, statement, parameters, context, executemany): """Capture SQL query call into appmap.""" if is_instrumentation_disabled(): @@ -45,7 +45,7 @@ def capture_sql_call(conn, cursor, statement, parameters, context, executemany): @event.listens_for(Engine, "after_cursor_execute") -# pylint: disable=too-many-arguments,unused-argument +# pylint: disable=too-many-arguments,unused-argument,too-many-positional-arguments def capture_sql(conn, cursor, statement, parameters, context, executemany): """Capture SQL query return into appmap.""" if is_instrumentation_disabled(): From eb0379f133b6fe8c1f69a98f4259a67de5725c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 26 Jan 2026 12:20:54 +0100 Subject: [PATCH 098/113] fix(recording): sanitize process recording filenames for Windows Replace colons in ISO 8601 timestamps with hyphens when generating filenames for process recordings. This prevents OSErrors on Windows systems where colons are invalid characters in filenames. Includes a regression test to verify that generated filenames do not contain colons. Also changes the format to use a dash to separate the timestamp from the PID for consistency. Fixes #377 --- _appmap/recording.py | 2 +- _appmap/test/test_recording.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/_appmap/recording.py b/_appmap/recording.py index 8513cbff..4a7d119a 100644 --- a/_appmap/recording.py +++ b/_appmap/recording.py @@ -117,7 +117,7 @@ def save_at_exit(): now = datetime.now(timezone.utc) iso_time = now.isoformat(timespec="seconds").replace("+00:00", "Z") process_id = os.getpid() - appmap_name = f"{iso_time}_{process_id}" + appmap_name = f"{iso_time}-{process_id}".replace(":","-") recorder_type = "process" metadata = { "name": appmap_name, diff --git a/_appmap/test/test_recording.py b/_appmap/test/test_recording.py index d7749d36..5700957c 100644 --- a/_appmap/test/test_recording.py +++ b/_appmap/test/test_recording.py @@ -226,3 +226,27 @@ def test_process_recording(data_dir, shell, tmp_path): actual = json.loads(appmap_files[0].read_text()) assert len(actual["events"]) > 0 assert len(actual["classMap"]) > 0 + + +def test_process_recording_filename_is_sanitized(data_dir, shell, tmp_path): + fixture = data_dir / "package1" + tmp = tmp_path / "process" + copytree(fixture, str(tmp / "package1"), dirs_exist_ok=True) + copy(data_dir / "appmap.yml", str(tmp)) + copytree(data_dir / "flask" / "init", str(tmp / "init"), dirs_exist_ok=True) + + ret = shell.run( + "python", + "-m", + "package1.package2", + env={"PYTHONPATH": "init", "APPMAP_RECORD_PROCESS": "true"}, + cwd=tmp, + ) + assert ret.returncode == 0 + + appmap_dir = tmp / "tmp" / "appmap" / "process" + appmap_files = list(appmap_dir.glob("*.appmap.json")) + assert len(appmap_files) == 1, "this only fails when run from VS Code?" + + filename = appmap_files[0].name + assert ":" not in filename From 17d180c12724709ba401bf8ff07b6a4a64e74476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Sat, 31 Jan 2026 12:27:04 +0100 Subject: [PATCH 099/113] ci: Use GitHub app for authorization --- .github/workflows/release.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2928b1af..75a80f60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,12 +63,16 @@ jobs: runs-on: ubuntu-latest needs: setup if: github.event_name == 'workflow_dispatch' || (github.event_name=='push' && startsWith(github.ref_name,'ci/') ) || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'ci/') ) ) - permissions: - contents: write - issues: write - pull-requests: write steps: + - name: Generate token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} - uses: actions/checkout@v5 + with: + token: ${{ steps.app-token.outputs.token }} - uses: ./.github/actions/setup-semantic-release # node+semantic-release - uses: ./.github/actions/setup # poetry - id: semantic-release # branch policies defined in .releaserc @@ -77,7 +81,7 @@ jobs: GIT_AUTHOR_EMAIL: release@app.land GIT_COMMITTER_NAME: appland-release GIT_COMMITTER_EMAIL: release@app.land - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} run: | if [ "$DRY_RUN" = "true" ]; then From dcf1844b650df280f4e4a5d096594e5f8d4d30fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 2 Feb 2026 22:40:32 +0100 Subject: [PATCH 100/113] ci: Fix paths in smoketest scripts --- ci/tests/test_pipenv.sh | 2 +- ci/tests/test_poetry.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/tests/test_pipenv.sh b/ci/tests/test_pipenv.sh index 09a6ba65..23a74e3b 100755 --- a/ci/tests/test_pipenv.sh +++ b/ci/tests/test_pipenv.sh @@ -6,4 +6,4 @@ pip -q install pipenv mkdir /pipenv || true cd /pipenv -pipenv run /ci/smoketest.sh +pipenv run /ci/tests/smoketest.sh diff --git a/ci/tests/test_poetry.sh b/ci/tests/test_poetry.sh index 561898a7..88e1e29d 100755 --- a/ci/tests/test_poetry.sh +++ b/ci/tests/test_poetry.sh @@ -9,4 +9,4 @@ cd /poetry poetry init -q # Yes, we need to set RUNNER, and we need to "poetry run" the script. -RUNNER="poetry run" poetry run /ci/smoketest.sh +RUNNER="poetry run" poetry run /ci/tests/smoketest.sh From 4eaa6d3a45f09f47da114d73cc66271252c9b8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 2 Feb 2026 22:40:04 +0100 Subject: [PATCH 101/113] refactor(ci): move GitHub release creation after smoke tests, auto-cleanup on failure Move GitHub release creation from semantic-release to the publish job, ensuring it only happens after smoke tests pass. If smoke tests fail, automatically clean up the release commit and tag from GitHub. Changes: - Remove @semantic-release/github plugin from .releaserc.yml - Extract version from pyproject.toml in release job and pass to downstream jobs - Add automatic cleanup in smoketest job if tests fail: - Verify HEAD commit is the release commit (sanity check) - Delete the release tag from remote - Revert the release commit with force push - Move GitHub release creation to publish job using --notes-from-tag - Use default GITHUB_TOKEN for creating releases (app token only needed for pushing to protected branch) Benefits: - GitHub releases only created after smoke tests pass - Automatic cleanup if tests fail (no manual intervention needed) - Simpler than preventing the tag/commit upfront - Handles the common case (tests pass) with no changes to workflow time --- .github/workflows/release.yml | 66 +++++++++++++++++++++++++++++++---- .releaserc.yml | 6 ++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75a80f60..2cd21342 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,6 +90,13 @@ jobs: semantic-release fi + - name: Get version + if: env.DRY_RUN != 'true' + id: version + run: | + VERSION=$(grep '^version = ' pyproject.toml | cut -d'"' -f2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Upload wheel if: env.DRY_RUN != 'true' uses: actions/upload-artifact@v4 @@ -102,16 +109,24 @@ jobs: with: name: sdist path: dist/*.tar.gz - outputs: # not reused in fact - release_tag: ${{ steps.semantic-release.outputs.next_release_tag }} + outputs: + version: ${{ steps.version.outputs.version }} smoketest: runs-on: ubuntu-latest needs: ['setup', 'release'] - if: github.event.inputs.dry_run!='true' + if: github.event.inputs.dry_run!='true' continue-on-error: ${{ needs.setup.outputs.distribution_name!='appmap' }} # altered names won't work anyway steps: + - name: Generate token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} - uses: actions/checkout@v5 + with: + token: ${{ steps.app-token.outputs.token }} - uses: ./.github/actions/refetch-artifacts - name: dockerhub login (for seamless docker pulling) uses: ./.github/actions/dockerhub-login @@ -119,15 +134,42 @@ jobs: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} continue-on-error: true - - run: ci/scripts/run_tests.sh + - name: Run smoke tests + id: smoketest + run: ci/scripts/run_tests.sh + continue-on-error: true env: SMOKETEST_DOCKER_IMAGE: python:3.12-slim DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + - name: Cleanup on failure + if: steps.smoketest.outcome == 'failure' + run: | + echo "::error::Smoke tests failed, cleaning up release v${{ needs.release.outputs.version }}" + + # Sanity check: verify HEAD commit is the release commit + COMMIT_MSG=$(git log -1 --pretty=%s) + if [[ "$COMMIT_MSG" != "chore(release): ${{ needs.release.outputs.version }}"* ]]; then + echo "::error::HEAD commit message doesn't match expected release commit!" + echo "::error::Expected: chore(release): ${{ needs.release.outputs.version }}" + echo "::error::Got: $COMMIT_MSG" + echo "::error::Aborting cleanup - manual intervention required" + exit 1 + fi + + # Delete the tag from remote + git push --delete origin "v${{ needs.release.outputs.version }}" || true + + # Reset to commit before the release commit and force push + git reset --hard HEAD~1 + git push --force origin HEAD:${{ github.ref_name }} + + exit 1 + # as a workaround to ownership issues (lost access to project) publish: - name: publish package on PyPI - needs: ['setup', 'release','smoketest'] + name: publish package on PyPI and create GitHub release + needs: ['setup', 'release', 'smoketest'] if: (( github.event.inputs.dry_run != 'true' ) && ( (needs.setup.outputs.publish_env == 'pypi') || (needs.setup.outputs.publish_env == 'testpypi') ) ) runs-on: ubuntu-latest environment: @@ -135,10 +177,22 @@ jobs: url: ${{ needs.setup.outputs.publish_to }} permissions: id-token: write + contents: write # needed for creating GitHub releases steps: - uses: actions/checkout@v5 + with: + ref: v${{ needs.release.outputs.version }} # checkout the tag + - uses: ./.github/actions/refetch-artifacts + - name: Create GitHub release + run: | + gh release create "v${{ needs.release.outputs.version }}" \ + dist/*.whl dist/*.tar.gz \ + --notes-from-tag + env: + GH_TOKEN: ${{ github.token }} + - name: Publish to PyPI if: needs.setup.outputs.publish_env=='pypi' uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.releaserc.yml b/.releaserc.yml index b5efcfc6..92dab71d 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -43,7 +43,5 @@ plugins: - - '@semantic-release/exec' - prepareCmd: | /bin/bash ./ci/scripts/build_with_poetry.sh -- - '@semantic-release/github': - - assets: - - dist/*.whl - - dist/*.tar.gz +# NOTE: @semantic-release/github plugin removed - GitHub release creation +# now happens in the publish job after smoke tests pass From 54ee2a364bb6e4500d7b81d555ae64dc2dcb42bc Mon Sep 17 00:00:00 2001 From: appland-release Date: Tue, 3 Feb 2026 16:31:41 +0000 Subject: [PATCH 102/113] chore(release): 2.1.9 [skip ci] ## [2.1.9](https://github.com/getappmap/appmap-python/compare/v2.1.8...v2.1.9) (2026-02-03) ### Bug Fixes * **recording:** sanitize process recording filenames for Windows ([eb0379f](https://github.com/getappmap/appmap-python/commit/eb0379f133b6fe8c1f69a98f4259a67de5725c95)), closes [#377](https://github.com/getappmap/appmap-python/issues/377) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d876d4..c1faf2ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [2.1.9](https://github.com/getappmap/appmap-python/compare/v2.1.8...v2.1.9) (2026-02-03) + + +### Bug Fixes + +* **recording:** sanitize process recording filenames for Windows ([eb0379f](https://github.com/getappmap/appmap-python/commit/eb0379f133b6fe8c1f69a98f4259a67de5725c95)), closes [#377](https://github.com/getappmap/appmap-python/issues/377) + ## [2.1.8](https://github.com/getappmap/appmap-python/compare/v2.1.7...v2.1.8) (2024-11-13) diff --git a/pyproject.toml b/pyproject.toml index e0b54baf..acd71e21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appmap" -version = "2.1.8" +version = "2.1.9" description = "Create AppMap files by recording a Python application." readme = "README.md" authors = [ From d1391cb874777e17be17573dac88a75e7305999a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Wed, 28 Jan 2026 18:31:50 +0100 Subject: [PATCH 103/113] chore: Migrate from Poetry to uv and Hatchling This commit migrates the project's dependency management and build system from Poetry to uv and Hatchling. This change significantly speeds up dependency installation and testing, providing a more efficient development workflow. Key changes include: - `pyproject.toml` is now configured for Hatchling and uv. - GitHub Actions workflows have been updated to use `astral-sh/setup-uv`. - `tox.ini` is reconfigured to use `tox-uv` for test environments. - `README.md` has been updated with new setup and usage instructions. - Obsolete Poetry and pyenv files (`tool-versions.example`, etc.) have been removed. --- .github/actions/setup/action.yml | 41 +---- .github/workflows/release.yml | 2 +- .gitignore | 2 + .releaserc.yml | 2 +- README.md | 86 ++++----- ...{build_with_poetry.sh => build_with_uv.sh} | 6 +- pylintrc | 1 - pyproject.toml | 171 ++++++++++-------- requirements-dev.txt | 10 - requirements-test.txt | 3 - tool-versions.example | 1 - tox.ini | 35 ++-- 12 files changed, 172 insertions(+), 188 deletions(-) rename ci/scripts/{build_with_poetry.sh => build_with_uv.sh} (81%) delete mode 100644 requirements-dev.txt delete mode 100644 requirements-test.txt delete mode 100644 tool-versions.example diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index daba8580..3bd9086d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,4 +1,5 @@ -name: Setup base (python, pip cache, tox) +name: Setup base (python, uv, tox) +description: "Setup Python, uv and tox for further steps" inputs: python: description: "Python version to use" @@ -11,41 +12,17 @@ outputs: runs: using: "composite" steps: - - name: pip cache - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ inputs.python }} - - - name: Cargo cache - uses: actions/cache/@v4 - with: - path: "~/.cargo" - key: ${{ runner.os }}-cargo - - - name: Poetry cache - uses: actions/cache/@v4 - with: - path: "~/.cache/pypoetry" - key: ${{ runner.os }}-poetry-${{ inputs.python }} - restore-keys: | - ${{ runner.os }}-poetry- - - uses: actions/setup-python@v6 id: python with: python-version: ${{ inputs.python }} - - name: upgrade pip and install tox + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Install tox, tox-uv and tox-gh-actions shell: bash run: | - python -m pip -q install --upgrade pip "setuptools==65.6.2" - pip -q install "tox<4" tox-gh-actions - - - name: install Rust and Poetry - shell: bash - run : | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal - source "$HOME/.cargo/env" - pip -q install poetry>=1.2.0 + uv tool install tox --with tox-uv --with tox-gh-actions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2cd21342..3f36b93a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,7 +74,7 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} - uses: ./.github/actions/setup-semantic-release # node+semantic-release - - uses: ./.github/actions/setup # poetry + - uses: ./.github/actions/setup - id: semantic-release # branch policies defined in .releaserc env: GIT_AUTHOR_NAME: appland-release diff --git a/.gitignore b/.gitignore index 4473e002..fac985bc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .tool-versions poetry.lock +uv.lock __pycache__/ *.py[cod] @@ -22,3 +23,4 @@ htmlcov/ /ruff.toml appmap.log +*.sqlite3 diff --git a/.releaserc.yml b/.releaserc.yml index 92dab71d..23ba947e 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -42,6 +42,6 @@ plugins: - pyproject.toml - - '@semantic-release/exec' - prepareCmd: | - /bin/bash ./ci/scripts/build_with_poetry.sh + /bin/bash ./ci/scripts/build_with_uv.sh # NOTE: @semantic-release/github plugin removed - GitHub release creation # now happens in the publish job after smoke tests pass diff --git a/README.md b/README.md index a73aa470..ad9f8e88 100644 --- a/README.md +++ b/README.md @@ -50,16 +50,19 @@ oldest version currently supported (see the ## Dependency management -[poetry](https://https://python-poetry.org/) for dependency management: +[uv](https://docs.astral.sh/uv/) is used for dependency management and provides fast package installation: -``` -% brew install poetry -% cd appmap-python -% poetry install +```bash +# Install uv (macOS/Linux) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install dependencies +cd appmap-python +uv sync --all-extras ``` ### wrapt -The one dependency that is not managed using `poetry` is `wrapt`. Because it's possible that +The one dependency that is not managed using `uv` is `wrapt`. Because it's possible that projects that use `appmap` may also need an unmodified version of `wrapt` (e.g. `pylint` depends on `astroid`, which in turn depends on `wrapt`), we use [vendoring](https://github.com/pradyunsg/vendoring) to vendor `wrapt`. @@ -69,64 +72,63 @@ To update `wrapt`, use `tox` (described below) to run the `vendoring` environmen ## Linting [pylint](https://www.pylint.org/) for linting: -``` -% cd appmap-python -% poetry run pylint appmap +```bash +cd appmap-python +uv run tox -e lint + +# Or run pylint directly +uv run pylint appmap -------------------------------------------------------------------- Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) - ``` -[Note that the current configuration has a threshold set which must be met for the Travis build to -pass. To make this easier to achieve, a number of checks have both been disabled. They should be -reenabled as soon as possible.] - ## Testing ### pytest -Note that you must install the dependencies contained in -[requirements-dev.txt](requirements-dev.txt) before running tests. See the explanation in -[pyproject.toml](pyproject.toml) for details. +[pytest](https://docs.pytest.org/en/stable/) for testing: -Additionally, the tests currently require that you set `APPMAP=true` and -`APPMAP_DISPLAY_PARAMS=true`. +```bash +cd appmap-python -[pytest](https://docs.pytest.org/en/stable/) for testing: +# Run all tests +APPMAP_DISPLAY_PARAMS=true uv run appmap-python pytest -``` -% cd appmap-python -% pip install -r requirements-test.txt -% APPMAP=true APPMAP_DISPLAY_PARAMS=true poetry run pytest +# Run tests with a specific Python version +APPMAP_DISPLAY_PARAMS=true uv run --python 3.9 appmap-python pytest + +# Run tests in parallel +APPMAP_DISPLAY_PARAMS=true uv run appmap-python pytest -n auto ``` ### tox -Additionally, the `tox` configuration provides the ability to run the tests for all -supported versions of Python and Django. +The `tox` configuration provides the ability to run the tests for all supported versions of Python and web frameworks (Django, Flask, SQLAlchemy). -`tox` requires that all the correct versions of Python to be available to create -the test environments. [pyenv](https://github.com/pyenv/pyenv) is an easy way to manage -multiple versions of Python, and the [xxenv-latest -plugin](https://github.com/momo-lab/xxenv-latest) can help get all the latest versions. +With `uv`, you don't need to pre-install Python versions - `uv` will automatically download and manage them: +```bash +cd appmap-python +# Run full test matrix (all Python versions and frameworks) +uv run tox -```sh -% brew install pyenv -% git clone https://github.com/momo-lab/xxenv-latest.git "$(pyenv root)"/plugins/xxenv-latest -% cd appmap-python -% pyenv latest local 3.{9,6,7,8} -% for v in 3.{9,6,7,8}; do pyenv latest install $v; done -% poetry run tox +# Run tests for a specific Python version +uv run tox -e py312-web + +# Run tests for specific framework +uv run tox -e py312-django5 + +# Update vendored wrapt dependency +uv run tox -e vendoring sync ``` ## Code Coverage [coverage](https://coverage.readthedocs.io/) for coverage: -``` -% cd appmap-python -% poetry run coverage run -m pytest -% poetry run coverage html -% open htmlcov/index.html +```bash +cd appmap-python +uv run coverage run -m pytest +uv run coverage html +open htmlcov/index.html ``` diff --git a/ci/scripts/build_with_poetry.sh b/ci/scripts/build_with_uv.sh similarity index 81% rename from ci/scripts/build_with_poetry.sh rename to ci/scripts/build_with_uv.sh index 53c89791..aa85b485 100755 --- a/ci/scripts/build_with_poetry.sh +++ b/ci/scripts/build_with_uv.sh @@ -4,16 +4,16 @@ set -e set -o pipefail if [ -z "$DISTRIBUTION_NAME" ] || [ "$DISTRIBUTION_NAME" = "appmap" ] ; then - exec poetry build $* + exec uv build $* fi -echo "Altering distribution name to $DISTRIBUTION_NAME" +echo "Altering distribution name to $DISTRIBUTION_NAME" cp -v pyproject.toml /tmp/pyproject.bak sed -i -e "s/^name = \".*\"/name = \"${DISTRIBUTION_NAME}\"/" pyproject.toml grep -n 'name = "' pyproject.toml -poetry build $* +uv build $* echo "Not patching artifacts with Provides-Dist, they won't work anyway (this flow is solely for publishing test)" cp -v /tmp/pyproject.bak pyproject.toml diff --git a/pylintrc b/pylintrc index 808bf36e..e281a970 100644 --- a/pylintrc +++ b/pylintrc @@ -95,7 +95,6 @@ recursive=no # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. -suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. diff --git a/pyproject.toml b/pyproject.toml index acd71e21..77004c01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,101 +1,114 @@ -[tool.poetry] +[project] name = "appmap" version = "2.1.9" description = "Create AppMap files by recording a Python application." readme = "README.md" +requires-python = ">=3.8" +license = { text = "MIT" } authors = [ - "Alan Potter ", - "Viraj Kanwade ", - "Rafał Rzepecki " + { name = "Alan Potter", email = "alan@app.land" }, + { name = "Viraj Kanwade", email = "viraj.kanwade@forgeahead.io" }, + { name = "Rafał Rzepecki", email = "rafal@app.land" } ] -homepage = "https://github.com/applandinc/appmap-python" -license = "MIT" classifiers = [ - 'Development Status :: 4 - Beta', - 'Framework :: Django', - 'Framework :: Django :: 3.2', - 'Framework :: Flask', - 'Framework :: Pytest', - 'Intended Audience :: Developers', - 'Topic :: Software Development', - 'Topic :: Software Development :: Debuggers', - 'Topic :: Software Development :: Documentation' + "Development Status :: 4 - Beta", + "Framework :: Django", + "Framework :: Django :: 3.2", + "Framework :: Flask", + "Framework :: Pytest", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development", + "Topic :: Software Development :: Debuggers", + "Topic :: Software Development :: Documentation", ] -include = [ - { path = 'appmap.pth', format = ['sdist','wheel'] }, - { path = '_appmap/test/**/*', format = 'sdist' } -] - -exclude = ['_appmap/wrapt'] - -packages = [ - { include = "appmap" }, - { include = "_appmap/*.py" }, - { include = "_appmap/wrapt/**/*", from = "vendor" } -] - -[tool.poetry.dependencies] # Please update the documentation if changing the supported python version # https://github.com/applandinc/applandinc.github.io/blob/master/_docs/reference/appmap-python.md#supported-versions -python = "^3.8" -PyYAML = ">=5.3.0" -inflection = ">=0.3.0" -importlib-resources = "^5.4.0" -packaging = ">=19.0" -# If you include "Django" as an optional dependency here, you'll be able to use poetry to install it -# in your dev environment. However, doing so causes poetry v1.2.0 to remove it from the virtualenv -# *created and managed by tox*, i.e. not your dev environment. -# -# So, if you'd like to run the tests outside of tox, run `pip install -r requirements-dev.txt` to -# install it and the rest of the dev dependencies. +dependencies = [ + "PyYAML>=5.3.0", + "inflection>=0.3.0", + "importlib-resources>=5.4.0", + "packaging>=19.0", +] -[tool.poetry.group.dev.dependencies] -Twisted = "^22.4.0" -incremental = "<24.7.0" -asgiref = "^3.7.2" -black = "^24.2.0" -coverage = "^5.3" -flake8 = "^3.8.4" -httpretty = "^1.0.5" -isort = "^5.10.1" -pprintpp = ">=0.4.0" -pyfakefs = "^5.3.5" -pylint = "^3.0" -pytest = "^7.3.2" -pytest-django = "~4.7" -pytest-mock = "^3.5.1" -pytest-randomly = "^3.5.0" -pytest-shell-utilities = "^1.8.0" -pytest-xprocess = "^0.23.0" -python-decouple = "^3.5" -requests = "^2.25.1" -tox = "^3.22.0" -# v2.30.0 of "requests" depends on urllib3 v2, which breaks the tests for http_client_requests. Pin -# to v1 until this gets fixed. -urllib3 = "^1" -uvicorn = "^0.27.1" -fastapi = "^0.110.0" -httpx = "^0.27.0" -pytest-env = "^1.1.3" -pytest-console-scripts = "^1.4.1" -pytest-xdist = "^3.6.1" -psutil = "^6.0.0" -ruff = "^0.5.3" +[project.optional-dependencies] +test = [ + "pytest>=7.3.2,<8.0", + "pytest-mock>=3.5.1,<4.0", + "pytest-randomly>=3.5.0,<4.0", + "pytest-shell-utilities>=1.8.0,<2.0", + "pytest-xprocess>=0.23.0,<1.0", + "pytest-env>=1.1.3,<2.0", + "pytest-console-scripts>=1.4.1,<2.0", + "pytest-xdist>=3.6.1,<4.0", + "httpretty>=1.0.5,<2.0", + "pyfakefs>=5.3.5,<6.0", + "requests>=2.25.1,<3.0", + "python-decouple>=3.5,<4.0", + "Twisted>=22.4.0,<23.0", + "incremental<24.7.0", + "asgiref>=3.7.2,<4.0", + "psutil>=6.0.0,<7.0", + "uvicorn>=0.27.1,<1.0", + "fastapi>=0.110.0,<1.0", + "httpx>=0.27.0,<1.0", + # v2.30.0 of "requests" depends on urllib3 v2, which breaks the tests for http_client_requests. Pin + # to v1 until this gets fixed. + "urllib3>=1,<2", + "Django", + "Flask", + "sqlalchemy", + "pytest-django>=4.7,<5.0", + "numpy>=1.24.4,<2.0; python_version < '3.9'", + "numpy>=2.0; python_version >= '3.9'", +] +dev = [ + "appmap[test]", + "black>=24.2.0,<25.0", + "coverage>=5.3,<6.0", + "flake8>=3.8.4,<4.0", + "isort>=5.10.1,<6.0", + "pprintpp>=0.4.0,<1.0", + "pylint>=3.0,<4.0", + "tox>=4.0,<5.0", + "tox-uv>=1.0,<2.0", + "tox-gh-actions>=3.0,<4.0", + "ruff>=0.5.3,<1.0", +] -[build-system] -requires = ["poetry-core>=1.1.0"] -build-backend = "poetry.core.masonry.api" +[project.urls] +Homepage = "https://github.com/applandinc/appmap-python" -[tool.poetry.plugins."pytest11"] -appmap = "appmap.pytest" - -[tool.poetry.scripts] +[project.scripts] appmap-agent-init = "appmap.command.appmap_agent_init:run" appmap-agent-status = "appmap.command.appmap_agent_status:run" appmap-agent-validate = "appmap.command.appmap_agent_validate:run" appmap-python = "appmap.command.runner:run" +[project.entry-points.pytest11] +appmap = "appmap.pytest" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["appmap", "_appmap"] +exclude = ["_appmap/test", "_appmap/wrapt"] +force-include = { "appmap.pth" = "appmap.pth", "vendor/_appmap/wrapt" = "_appmap/wrapt" } + +[tool.hatch.build.targets.sdist] +only-include = [ + "appmap", + "_appmap", + "appmap.pth", + "vendor", + "pyproject.toml", + "README.md", + "LICENSE", +] + [tool.black] line-length = 102 extend-exclude = ''' diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f70988fa..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,10 +0,0 @@ -#requirements-dev.txt -tox -django -flask >=2, <= 3 -pytest-django<4.8 -fastapi -httpx -sqlalchemy -debugpy -numpy \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 16b853d6..00000000 --- a/requirements-test.txt +++ /dev/null @@ -1,3 +0,0 @@ -django ~= 3.2 -pytest-django < 4.8 -sqlalchemy < 2.0 diff --git a/tool-versions.example b/tool-versions.example deleted file mode 100644 index 4a4d044f..00000000 --- a/tool-versions.example +++ /dev/null @@ -1 +0,0 @@ -python 3.8.18 3.9.18 3.10.13 3.11.7 3.12.1 diff --git a/tox.ini b/tox.ini index be81c9d2..2b8ed82f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,8 @@ [tox] +requires = + tox>=4 + tox-uv + tox-gh-actions isolated_build = true # The *-web environments test the latest versions of Django and Flask with the full test suite. For @@ -22,13 +26,15 @@ deps= sqlalchemy >=2.0, <3.0 [testenv] -passenv = +usedevelop = true +extras = test +passenv = PYTEST_XDIST_AUTO_NUM_WORKERS -setenv = +setenv = APPMAP_DISPLAY_PARAMS=true deps= - poetry web: {[web-deps]deps} + web,django3,django4,django5: pytest-django >=4.7, <5.0 py38: numpy==1.24.4 py3{9,10,11,12}: numpy >=2 flask2: Flask >= 2.0, <3.0 @@ -38,31 +44,30 @@ deps= sqlalchemy1: sqlalchemy >=1.4.11, <2.0 commands = - poetry install -v - web: poetry run appmap-python {posargs:pytest -n logical} - django3: poetry run appmap-python pytest -n logical _appmap/test/test_django.py - django4: poetry run appmap-python pytest -n logical _appmap/test/test_django.py - django5: poetry run appmap-python pytest -n logical _appmap/test/test_django.py - flask2: poetry run appmap-python pytest -n logical _appmap/test/test_flask.py - sqlalchemy1: poetry run appmap-python pytest -n logical _appmap/test/test_sqlalchemy.py + web: appmap-python {posargs:pytest -n logical} + django3: appmap-python pytest -n logical _appmap/test/test_django.py + django4: appmap-python pytest -n logical _appmap/test/test_django.py + django5: appmap-python pytest -n logical _appmap/test/test_django.py + flask2: appmap-python pytest -n logical _appmap/test/test_flask.py + sqlalchemy1: appmap-python pytest -n logical _appmap/test/test_sqlalchemy.py [testenv:lint] -skip_install = True +skip_install = False +extras = test deps = - poetry {[web-deps]deps} numpy >=2 + pylint >=3.0 commands = - poetry install # It doesn't seem great to disable cyclic-import checking, but the imports # aren't currently causing any problems. They should probably get fixed # sometime soon. - {posargs:poetry run pylint --disable=cyclic-import -j 0 appmap _appmap} + {posargs:pylint --disable=cyclic-import -j 0 appmap _appmap} [testenv:vendoring] skip_install = True deps = vendoring commands = - poetry run vendoring {posargs:sync} + vendoring {posargs:sync} # We don't need the .pyi files vendoring generates python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' From a239251d39342a5b3d255bd86fb2ff3cbe6408c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Sat, 4 Apr 2026 18:06:47 +0200 Subject: [PATCH 104/113] chore: move cyclic-imports pylint exclusion from tox.ini to pylintrc --- pylintrc | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index e281a970..853e0308 100644 --- a/pylintrc +++ b/pylintrc @@ -418,6 +418,7 @@ confidence=HIGH, # Disable unidiomatic-typecheck. Using isinstance() invokes the descriptor protocol, which can have # side effects. Using type() avoids this. disable=unidiomatic-typecheck, + cyclic-import, raw-checker-failed, bad-inline-option, locally-disabled, diff --git a/tox.ini b/tox.ini index 2b8ed82f..852b3390 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,7 @@ commands = # It doesn't seem great to disable cyclic-import checking, but the imports # aren't currently causing any problems. They should probably get fixed # sometime soon. - {posargs:pylint --disable=cyclic-import -j 0 appmap _appmap} + {posargs:pylint -j 0 appmap _appmap} [testenv:vendoring] skip_install = True From 453b697512f04d235e09629b582d1a12f1dac2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Sat, 4 Apr 2026 18:20:50 +0200 Subject: [PATCH 105/113] feat: Capture argument values of labeled functions by default Introduces a new mode to APPMAP_DISPLAY_PARAMS, which now can be: - 'true': capture argument values for all functions, - 'false': disable all argument value capture, - 'labeled' (new, default): capture only for labeled functions. The labeled default provides useful argument visibility for semantically important functions (such as HTTP, crypto, etc.) without the performance cost of global capture. The capture policy is evaluated once per function at instrumentation time and cached on the `_InstrumentedFn` tuple, eliminating per-call environment lookups from the hot path. Co-Authored-By: kgilpin --- _appmap/env.py | 16 +++++++-- _appmap/event.py | 42 ++++++++++++---------- _appmap/instrument.py | 19 +++++++--- _appmap/test/data/example_class.py | 4 +++ _appmap/test/test_describe_value.py | 24 ++++++++++--- _appmap/test/test_events.py | 56 +++++++++++++++++++++++++++++ _appmap/test/test_params.py | 4 ++- _appmap/web_framework.py | 2 +- appmap/__init__.py | 4 +-- 9 files changed, 138 insertions(+), 33 deletions(-) diff --git a/_appmap/env.py b/_appmap/env.py index aa61b233..be23a61d 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -45,8 +45,16 @@ def __init__(self, env=None, cwd=None): # them. enabled = self._env.get("_APPMAP", None) self._enabled = enabled is not None and enabled.lower() != "false" - display_params = self._env.get("_APPMAP_DISPLAY_PARAMS", None) - self._display_params = display_params is not None and display_params.lower() != "false" + display_params = self._env.get("_APPMAP_DISPLAY_PARAMS", "labeled").lower() + if display_params == "true": + self._display_params = True + self._display_labeled_params = True + elif display_params == "false": + self._display_params = False + self._display_labeled_params = False + else: # "labeled" or "auto" or anything else defaults to labeled + self._display_params = False + self._display_labeled_params = True logger = logging.getLogger(__name__) # The user shouldn't set APPMAP_OUTPUT_DIR, but some tests depend on being able to use it. @@ -134,6 +142,10 @@ def is_appmap_repo(self): def display_params(self): return self._display_params + @property + def display_labeled_params(self): + return self._display_labeled_params + def getLogger(self, name) -> trace_logger.TraceLogger: return cast(trace_logger.TraceLogger, logging.getLogger(name)) diff --git a/_appmap/event.py b/_appmap/event.py index 0094412e..4bdcf3af 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -45,13 +45,13 @@ def reset(cls): cls._next_thread_id = 0 -def display_string(val): +def display_string(val, display_value=False): # If we're asked to display parameters, make a best-effort attempt # to get a string value for the parameter using repr(). If parameter # display is disabled, or repr() has raised, just formulate a value # from the class and id. value = None - if Env.current.display_params: + if display_value: try: value = repr(val) except Exception: # pylint: disable=broad-except @@ -79,7 +79,7 @@ def _is_list_or_dict(val_type): return issubclass(val_type, list), issubclass(val_type, dict) -def _describe_schema(name, val, depth, max_depth): +def _describe_schema(name, val, depth, max_depth, display_value=False): val_type = type(val) @@ -88,6 +88,9 @@ def _describe_schema(name, val, depth, max_depth): ret["name"] = name ret["class"] = fqname(val_type) + if not display_value: + return ret + islist, isdict = _is_list_or_dict(val_type) if not (islist or isdict) or (depth >= max_depth and isdict): return ret @@ -100,7 +103,10 @@ def _describe_schema(name, val, depth, max_depth): elts = val.items() schema_key = "properties" - schema = [_describe_schema(k, v, depth + 1, max_depth) for k, v in elts] + schema = [ + _describe_schema(k, v, depth + 1, max_depth, display_value=display_value) + for k, v in elts + ] # schema will be [None] if depth is exceeded, don't use it if any(schema): ret[schema_key] = schema @@ -108,15 +114,14 @@ def _describe_schema(name, val, depth, max_depth): return ret -def describe_value(name, val, max_depth=5): - ret = { +def describe_value(name, val, max_depth=5, display_value=False): + ret = _describe_schema(name, val, 0, max_depth, display_value=display_value) + ret.update({ "object_id": id(val), - "value": display_string(val), - } - if Env.current.display_params: - ret.update(_describe_schema(name, val, 0, max_depth)) + "value": display_string(val, display_value=display_value), + }) - if any(_is_list_or_dict(type(val))): + if display_value and any(_is_list_or_dict(type(val))): ret["size"] = len(val) return ret @@ -170,9 +175,9 @@ def __init__(self, sigp): def __repr__(self): return "" % (self.name, self.kind) - def to_dict(self, value): + def to_dict(self, value, display_value=False): ret = {"kind": self.kind} - ret.update(describe_value(self.name, value)) + ret.update(describe_value(self.name, value, display_value=display_value)) return ret def _get_name_parts(filterable): @@ -247,7 +252,7 @@ def make_params(filterable): return [Param(p) for p in sig.parameters.values()] @staticmethod - def set_params(params, instance, args, kwargs): + def set_params(params, instance, args, kwargs, display_value=False): # pylint: disable=too-many-branches # Note that set_params expects args and kwargs as a tuple and # dict, respectively. It operates on them as collections, so @@ -295,7 +300,7 @@ def set_params(params, instance, args, kwargs): # If all the parameter types are handled, this # shouldn't ever happen... raise RuntimeError("Unknown parameter with desc %s" % (repr(p))) - ret.append(p.to_dict(value)) + ret.append(p.to_dict(value, display_value=display_value)) return ret @property @@ -405,8 +410,9 @@ def message_parameters(self): @message_parameters.setter def message_parameters(self, params): + display_params = Env.current.display_params for name, value in params.items(): - message_object = describe_value(name, value) + message_object = describe_value(name, value, display_value=display_params) self.message.append(message_object) @@ -497,13 +503,13 @@ def __init__(self, parent_id, elapsed): class FuncReturnEvent(ReturnEvent): __slots__ = ["return_value"] - def __init__(self, parent_id, elapsed, return_value): + def __init__(self, parent_id, elapsed, return_value, display_value=False): super().__init__(parent_id, elapsed) # Import here to prevent circular dependency # pylint: disable=import-outside-toplevel from _appmap.instrument import recording_disabled # noqa: F401 with recording_disabled(): - self.return_value = describe_value(None, return_value) + self.return_value = describe_value(None, return_value, display_value=display_value) class HttpResponseEvent(ReturnEvent): diff --git a/_appmap/instrument.py b/_appmap/instrument.py index 179e425c..784eca4c 100644 --- a/_appmap/instrument.py +++ b/_appmap/instrument.py @@ -70,7 +70,7 @@ def saved_shallow_rule(): _InstrumentedFn = namedtuple( - "_InstrumentedFn", "fn fntype instrumented_fn make_call_event params" + "_InstrumentedFn", "fn fntype instrumented_fn make_call_event params display_params" ) @@ -84,7 +84,9 @@ def call_instrumented(f, instance, args, kwargs): with recording_disabled(): logger.trace("%s args %s kwargs %s", f.fn, args, kwargs) - params = CallEvent.set_params(f.params, instance, args, kwargs) + params = CallEvent.set_params( + f.params, instance, args, kwargs, display_value=f.display_params + ) call_event = f.make_call_event(parameters=params) Recorder.add_event(call_event) call_event_id = call_event.id @@ -95,7 +97,8 @@ def call_instrumented(f, instance, args, kwargs): elapsed_time = time.time() - start_time return_event = event.FuncReturnEvent( - return_value=ret, parent_id=call_event_id, elapsed=elapsed_time + return_value=ret, parent_id=call_event_id, elapsed=elapsed_time, + display_value=f.display_params ) Recorder.add_event(return_event) return ret @@ -120,9 +123,16 @@ def instrument(filterable): """return an instrumented function""" logger.debug("hooking %s", filterable.fqname) + # note this has to happen before CallEvent.make, which clears this attribute + has_labels = hasattr(filterable.obj, "_appmap_labels") + make_call_event = event.CallEvent.make(filterable) params = CallEvent.make_params(filterable) + display_params = Env.current.display_params or ( + has_labels and Env.current.display_labeled_params + ) + # django depends on being able to find the cache_clear attribute # on functions. (You can see this by trying to map # https://github.com/chicagopython/chypi.org.) Make sure it gets @@ -132,7 +142,8 @@ def instrument(filterable): def instrumented_fn(wrapped, instance, args, kwargs): with saved_shallow_rule(): f = _InstrumentedFn( - wrapped, filterable.fntype, instrumented_fn, make_call_event, params + wrapped, filterable.fntype, instrumented_fn, make_call_event, params, + display_params ) return call_instrumented(f, instance, args, kwargs) diff --git a/_appmap/test/data/example_class.py b/_appmap/test/data/example_class.py index 4d7c2c93..e446efaf 100644 --- a/_appmap/test/data/example_class.py +++ b/_appmap/test/data/example_class.py @@ -56,6 +56,10 @@ def test_exception(self): def labeled_method(self): return "super important" + @appmap.labels("super", "important") + def labeled_method_with_param(self, p): + return p + @staticmethod @wrap_fn def wrapped_static_method(): diff --git a/_appmap/test/test_describe_value.py b/_appmap/test/test_describe_value.py index fe5706d3..24f90357 100644 --- a/_appmap/test/test_describe_value.py +++ b/_appmap/test/test_describe_value.py @@ -24,7 +24,7 @@ def value(self): return {"id": 1, "contents": "some text"} def test_one_level_schema(self, value): - actual = describe_value(None, value) + actual = describe_value(None, value, display_value=True) assert actual == DictIncluding( { "properties": [ @@ -34,6 +34,13 @@ def test_one_level_schema(self, value): } ) + def test_one_level_schema_display_false(self, value): + actual = describe_value(None, value, display_value=False) + assert "properties" not in actual + assert actual["class"] == "builtins.dict" + assert "builtins.dict object at" in actual["value"] + assert actual["object_id"] == id(value) + class TestNestedDictValue: @pytest.fixture @@ -41,7 +48,7 @@ def value(self): return {"page": {"page_number": 1, "page_size": 20, "total": 2383}} def test_two_level_schema(self, value): - actual = describe_value(None, value) + actual = describe_value(None, value, display_value=True) assert actual == DictIncluding( { "properties": [ @@ -60,7 +67,7 @@ def test_two_level_schema(self, value): def test_respects_max_depth(self, value): expected = {"properties": [{"name": "page", "class": "builtins.dict"}]} - actual = describe_value(None, value, max_depth=1) + actual = describe_value(None, value, max_depth=1, display_value=True) assert actual == DictIncluding(expected) @@ -70,7 +77,7 @@ def value(self): return [{"id": 1, "contents": "some text"}, {"id": 2}] def test_an_array_containing_schema(self, value): - actual = describe_value(None, value) + actual = describe_value(None, value, display_value=True) assert actual["class"] == "builtins.list" assert actual["items"][0] == DictIncluding( { @@ -88,6 +95,13 @@ def test_an_array_containing_schema(self, value): } ) + def test_an_array_display_false(self, value): + actual = describe_value(None, value, display_value=False) + assert "items" not in actual + assert actual["class"] == "builtins.list" + assert "builtins.list object at" in actual["value"] + assert actual["object_id"] == id(value) + class TestNestedArrays: @pytest.fixture @@ -95,7 +109,7 @@ def value(self): return [[["one"]]] def test_arrays_ignore_max_depth(self, value): - actual = describe_value(None, value, max_depth=1) + actual = describe_value(None, value, max_depth=1, display_value=True) expected = { "class": "builtins.list", "items": [ diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index bef98577..cbab31e8 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -52,6 +52,7 @@ def test_recursion_protection(self): # is working assert True + @pytest.mark.appmap_enabled(env={"APPMAP_DISPLAY_PARAMS": "true"}) def test_when_str_raises(self, mocker): r = appmap.Recording() with r: @@ -68,6 +69,7 @@ def test_when_str_raises(self, mocker): actual_value = r.events[0].parameters[0]["value"] assert expected_value == actual_value + @pytest.mark.appmap_enabled(env={"APPMAP_DISPLAY_PARAMS": "true"}) def test_when_both_raise(self, mocker): r = appmap.Recording() with r: @@ -117,6 +119,60 @@ def test_describe_return_value_recursion_protection(self): "return_self" ] + @pytest.mark.appmap_enabled(env={"APPMAP_DISPLAY_PARAMS": None}) + def test_labeled_params_displayed_by_default(self): + """When display_params is 'labeled' (default), + labeled functions should still have their params displayed via repr().""" + r = appmap.Recording() + with r: + from example_class import ExampleClass # pylint: disable=import-outside-toplevel + + result = ExampleClass().labeled_method_with_param("hello") + ExampleClass().instance_with_param("hello") + + assert result == "hello" + call_event = r.events[0] + # Parameter value should be the repr, not the opaque object string + assert call_event.parameters[0]["value"] == "'hello'" + # Return value should also be displayed + return_event = r.events[1] + assert return_event.return_value["value"] == "'hello'" + + # Unlabeled method should not have its params displayed, even in the same recording + call_event_unlabeled = r.events[2] + assert "object at" in call_event_unlabeled.parameters[0]["value"] + + @pytest.mark.appmap_enabled( + env={ + "APPMAP_DISPLAY_PARAMS": "false", + } + ) + def test_labeled_params_not_displayed_when_disabled(self): + """When display_params is off, labeled functions should NOT have their params displayed.""" + r = appmap.Recording() + with r: + from example_class import ExampleClass # pylint: disable=import-outside-toplevel + + ExampleClass().labeled_method_with_param("hello") + + call_event = r.events[0] + # Parameter value should be the opaque object string + assert "object at" in call_event.parameters[0]["value"] + + @pytest.mark.appmap_enabled(env={"APPMAP_DISPLAY_PARAMS": "labeled"}) + def test_unlabeled_params_not_displayed(self): + """When display_params is 'labeled', unlabeled functions should NOT + have their params displayed.""" + r = appmap.Recording() + with r: + from example_class import ExampleClass # pylint: disable=import-outside-toplevel + + ExampleClass().instance_with_param("hello") + + call_event = r.events[0] + # Parameter value should be the opaque object string + assert "object at" in call_event.parameters[0]["value"] + # There should be an exception return event generated even when the raised exception is a # BaseException. def test_exception_event_with_base_exception(self): diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index 26669861..7f6d3436 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -33,7 +33,9 @@ def prepare(cls, ffn): def wrapped_fn(_, instance, args, kwargs): return make_call_event( - parameters=CallEvent.set_params(params, instance, args, kwargs) + parameters=CallEvent.set_params( + params, instance, args, kwargs, display_value=True + ) ) return wrapped_fn diff --git a/_appmap/web_framework.py b/_appmap/web_framework.py index c6edb39b..086154b6 100644 --- a/_appmap/web_framework.py +++ b/_appmap/web_framework.py @@ -44,7 +44,7 @@ class TemplateEvent(Event): # pylint: disable=too-few-public-methods def __init__(self, path, instance=None): super().__init__("call") - self.receiver = describe_value(None, instance) + self.receiver = describe_value(None, instance, display_value=Env.current.display_params) self.path = root_relative_path(path) def to_dict(self, attrs=None): diff --git a/appmap/__init__.py b/appmap/__init__.py index a60e1af4..60a1ea7d 100644 --- a/appmap/__init__.py +++ b/appmap/__init__.py @@ -13,7 +13,7 @@ if _enabled is not None: # Use setdefault so tests can manage settings as necessary os.environ.setdefault("_APPMAP", _enabled) - _display_params = os.environ.get("APPMAP_DISPLAY_PARAMS", "false") + _display_params = os.environ.get("APPMAP_DISPLAY_PARAMS", "labeled") os.environ.setdefault("_APPMAP_DISPLAY_PARAMS", _display_params) from _appmap import generation # noqa: F401 @@ -58,7 +58,7 @@ def enabled(): os.environ.pop("_APPMAP_DISPLAY_PARAMS", None) else: os.environ.setdefault("_APPMAP", "false") - os.environ.setdefault("_APPMAP_DISPLAY_PARAMS", "false") + os.environ.setdefault("_APPMAP_DISPLAY_PARAMS", "labeled") if not _recording_exported: # Client code that imports appmap.Recording should run correctly From c5efa3e256e738889571ccc1319e7e7cc82e225a Mon Sep 17 00:00:00 2001 From: appland-release Date: Sat, 4 Apr 2026 16:35:24 +0000 Subject: [PATCH 106/113] chore(release): 2.2.0 [skip ci] # [2.2.0](https://github.com/getappmap/appmap-python/compare/v2.1.9...v2.2.0) (2026-04-04) ### Features * Capture argument values of labeled functions by default ([453b697](https://github.com/getappmap/appmap-python/commit/453b697512f04d235e09629b582d1a12f1dac2cc)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1faf2ab..6b3a8f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [2.2.0](https://github.com/getappmap/appmap-python/compare/v2.1.9...v2.2.0) (2026-04-04) + + +### Features + +* Capture argument values of labeled functions by default ([453b697](https://github.com/getappmap/appmap-python/commit/453b697512f04d235e09629b582d1a12f1dac2cc)) + ## [2.1.9](https://github.com/getappmap/appmap-python/compare/v2.1.8...v2.1.9) (2026-02-03) diff --git a/pyproject.toml b/pyproject.toml index 77004c01..113ea045 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "appmap" -version = "2.1.9" +version = "2.2.0" description = "Create AppMap files by recording a Python application." readme = "README.md" requires-python = ">=3.8" From c4d7d17b6bbdbc83749e0e92b5cfea9d3c6e0220 Mon Sep 17 00:00:00 2001 From: kgilpin Date: Tue, 31 Mar 2026 19:53:37 -0400 Subject: [PATCH 107/113] docs: Add CLAUDE.md with test running instructions Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..848fa393 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,28 @@ +# appmap-python + +Python agent for AppMap. Records function calls, HTTP requests, SQL queries, parameters, return values, and exceptions into `.appmap.json` files. + +## Running tests + +Tests must be run via `tox` or the `appmap-python` wrapper, not bare `pytest`. The wrapper sets `APPMAP=true`, which is required for conditional imports in `appmap/__init__.py` (e.g. `generation`). Subprocess-based tests also need the `appmap-python` script in PATH. + +```sh +# Correct - via tox (how CI runs them) +tox + +# Correct - via appmap-python wrapper +appmap-python pytest + +# Also works for quick local iteration on non-subprocess tests +APPMAP=true .venv/bin/python -m pytest _appmap/test/test_events.py + +# WRONG - will fail on subprocess tests +pytest +``` + +## Project structure + +- `appmap/` - Public package entry point (conditional imports based on APPMAP env var) +- `_appmap/` - Internal implementation (event recording, instrumentation, web framework integration) +- `_appmap/test/` - Test suite +- `_appmap/test/data/` - Test fixtures and expected appmap JSON files From f5a6665b952a4fe3e9c2c452789048025c96ffce Mon Sep 17 00:00:00 2001 From: kgilpin Date: Wed, 1 Apr 2026 11:01:33 -0400 Subject: [PATCH 108/113] chore: Test that labeled functions are always recorded --- _appmap/test/test_labels.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/_appmap/test/test_labels.py b/_appmap/test/test_labels.py index a5a11887..8abcc9fb 100644 --- a/_appmap/test/test_labels.py +++ b/_appmap/test/test_labels.py @@ -1,5 +1,6 @@ import pytest +import appmap from _appmap.wrapt import BoundFunctionWrapper, FunctionWrapper @@ -55,6 +56,22 @@ def check_labels(*_): verify_example_appmap(check_labels, "instance_method") + @pytest.mark.appmap_enabled(config="appmap-no-pyyaml.yml") + def test_labeled_function_recorded_without_package(self): + """A labeled function is recorded even when its package is not in the config.""" + import yaml # pylint: disable=import-outside-toplevel + + rec = appmap.Recording() + with rec: + yaml.dump({"key": "value"}) + + # yaml.dump should appear in the recording events because it's labeled + # by the formats preset, even though PyYAML is not in the packages list. + call_events = [e for e in rec.events if e.event == "call"] + assert any( + e.method_id == "dump" and "yaml" in e.defined_class for e in call_events + ), f"Expected yaml.dump in recorded events, got: {[e.method_id for e in call_events]}" + def test_function_only_in_mod(self, verify_example_appmap): def check_labels(*_): # pylint: disable=import-outside-toplevel From 5ee53e74054b9872d3f59d769598f84762b340f5 Mon Sep 17 00:00:00 2001 From: kgilpin Date: Tue, 31 Mar 2026 19:43:28 -0400 Subject: [PATCH 109/113] feat!: Use raw string values instead of repr() for str types in display_string BREAKING CHANGE: String values in appmap events are now recorded verbatim (e.g. "hello") rather than as Python repr (e.g. "'hello'"). This affects parameters, return values, and HTTP message fields of type builtins.str. The class field already identifies the type, so repr-quoting was redundant. Using raw string values also enables proper secret leak detection, since recorded values now match what appears in log messages. Co-Authored-By: Claude Opus 4.6 (1M context) --- _appmap/event.py | 9 +++++---- _appmap/test/data/expected.appmap.json | 18 ++++++++--------- .../pytest-numpy1-no-test-cases.appmap.json | 8 ++++---- .../pytest/expected/pytest-numpy1.appmap.json | 8 ++++---- .../pytest-numpy2-no-test-cases.appmap.json | 8 ++++---- .../pytest/expected/pytest-numpy2.appmap.json | 8 ++++---- .../data/unittest/expected/pytest.appmap.json | 20 ++++++++++--------- .../unittest-no-test-cases.appmap.json | 10 +++++----- .../unittest/expected/unittest.appmap.json | 20 ++++++++++--------- _appmap/test/test_events.py | 6 +++--- _appmap/test/test_http.py | 4 ++-- _appmap/test/test_params.py | 8 ++++---- _appmap/test/web_framework.py | 16 +++++++-------- 13 files changed, 74 insertions(+), 69 deletions(-) diff --git a/_appmap/event.py b/_appmap/event.py index 4bdcf3af..9dc446ea 100644 --- a/_appmap/event.py +++ b/_appmap/event.py @@ -47,13 +47,14 @@ def reset(cls): def display_string(val, display_value=False): # If we're asked to display parameters, make a best-effort attempt - # to get a string value for the parameter using repr(). If parameter - # display is disabled, or repr() has raised, just formulate a value - # from the class and id. + # to get a string value for the parameter. str types are returned as-is; + # other types use repr(). If parameter display is disabled, or repr() has + # raised, just formulate a value from the class and id. value = None if display_value: try: - value = repr(val) + # Use issubclass(type()) instead of isinstance() to avoid side effects on lazy objects + value = val if issubclass(type(val), str) else repr(val) except Exception: # pylint: disable=broad-except pass diff --git a/_appmap/test/data/expected.appmap.json b/_appmap/test/data/expected.appmap.json index 311430be..9e691cb5 100644 --- a/_appmap/test/data/expected.appmap.json +++ b/_appmap/test/data/expected.appmap.json @@ -23,7 +23,7 @@ { "return_value": { "class": "builtins.str", - "value": "'ExampleClass.static_method\\n...\\n'" + "value": "ExampleClass.static_method\n...\n" }, "parent_id": 1, "id": 2, @@ -49,7 +49,7 @@ { "return_value": { "class": "builtins.str", - "value": "'ClassMethodMixin#class_method, cls ExampleClass'" + "value": "ClassMethodMixin#class_method, cls ExampleClass" }, "parent_id": 3, "id": 4, @@ -75,7 +75,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Super#instance_method'" + "value": "Super#instance_method" }, "parent_id": 5, "id": 6, @@ -127,7 +127,7 @@ "name": "data", "kind": "req", "class": "builtins.str", - "value": "'ExampleClass.call_yaml'" + "value": "ExampleClass.call_yaml" } ], "id": 10, @@ -144,7 +144,7 @@ "name": "data", "kind": "req", "class": "builtins.str", - "value": "'ExampleClass.call_yaml'" + "value": "ExampleClass.call_yaml" }, { "name": "stream", @@ -176,7 +176,7 @@ { "return_value": { "class": "builtins.str", - "value": "'ExampleClass.call_yaml\\n...\\n'" + "value": "ExampleClass.call_yaml\n...\n" }, "parent_id": 11, "id": 12, @@ -190,7 +190,7 @@ "name": "data", "kind": "req", "class": "builtins.str", - "value": "'ExampleClass.call_yaml'" + "value": "ExampleClass.call_yaml" }, { "name": "stream", @@ -222,7 +222,7 @@ { "return_value": { "class": "builtins.str", - "value": "'ExampleClass.call_yaml\\n...\\n'" + "value": "ExampleClass.call_yaml\n...\n" }, "parent_id": 13, "id": 14, @@ -334,4 +334,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json index bc711efa..2ffc2d4c 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy1-no-test-cases.appmap.json @@ -56,7 +56,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello'" + "value": "Hello" }, "parent_id": 2, "id": 3, @@ -83,7 +83,7 @@ { "return_value": { "class": "builtins.str", - "value": "'world!'" + "value": "world!" }, "parent_id": 4, "id": 5, @@ -93,7 +93,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello world!'" + "value": "Hello world!" }, "parent_id": 1, "id": 6, @@ -239,4 +239,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json index 6a90f1bc..b8e4f595 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy1.appmap.json @@ -66,7 +66,7 @@ }, { "return_value": { - "value": "'Hello'", + "value": "Hello", "class": "builtins.str" }, "parent_id": 3, @@ -93,7 +93,7 @@ }, { "return_value": { - "value": "'world!'", + "value": "world!", "class": "builtins.str" }, "parent_id": 5, @@ -103,7 +103,7 @@ }, { "return_value": { - "value": "'Hello world!'", + "value": "Hello world!", "class": "builtins.str" }, "parent_id": 2, @@ -278,4 +278,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json index b6d96002..c6c93ef4 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy2-no-test-cases.appmap.json @@ -56,7 +56,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello'" + "value": "Hello" }, "parent_id": 2, "id": 3, @@ -83,7 +83,7 @@ { "return_value": { "class": "builtins.str", - "value": "'world!'" + "value": "world!" }, "parent_id": 4, "id": 5, @@ -93,7 +93,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello world!'" + "value": "Hello world!" }, "parent_id": 1, "id": 6, @@ -239,4 +239,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json index 8d6436b4..b4367584 100644 --- a/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json +++ b/_appmap/test/data/pytest/expected/pytest-numpy2.appmap.json @@ -66,7 +66,7 @@ }, { "return_value": { - "value": "'Hello'", + "value": "Hello", "class": "builtins.str" }, "parent_id": 3, @@ -93,7 +93,7 @@ }, { "return_value": { - "value": "'world!'", + "value": "world!", "class": "builtins.str" }, "parent_id": 5, @@ -103,7 +103,7 @@ }, { "return_value": { - "value": "'Hello world!'", + "value": "Hello world!", "class": "builtins.str" }, "parent_id": 2, @@ -278,4 +278,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/unittest/expected/pytest.appmap.json b/_appmap/test/data/unittest/expected/pytest.appmap.json index 972cf161..57a69bb5 100644 --- a/_appmap/test/data/unittest/expected/pytest.appmap.json +++ b/_appmap/test/data/unittest/expected/pytest.appmap.json @@ -53,12 +53,14 @@ "class": "simple.Simple", "value": "" }, - "parameters": [{ - "class": "builtins.str", - "kind": "req", - "name": "bang", - "value": "'!'" - }], + "parameters": [ + { + "class": "builtins.str", + "kind": "req", + "name": "bang", + "value": "!" + } + ], "id": 2, "event": "call", "thread_id": 1 @@ -83,7 +85,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello'" + "value": "Hello" }, "parent_id": 3, "id": 4, @@ -110,7 +112,7 @@ { "return_value": { "class": "builtins.str", - "value": "'world'" + "value": "world" }, "parent_id": 5, "id": 6, @@ -120,7 +122,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello world!'" + "value": "Hello world!" }, "parent_id": 2, "id": 7, diff --git a/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json b/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json index 2880ba33..f5f5e727 100644 --- a/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json +++ b/_appmap/test/data/unittest/expected/unittest-no-test-cases.appmap.json @@ -35,7 +35,7 @@ "parameters": [ { "kind": "req", - "value": "'!'", + "value": "!", "name": "bang", "class": "builtins.str" } @@ -67,7 +67,7 @@ }, { "return_value": { - "value": "'Hello'", + "value": "Hello", "class": "builtins.str" }, "parent_id": 2, @@ -94,7 +94,7 @@ }, { "return_value": { - "value": "'world'", + "value": "world", "class": "builtins.str" }, "parent_id": 4, @@ -104,7 +104,7 @@ }, { "return_value": { - "value": "'Hello world!'", + "value": "Hello world!", "class": "builtins.str" }, "parent_id": 1, @@ -145,4 +145,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/_appmap/test/data/unittest/expected/unittest.appmap.json b/_appmap/test/data/unittest/expected/unittest.appmap.json index f3fa7526..a4081904 100644 --- a/_appmap/test/data/unittest/expected/unittest.appmap.json +++ b/_appmap/test/data/unittest/expected/unittest.appmap.json @@ -53,12 +53,14 @@ "class": "simple.Simple", "value": "" }, - "parameters": [{ - "class": "builtins.str", - "kind": "req", - "name": "bang", - "value": "'!'" - }], + "parameters": [ + { + "class": "builtins.str", + "kind": "req", + "name": "bang", + "value": "!" + } + ], "id": 2, "event": "call", "thread_id": 1 @@ -83,7 +85,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello'" + "value": "Hello" }, "parent_id": 3, "id": 4, @@ -110,7 +112,7 @@ { "return_value": { "class": "builtins.str", - "value": "'world'" + "value": "world" }, "parent_id": 5, "id": 6, @@ -120,7 +122,7 @@ { "return_value": { "class": "builtins.str", - "value": "'Hello world!'" + "value": "Hello world!" }, "parent_id": 2, "id": 7, diff --git a/_appmap/test/test_events.py b/_appmap/test/test_events.py index cbab31e8..65142e68 100644 --- a/_appmap/test/test_events.py +++ b/_appmap/test/test_events.py @@ -132,11 +132,11 @@ def test_labeled_params_displayed_by_default(self): assert result == "hello" call_event = r.events[0] - # Parameter value should be the repr, not the opaque object string - assert call_event.parameters[0]["value"] == "'hello'" + # Parameter value should be the raw string, not repr-quoted + assert call_event.parameters[0]["value"] == "hello" # Return value should also be displayed return_event = r.events[1] - assert return_event.return_value["value"] == "'hello'" + assert return_event.return_value["value"] == "hello" # Unlabeled method should not have its params displayed, even in the same recording call_event_unlabeled = r.events[2] diff --git a/_appmap/test/test_http.py b/_appmap/test/test_http.py index b0cc048b..92ea305a 100644 --- a/_appmap/test/test_http.py +++ b/_appmap/test/test_http.py @@ -29,8 +29,8 @@ def test_http_client_capture(mock_requests, events): } message = request.message assert message[0] == DictIncluding({"name": "q", "value": "['one', 'two']"}) - assert (message[1] == DictIncluding({"name": "q2", "value": "'🦠'"})) or ( - message[1] == DictIncluding({"name": "q2", "value": "'\\U0001f9a0'"}) + assert (message[1] == DictIncluding({"name": "q2", "value": "🦠"})) or ( + message[1] == DictIncluding({"name": "q2", "value": "\\U0001f9a0"}) ) assert events[3].http_client_response == DictIncluding( diff --git a/_appmap/test/test_params.py b/_appmap/test/test_params.py index 7f6d3436..bf836dcd 100644 --- a/_appmap/test/test_params.py +++ b/_appmap/test/test_params.py @@ -108,7 +108,7 @@ def test_one_param(self, params): "name": "p", "class": "builtins.str", "kind": "req", - "value": "'static'", + "value": "static", } @@ -131,7 +131,7 @@ def test_one_param(self, params): self.assert_parameter( evt, 0, - {"name": "p", "class": "builtins.str", "kind": "req", "value": "'cls'"}, + {"name": "p", "class": "builtins.str", "kind": "req", "value": "cls"}, ) @@ -145,7 +145,7 @@ def test_no_args(self, params): @pytest.mark.parametrize( "params,arg,expected", [ - ("one", "world", ("builtins.str", "'world'")), + ("one", "world", ("builtins.str", "world")), ("one", None, ("builtins.NoneType", "None")), ], indirect=["params"], @@ -192,7 +192,7 @@ def test_one_receiver_none(self, params): @pytest.mark.parametrize( "params,arg,expected", [ - ("one", "world", ("builtins.str", "'world'")), + ("one", "world", ("builtins.str", "world")), ("one", None, ("builtins.NoneType", "None")), ], indirect=["params"], diff --git a/_appmap/test/web_framework.py b/_appmap/test/web_framework.py index 0729cace..425088ed 100644 --- a/_appmap/test/web_framework.py +++ b/_appmap/test/web_framework.py @@ -45,7 +45,7 @@ def test_post_bad_json(events, client, bad_json): ) assert events[0].message == [ - DictIncluding({"name": "my_param", "class": "builtins.str", "value": "'example'"}) + DictIncluding({"name": "my_param", "class": "builtins.str", "value": "example"}) ] @staticmethod @@ -53,7 +53,7 @@ def test_post_multipart(events, client): client.post("/test", data={"my_param": "example"}, content_type="multipart/form-data") assert events[0].message == [ - DictIncluding({"name": "my_param", "class": "builtins.str", "value": "'example'"}) + DictIncluding({"name": "my_param", "class": "builtins.str", "value": "example"}) ] @@ -119,7 +119,7 @@ def test_post(events, client): assert events[0].message == [ DictIncluding( - {"name": "my_param", "class": "builtins.str", "value": "'example'"} + {"name": "my_param", "class": "builtins.str", "value": "example"} ) ] assert events[0].http_server_request == DictIncluding( @@ -142,7 +142,7 @@ def test_get(events, client): assert events[0].message == [ DictIncluding( - {"name": "my_param", "class": "builtins.str", "value": "'example'"} + {"name": "my_param", "class": "builtins.str", "value": "example"} ) ] @@ -166,7 +166,7 @@ def test_put(events, client): assert events[0].message == [ DictIncluding( - {"name": "my_param", "class": "builtins.str", "value": "'example'"} + {"name": "my_param", "class": "builtins.str", "value": "example"} ) ] @@ -205,7 +205,7 @@ def test_message_path_segments(events, client): assert events[0].message == [ DictIncluding( - {"name": "username", "class": "builtins.str", "value": "'alice'"} + {"name": "username", "class": "builtins.str", "value": "alice"} ), DictIncluding({"name": "post_id", "class": "builtins.int", "value": "42"}), ] @@ -222,7 +222,7 @@ def test_post_form_urlencoded(events, client): ) assert events[0].message == [ - DictIncluding({"name": "my_param", "class": "builtins.str", "value": "'example'"}) + DictIncluding({"name": "my_param", "class": "builtins.str", "value": "example"}) ] @staticmethod @@ -230,7 +230,7 @@ def test_post_multipart(events, client): client.post("/test", data={"my_param": "example"}, content_type="multipart/form-data") assert events[0].message == [ - DictIncluding({"name": "my_param", "class": "builtins.str", "value": "'example'"}) + DictIncluding({"name": "my_param", "class": "builtins.str", "value": "example"}) ] From 870565102b6a98c28c518424b34f70f4ada91b17 Mon Sep 17 00:00:00 2001 From: appland-release Date: Tue, 14 Apr 2026 12:49:37 +0000 Subject: [PATCH 110/113] chore(release): 3.0.0 [skip ci] # [3.0.0](https://github.com/getappmap/appmap-python/compare/v2.2.0...v3.0.0) (2026-04-14) * feat!: Use raw string values instead of repr() for str types in display_string ([5ee53e7](https://github.com/getappmap/appmap-python/commit/5ee53e74054b9872d3f59d769598f84762b340f5)) ### BREAKING CHANGES * String values in appmap events are now recorded verbatim (e.g. "hello") rather than as Python repr (e.g. "'hello'"). This affects parameters, return values, and HTTP message fields of type builtins.str. The class field already identifies the type, so repr-quoting was redundant. Using raw string values also enables proper secret leak detection, since recorded values now match what appears in log messages. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 18 ++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3a8f23..a7b6137f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# [3.0.0](https://github.com/getappmap/appmap-python/compare/v2.2.0...v3.0.0) (2026-04-14) + + +* feat!: Use raw string values instead of repr() for str types in display_string ([5ee53e7](https://github.com/getappmap/appmap-python/commit/5ee53e74054b9872d3f59d769598f84762b340f5)) + + +### BREAKING CHANGES + +* String values in appmap events are now recorded verbatim +(e.g. "hello") rather than as Python repr (e.g. "'hello'"). This affects +parameters, return values, and HTTP message fields of type builtins.str. +The class field already identifies the type, so repr-quoting was redundant. + +Using raw string values also enables proper secret leak detection, since +recorded values now match what appears in log messages. + +Co-Authored-By: Claude Opus 4.6 (1M context) + # [2.2.0](https://github.com/getappmap/appmap-python/compare/v2.1.9...v2.2.0) (2026-04-04) diff --git a/pyproject.toml b/pyproject.toml index 113ea045..6dede8d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "appmap" -version = "2.2.0" +version = "3.0.0" description = "Create AppMap files by recording a Python application." readme = "README.md" requires-python = ">=3.8" From 60a0c2e16acaf17878b4a983c51c3ed26a41a58f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 20:11:31 +0000 Subject: [PATCH 111/113] fix: don't create log file by default, never log full environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user hit a real security incident: the agent silently wrote appmap.log (including a full dump of os.environ, i.e. any secrets in the process environment) to disk, and the file got accidentally committed to git. Two bugs combined to cause this: - appmap-python (the wrapper script) had a bug where APPMAP_DISABLE_LOG_FILE was always set to "false", regardless of the --enable-log/--no-enable-log flag — it checked a namespace key (no_enable_log) that never actually exists, so the log file was always created even though --help documents the flag's default as False. - _appmap/env.py independently defaulted to creating the log file (APPMAP_DISABLE_LOG_FILE defaulting to "false") when the wrapper wasn't used at all. - _appmap/configuration.py logged the entire os.environ at startup, which is how arbitrary secrets ended up in the file once one existed. Changes: - appmap/command/runner.py: fixed --enable-log/--no-enable-log to actually control APPMAP_DISABLE_LOG_FILE. - _appmap/env.py: APPMAP_DISABLE_LOG_FILE now defaults to true — no log file unless a user explicitly opts in. - _appmap/configuration.py: only the exact APPMAP/_APPMAP settings and APPMAP_*/_APPMAP_* prefixed settings are logged at startup, never the full environment or unrelated variables that merely share the prefix (e.g. APPMAPX_*). - ci/tests/smoketest.sh: updated the read-only-log-file smoketest to explicitly opt in to log file creation, since it's no longer on by default. - _appmap/test/test_runner.py: added coverage locking in the corrected --enable-log behavior. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014Nw1eEFHvABs5hdjMej5bf --- _appmap/configuration.py | 10 +++++++++- _appmap/env.py | 5 ++++- _appmap/test/test_runner.py | 14 ++++++++++++++ appmap/command/runner.py | 2 +- ci/tests/smoketest.sh | 4 +++- 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/_appmap/configuration.py b/_appmap/configuration.py index ed1965e3..ab370436 100644 --- a/_appmap/configuration.py +++ b/_appmap/configuration.py @@ -508,5 +508,13 @@ def initialize(): logger.info("file: %s", c._file if c.file_present else "[no appmap.yml]") logger.info("config: %r", c) logger.debug("package_functions: %s", c.package_functions) - logger.info("env: %r", os.environ) + # Only log AppMap's own settings, never the full environment: arbitrary + # environment variables (API keys, credentials, tokens, etc.) must never + # end up in application logs. + appmap_env = { + k: v + for k, v in os.environ.items() + if k in ("APPMAP", "_APPMAP") or k.startswith(("APPMAP_", "_APPMAP_")) + } + logger.info("env: %r", appmap_env) os.environ["_APPMAP_MESSAGES_SHOWN"] = "true" diff --git a/_appmap/env.py b/_appmap/env.py index be23a61d..09844269 100644 --- a/_appmap/env.py +++ b/_appmap/env.py @@ -168,7 +168,10 @@ def _configure_logging(self): trace_logger.install() log_level = self.get("APPMAP_LOG_LEVEL", "warn").upper() - disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "false").upper() != "FALSE" + # No log file unless the user opts in: it can contain data (e.g. + # rendered parameter values) that shouldn't be written to disk or + # committed to source control by default. + disable_log = os.environ.get("APPMAP_DISABLE_LOG_FILE", "true").upper() != "FALSE" log_config = self.get("APPMAP_LOG_CONFIG") config_dict = { "version": 1, diff --git a/_appmap/test/test_runner.py b/_appmap/test/test_runner.py index 140df2e2..be1e9831 100644 --- a/_appmap/test/test_runner.py +++ b/_appmap/test/test_runner.py @@ -37,6 +37,20 @@ def test_runner_multi_recording_type(script_runner, flag, expected): assert len(re.findall("(?m)^APPMAP_RECORD_PYTEST=true$", result.stdout)) == expected +@pytest.mark.parametrize( + "flags,expected", + [ + ([], "true"), + (["--no-enable-log"], "true"), + (["--enable-log"], "false"), + ], +) +def test_runner_log_file_disabled_by_default(script_runner, flags, expected): + result = script_runner.run(["appmap-python", *flags, "--record", "process"]) + assert result.returncode == 0 + assert re.search(f"(?m)^APPMAP_DISABLE_LOG_FILE={expected}$", result.stdout) is not None + + @pytest.mark.script_launch_mode("subprocess") class TestEnv: def test_appmap_present(self, script_runner): diff --git a/appmap/command/runner.py b/appmap/command/runner.py index dd5a8e4b..79d2db9a 100644 --- a/appmap/command/runner.py +++ b/appmap/command/runner.py @@ -122,7 +122,7 @@ def run(): envvars[f"APPMAP_RECORD_{disabled.upper()}"] = "false" envvars["APPMAP_DISABLE_LOG_FILE"] = ( - "true" if parsed_args.get("no_enable_log", set()) else "false" + "false" if parsed_args.get("enable_log", False) else "true" ) if len(cmd) == 0: diff --git a/ci/tests/smoketest.sh b/ci/tests/smoketest.sh index cd0abade..f7f8dc7a 100755 --- a/ci/tests/smoketest.sh +++ b/ci/tests/smoketest.sh @@ -26,7 +26,9 @@ test_log_file_not_writable() import appmap EOF - python test_log_file_not_writable.py + # Log file creation is opt-in, so force it on to exercise the fallback + # when the log file can't be created (e.g. read-only mount). + APPMAP_DISABLE_LOG_FILE=false python test_log_file_not_writable.py if [[ $? -eq 0 ]]; then echo 'Script executed successfully' From 6594576fd2230128852eda6e35ac712e646d9a5e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 20:11:42 +0000 Subject: [PATCH 112/113] fix: don't leak the wrapper's own internal state into the child process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit appmap-python is a plain Python script, so it gets self-instrumented at interpreter startup via appmap.pth (AppMap instruments any process by default), before runner.py's own code has computed the environment the target command should actually run with. That incidental self-init writes internal, process-scoped state under _APPMAP*-prefixed env var names using whatever it inherited — notably _APPMAP_MESSAGES_SHOWN, the once-per-process guard around the startup config-dump log lines. Since os.execvpe inherits the current environment, that stale marker carried into the exec'd child, so anyone using the wrapper (e.g. `appmap-python --enable-log flask run`) never saw the config-dump log lines, even with everything else configured correctly: the wrapper's own throwaway initialization had already tripped the "already shown" guard before the child process's real initialization ran. Strip all _APPMAP*-prefixed env vars before exec'ing the child and let the already-computed envvars re-set whatever it actually needs, so the child always starts from a clean slate regardless of what the wrapper's incidental self-init happened to do. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014Nw1eEFHvABs5hdjMej5bf --- _appmap/test/test_runner.py | 17 +++++++++++++++++ appmap/command/runner.py | 12 +++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/_appmap/test/test_runner.py b/_appmap/test/test_runner.py index be1e9831..118c60e3 100644 --- a/_appmap/test/test_runner.py +++ b/_appmap/test/test_runner.py @@ -1,3 +1,4 @@ +import os import re import pytest @@ -64,3 +65,19 @@ def test_recording_type_present(self, script_runner): ) assert result.returncode == 0 assert re.match(r"true", result.stdout) is not None + + def test_internal_state_not_leaked_to_child(self, script_runner): + # appmap-python is itself instrumented at interpreter startup (via + # appmap.pth), which can set internal, process-scoped _APPMAP* + # markers using whatever it inherited, before this script has + # computed the environment the child command should actually run + # with. Simulate that by pre-setting one such marker (the + # once-per-process "startup messages already shown" guard) and + # confirm it doesn't leak into the child's environment, which would + # otherwise silently suppress the child's own startup logging. + env = {**os.environ, "_APPMAP_MESSAGES_SHOWN": "true"} + result = script_runner.run( + ["appmap-python", "printenv", "_APPMAP_MESSAGES_SHOWN"], env=env + ) + assert result.returncode != 0 + assert result.stdout == "" diff --git a/appmap/command/runner.py b/appmap/command/runner.py index 79d2db9a..a5c0bbe8 100644 --- a/appmap/command/runner.py +++ b/appmap/command/runner.py @@ -130,7 +130,17 @@ def run(): print(f"{k}={v}") sys.exit(0) - os.execvpe(cmd[0], cmd, {**os.environ, **envvars}) + # appmap-python is itself instrumented on interpreter startup (via + # appmap.pth), before this point, using whatever environment it inherited + # rather than the envvars computed above. That incidental self-init can + # set internal, process-scoped state under _APPMAP*-prefixed names (e.g. + # the once-per-process "startup messages already shown" guard); left in + # place, it would carry over into the child's environment and suppress + # or corrupt the child's own startup behavior. Drop all of it and let + # envvars below re-set whatever the child actually needs. + child_env = {k: v for k, v in os.environ.items() if not k.startswith("_APPMAP")} + child_env.update(envvars) + os.execvpe(cmd[0], cmd, child_env) if __name__ == "__main__": From 352c573a68aedf99cb1b16eae92b4337aab30d05 Mon Sep 17 00:00:00 2001 From: appland-release Date: Fri, 10 Jul 2026 10:54:05 +0000 Subject: [PATCH 113/113] chore(release): 3.0.1 [skip ci] ## [3.0.1](https://github.com/getappmap/appmap-python/compare/v3.0.0...v3.0.1) (2026-07-10) ### Bug Fixes * don't create log file by default, never log full environment ([60a0c2e](https://github.com/getappmap/appmap-python/commit/60a0c2e16acaf17878b4a983c51c3ed26a41a58f)) * don't leak the wrapper's own internal state into the child process ([6594576](https://github.com/getappmap/appmap-python/commit/6594576fd2230128852eda6e35ac712e646d9a5e)) --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b6137f..a201a947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [3.0.1](https://github.com/getappmap/appmap-python/compare/v3.0.0...v3.0.1) (2026-07-10) + + +### Bug Fixes + +* don't create log file by default, never log full environment ([60a0c2e](https://github.com/getappmap/appmap-python/commit/60a0c2e16acaf17878b4a983c51c3ed26a41a58f)) +* don't leak the wrapper's own internal state into the child process ([6594576](https://github.com/getappmap/appmap-python/commit/6594576fd2230128852eda6e35ac712e646d9a5e)) + # [3.0.0](https://github.com/getappmap/appmap-python/compare/v2.2.0...v3.0.0) (2026-04-14) diff --git a/pyproject.toml b/pyproject.toml index 6dede8d2..55e8cddd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "appmap" -version = "3.0.0" +version = "3.0.1" description = "Create AppMap files by recording a Python application." readme = "README.md" requires-python = ">=3.8"