diff --git a/README.rst b/README.rst index f82a4bb9..8952eb53 100644 --- a/README.rst +++ b/README.rst @@ -46,8 +46,8 @@ Overview **Commit Check** is a lightweight policy engine for Git commit metadata. It validates commit messages, branch names, author identity, signoff trailers, -and push safety ā€” using one versioned TOML policy across local hooks, CI, -GitHub Actions, and AI automation. +AI attribution policy, and push safety ā€” using one versioned TOML policy across +local hooks, CI, GitHub Actions, and AI automation. - **One policy file:** ``cchk.toml`` - **Multiple enforcement points:** CLI, pre-commit, CI / GitHub Actions @@ -139,6 +139,9 @@ To customize the behavior, create a configuration file named ``cchk.toml`` or `` require_signed_off_by = false # Bypass checks for bot/automation authors and co-authors: ignore_authors = ["dependabot[bot]", "renovate[bot]", "copilot[bot]"] + # AI attribution policy: "ignore" (default) or "forbid" + # "forbid" rejects commits with known AI tool signatures + ai_attribution = "forbid" [branch] # https://conventionalbranch.org diff --git a/commit_check/__init__.py b/commit_check/__init__.py index 4ad25d48..fb2cffbb 100644 --- a/commit_check/__init__.py +++ b/commit_check/__init__.py @@ -72,5 +72,8 @@ "require_signed_off_by": False, } +# AI attribution defaults +DEFAULT_AI_ATTRIBUTION = "ignore" # "ignore" | "forbid" + __version__ = version("commit-check") diff --git a/commit_check/ai_signatures.py b/commit_check/ai_signatures.py new file mode 100644 index 00000000..022dabe3 --- /dev/null +++ b/commit_check/ai_signatures.py @@ -0,0 +1,76 @@ +"""AI tool signature detection logic. + +This module provides the public API for detecting AI tool signatures in commit +messages. The signature data (tool definitions and patterns) lives in +:mod:`commit_check.ai_signatures_data`. + +Typical usage:: + + from commit_check.ai_signatures import detect_ai_signatures + + result = detect_ai_signatures( + "feat: init\\n\\nCo-authored-by: Claude " + ) +""" + +from __future__ import annotations + +import re + +from commit_check.ai_signatures_data import ALL_KNOWN_TOOLS as _ALL_KNOWN_TOOLS + +# Re-export for convenience ā€” consumers can import everything from +# commit_check.ai_signatures without knowing about the data/logic split. +ALL_KNOWN_TOOLS = _ALL_KNOWN_TOOLS + + +#: Flat list of all compiled patterns for bulk scanning. +#: Each tuple is ``(regex, tool_name, description, kind)``. +ALL_PATTERNS: list[tuple[re.Pattern[str], str, str, str]] = [ + (p.regex, tool.name, p.description, p.kind) + for tool in ALL_KNOWN_TOOLS + for p in tool.patterns +] + + +def detect_ai_signatures(message: str) -> list[dict[str, str]]: + """Scan *message* for known AI tool signatures. + + :param message: The full commit message (subject + body) to scan. + :returns: A list of dicts, one per matched signature, each with keys + ``"tool"``, ``"kind"``, ``"description"``, and ``"matched_text"``. + Returns an empty list when no signatures are found. + + Example:: + + >>> detect_ai_signatures( + ... "feat: init\\n\\nCo-authored-by: Claude " + ... ) + [{'tool': 'Claude Code', 'kind': 'trailer', ...}] + """ + results: list[dict[str, str]] = [] + seen: set[str] = set() + + for regex, tool_name, desc, kind in ALL_PATTERNS: + for match in regex.finditer(message): + matched = match.group(0).strip() + if matched not in seen: + seen.add(matched) + results.append( + { + "tool": tool_name, + "kind": kind, + "description": desc, + "matched_text": matched, + } + ) + + return results + + +def has_ai_signature(message: str) -> bool: + """Return ``True`` if *message* contains any known AI signature.""" + for regex, _tool_name, _desc, _kind in ALL_PATTERNS: + if regex.search(message): + return True + return False diff --git a/commit_check/ai_signatures_data.py b/commit_check/ai_signatures_data.py new file mode 100644 index 00000000..2924eea9 --- /dev/null +++ b/commit_check/ai_signatures_data.py @@ -0,0 +1,257 @@ +"""Known AI tool signatures ā€” pure data, no detection logic. + +This module defines the data structures and the curated registry of known AI +coding tool signatures. To add a new tool, define a ``KnownAiTool`` entry +with its patterns and add it to ``ALL_KNOWN_TOOLS``. + +The detection logic lives in :mod:`commit_check.ai_signatures`. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class AiSignaturePattern: + """A single pattern that identifies AI tool usage in a commit message. + + :param regex: A compiled regex that, if matched anywhere in the commit + message body, indicates the corresponding tool was involved. + :param kind: ``"trailer"`` for structured ``Key: value`` footer lines + (matched case-insensitively), ``"body_marker"`` for any other text + marker. + :param description: Human-readable description of what is matched. + """ + + regex: re.Pattern[str] + kind: str # "trailer" | "body_marker" + description: str = "" + + +@dataclass(frozen=True) +class KnownAiTool: + """A known AI coding tool and its commit-message signatures. + + :param name: Short display name (e.g. ``"Claude Code"``, ``"GitHub Copilot"``). + :param patterns: One or more signature patterns that indicate this tool. + """ + + name: str + patterns: list[AiSignaturePattern] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Pattern helpers +# --------------------------------------------------------------------------- + + +def _trailer( + key: str, value_pattern: str = r".*", description: str = "" +) -> AiSignaturePattern: + """Build a trailer pattern for a structured ``Key: value`` line. + + The match is case-insensitive and anchors the key at the start of a line. + """ + raw = rf"^{re.escape(key)}:\s*{value_pattern}\s*$" + return AiSignaturePattern( + regex=re.compile(raw, re.IGNORECASE | re.MULTILINE), + kind="trailer", + description=description or f"``{key}:`` trailer", + ) + + +def _body_marker(pattern: str, description: str = "") -> AiSignaturePattern: + """Build a free-text body marker pattern.""" + return AiSignaturePattern( + regex=re.compile(pattern, re.MULTILINE), + kind="body_marker", + description=description, + ) + + +# --------------------------------------------------------------------------- +# Known tool signatures +# --------------------------------------------------------------------------- + +# --- Anthropic Claude Code / Claude CLI --- +CLAUDE_CODE = KnownAiTool( + name="Claude Code", + patterns=[ + # Standard Co-authored-by trailer added by Claude Code. + # When an email is present, anchor to known AI noreply addresses + # to avoid false positives with human co-authors named Claude. + _trailer( + "Co-authored-by", + r"Claude(?: Code)?" + r"(?:\s*<(?:noreply@anthropic\.com" + r"|\d+\+Claude@users\.noreply\.github\.com)>)?", + "``Co-authored-by: Claude`` trailer", + ), + # Assisted-by trailer (Linux kernel style, with optional tool list) + _trailer( + "Assisted-by", + r"Claude:\S+(?:\s+\S+)*", + "``Assisted-by: Claude: [tools]`` trailer", + ), + # Body marker: generated-with notice + _body_marker( + r"šŸ¤–\s*Generated\s+(?:with|by)\s+\[?Claude", + "``šŸ¤– Generated with Claude`` body marker", + ), + # Session ID trailer (Claude Code sometimes adds this) + _trailer("Claude-Session", r"\S+", "``Claude-Session:`` trailer"), + # Workflow ID trailer + _trailer("Claude-Workflow", r"\S+", "``Claude-Workflow:`` trailer"), + ], +) + +# --- GitHub Copilot --- +COPILOT = KnownAiTool( + name="GitHub Copilot", + patterns=[ + _trailer( + "Co-authored-by", + r"Copilot" + r"(?:\s*<\d+\+Copilot@users\.noreply\.github\.com>)?", + "``Co-authored-by: Copilot`` trailer", + ), + ], +) + +# --- OpenAI Codex --- +CODEX = KnownAiTool( + name="OpenAI Codex", + patterns=[ + _trailer( + "Co-authored-by", + r"Codex\s*(?:<[^>]*>)?", + "``Co-authored-by: Codex`` trailer", + ), + ], +) + +# --- Gemini (Google) --- +GEMINI = KnownAiTool( + name="Gemini", + patterns=[ + _trailer( + "Co-authored-by", + r"Gemini\s*(?:<[^>]*>)?", + "``Co-authored-by: Gemini`` trailer", + ), + ], +) + +# --- Cursor --- +CURSOR = KnownAiTool( + name="Cursor", + patterns=[ + _trailer( + "Co-authored-by", + r"Cursor\s*(?:<[^>]*>)?", + "``Co-authored-by: Cursor`` trailer", + ), + ], +) + +# --- Devin --- +DEVIN = KnownAiTool( + name="Devin", + patterns=[ + _trailer( + "Co-authored-by", + r"Devin\s*(?:<[^>]*>)?", + "``Co-authored-by: Devin`` trailer", + ), + ], +) + +# --- Aider --- +AIDER = KnownAiTool( + name="Aider", + patterns=[ + _trailer( + "Co-authored-by", + r"Aider\s*(?:<[^>]*>)?", + "``Co-authored-by: Aider`` trailer", + ), + # aider appends "(aider)" to the author name + _trailer( + "Co-authored-by", + r"[^<]+\(aider\)\s*(?:<[^>]*>)?", + "``Co-authored-by: ... (aider)`` trailer", + ), + ], +) + +# --- Windsurf (Codeium) --- +WINDSURF = KnownAiTool( + name="Windsurf", + patterns=[ + _trailer( + "Co-authored-by", + r"Windsurf\s*(?:<[^>]*>)?", + "``Co-authored-by: Windsurf`` trailer", + ), + ], +) + +# --- Tabby --- +TABBY = KnownAiTool( + name="Tabby", + patterns=[ + _trailer( + "Co-authored-by", + r"Tabby\s*(?:<[^>]*>)?", + "``Co-authored-by: Tabby`` trailer", + ), + ], +) + +# --- Generic / catch-all AI patterns --- +GENERIC_AI = KnownAiTool( + name="Generic AI", + patterns=[ + # Catch AI agent model identifiers in Co-authored-by + # (e.g. claude-sonnet-4, gpt-4-turbo, gemini-1.5-pro). + # A hyphenated model suffix is required so bare human first names + # ("Claude", "Gemini") are NOT flagged, regardless of the email. + _trailer( + "Co-authored-by", + r"(?:claude|gpt|gemini)[\w.]*-[\w.-]+(?:\s*<[^>]*>)?", + "``Co-authored-by`` with AI model name", + ), + # Catch Assisted-by trailer (Linux kernel style) regardless of agent, + # with optional trailing tool list. + _trailer( + "Assisted-by", + r"\S+:\S+(?:\s+\S+)*", + "``Assisted-by: : [tools]`` trailer (kernel style)", + ), + # Catch common body markers + _body_marker( + r"^Generated\s+(?:by|with)\s+(?:AI|artificial intelligence)", + "``Generated by AI`` body marker", + ), + ], +) + +# --------------------------------------------------------------------------- +# Master registry ā€” ordered by specificity (most specific first) +# --------------------------------------------------------------------------- + +#: All known AI tools, ordered so that more specific patterns are checked first. +ALL_KNOWN_TOOLS: list[KnownAiTool] = [ + CLAUDE_CODE, + COPILOT, + CODEX, + GEMINI, + CURSOR, + DEVIN, + AIDER, + WINDSURF, + TABBY, + GENERIC_AI, +] diff --git a/commit_check/api.py b/commit_check/api.py index f7876858..0458e68b 100644 --- a/commit_check/api.py +++ b/commit_check/api.py @@ -132,6 +132,7 @@ def validate_message( "allow_empty_commits", "allow_fixup_commits", "allow_wip_commits", + "ai_attribution", ] return _run_checks(check_names, context, cfg) diff --git a/commit_check/config_merger.py b/commit_check/config_merger.py index f801869a..149e9da0 100644 --- a/commit_check/config_merger.py +++ b/commit_check/config_merger.py @@ -13,6 +13,7 @@ DEFAULT_BRANCH_NAMES, DEFAULT_BOOLEAN_RULES, DEFAULT_PUSH_RULES, + DEFAULT_AI_ATTRIBUTION, ) @@ -76,6 +77,7 @@ def get_default_config() -> dict[str, Any]: "require_body": DEFAULT_BOOLEAN_RULES["require_body"], "require_signed_off_by": DEFAULT_BOOLEAN_RULES["require_signed_off_by"], "ignore_authors": [], + "ai_attribution": DEFAULT_AI_ATTRIBUTION, }, "branch": { "conventional_branch": True, @@ -120,6 +122,7 @@ class ConfigMerger: "CCHK_REQUIRE_BODY": ("commit", "require_body", parse_bool), "CCHK_REQUIRE_SIGNED_OFF_BY": ("commit", "require_signed_off_by", parse_bool), "CCHK_IGNORE_AUTHORS": ("commit", "ignore_authors", parse_list), + "CCHK_AI_ATTRIBUTION": ("commit", "ai_attribution", str), # Branch section "CCHK_CONVENTIONAL_BRANCH": ("branch", "conventional_branch", parse_bool), "CCHK_ALLOW_BRANCH_TYPES": ("branch", "allow_branch_types", parse_list), @@ -147,6 +150,7 @@ class ConfigMerger: "require_body": ("commit", "require_body"), "require_signed_off_by": ("commit", "require_signed_off_by"), "ignore_authors": ("commit", "ignore_authors"), + "ai_attribution": ("commit", "ai_attribution"), # Branch section "conventional_branch": ("branch", "conventional_branch"), "allow_branch_types": ("branch", "allow_branch_types"), diff --git a/commit_check/engine.py b/commit_check/engine.py index a475ff82..edb42278 100644 --- a/commit_check/engine.py +++ b/commit_check/engine.py @@ -7,6 +7,9 @@ from dataclasses import field from commit_check.rule_builder import ValidationRule +from commit_check.ai_signatures import ( + detect_ai_signatures, +) from commit_check.util import ( fetch_remote_ref, fetch_upstream_ref, @@ -708,6 +711,62 @@ def _is_wip_commit_allowed(self, message: str) -> bool: return not is_wip or self.rule.value +class AiAttributionValidator(BaseValidator): + """Validates commit messages against AI attribution policy. + + Single responsibility: when configured to ``forbid``, rejects any commit + that contains known AI tool signatures. When set to ``ignore`` (the + default), the check is a no-op. + """ + + def validate(self, context: ValidationContext) -> ValidationResult: + if self._should_skip_commit_validation(context): + return ValidationResult.PASS + + message = self._get_commit_body(context) + if not message: + return ValidationResult.PASS + + policy = self.rule.value # "ignore" | "forbid" + if policy != "forbid": + return ValidationResult.PASS + + signatures = detect_ai_signatures(message) + if not signatures: + return ValidationResult.PASS + + tools = {s["tool"] for s in signatures} + self._record_failure( + value=", ".join(sorted(tools)), + error=f"AI-assisted commit is forbidden ā€” detected tools: {', '.join(sorted(tools))}", + suggest="This project forbids AI-assisted commits. Remove AI trailers and re-commit.", + ) + return ValidationResult.FAIL + + def _record_failure(self, value: str, error: str, suggest: str) -> None: + """Record a failure with dynamic error/suggest messages.""" + self._last_failure = { + "check": self.rule.check, + "value": value, + "error": error, + "suggest": suggest, + } + if not self._suppress_output: + # Pass dynamic messages to the printer by creating a dict with + # the live error/suggest instead of the catalog templates. + rule_dict = self.rule.to_dict() + rule_dict["error"] = error + rule_dict["suggest"] = suggest + from commit_check.util import _print_failure + + _print_failure( + rule_dict, + value, + no_banner=self._no_banner, + compact=self._compact, + ) + + class ValidationEngine: """Main validation engine that orchestrates all validations.""" @@ -730,6 +789,7 @@ class ValidationEngine: "allow_wip_commits": CommitTypeValidator, "ignore_authors": CommitTypeValidator, "no_force_push": ForcePushValidator, + "ai_attribution": AiAttributionValidator, } def __init__(self, rules: list[ValidationRule]): diff --git a/commit_check/main.py b/commit_check/main.py index 3fe46293..c5c814eb 100644 --- a/commit_check/main.py +++ b/commit_check/main.py @@ -293,6 +293,16 @@ def _get_parser() -> argparse.ArgumentParser: help="comma-separated list of authors to ignore for commit checks", ) + commit_group.add_argument( + "--ai-attribution", + type=str, + default=None, + choices=["ignore", "forbid"], + metavar="POLICY", + help="AI attribution policy: ignore (default) or forbid. " + "'forbid' rejects commits with known AI tool signatures.", + ) + # Branch configuration options branch_group = parser.add_argument_group( "branch options", "Configuration options for --branch validation" @@ -429,6 +439,7 @@ def _get_requested_checks(args: argparse.Namespace) -> list[str]: "allow_empty_commits", "allow_fixup_commits", "allow_wip_commits", + "ai_attribution", ] ) if args.branch: diff --git a/commit_check/rule_builder.py b/commit_check/rule_builder.py index 63832b74..cc768a00 100644 --- a/commit_check/rule_builder.py +++ b/commit_check/rule_builder.py @@ -15,6 +15,7 @@ DEFAULT_BRANCH_NAMES, DEFAULT_BOOLEAN_RULES, DEFAULT_PUSH_RULES, + DEFAULT_AI_ATTRIBUTION, ) @@ -138,6 +139,8 @@ def _build_single_rule( return self._build_length_rule(catalog_entry, "subject_min_length") elif check == "ignore_authors": return self._build_author_list_rule(catalog_entry, "ignore_authors") + elif check == "ai_attribution": + return self._build_ai_attribution_rule(catalog_entry) elif check == "merge_base": return self._build_merge_base_rule(catalog_entry) else: @@ -248,6 +251,25 @@ def _build_merge_base_rule( suggest=catalog_entry.suggest, ) + def _build_ai_attribution_rule( + self, catalog_entry: RuleCatalogEntry + ) -> ValidationRule | None: + """Build AI attribution validation rule. + + Only active when policy is ``"forbid"`` ā€” rejects any commit with + known AI tool signatures. + """ + policy = self.commit_config.get("ai_attribution", DEFAULT_AI_ATTRIBUTION) + if policy != "forbid": + return None + + return ValidationRule( + check=catalog_entry.check, + value=policy, + error=catalog_entry.error or "", + suggest=catalog_entry.suggest or "", + ) + def _build_boolean_rule( self, catalog_entry: RuleCatalogEntry, section_config: dict[str, Any] ) -> ValidationRule | None: diff --git a/commit_check/rules_catalog.py b/commit_check/rules_catalog.py index fa470735..108b4134 100644 --- a/commit_check/rules_catalog.py +++ b/commit_check/rules_catalog.py @@ -104,6 +104,12 @@ class RuleCatalogEntry: error="Signed-off-by not found in latest commit", suggest="git commit --amend --signoff or use --signoff on commit", ), + RuleCatalogEntry( + check="ai_attribution", + regex=None, + error="AI attribution policy violation", + suggest="This project forbids AI-assisted commits. Remove AI trailers and re-commit.", + ), ] # Push rules diff --git a/docs/configuration.rst b/docs/configuration.rst index 03184bb7..1a4416d2 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -126,6 +126,7 @@ Example Configuration require_signed_off_by = false # required_signoff_name = "Your Name" # Optional # required_signoff_email = "your.email@example.com" # Optional + ai_attribution = "forbid" # "ignore" (default) or "forbid" ā€” rejects AI tool trailers [push] # Block force pushes when used as a pre-push hook or with --no-force-push @@ -288,6 +289,9 @@ Configuration can also be set via environment variables with the ``CCHK_`` prefi * - ``allow_force_push = true`` - ``CCHK_ALLOW_FORCE_PUSH=false`` - ``--no-force-push`` (enable via ``--no-force-push`` flag) + * - ``ai_attribution = "forbid"`` + - ``CCHK_AI_ATTRIBUTION=forbid`` + - ``--ai-attribution=forbid`` * - ``ignore_authors = ["bot"]`` (in branch section) - ``CCHK_BRANCH_IGNORE_AUTHORS=bot,user`` - ``--branch-ignore-authors=bot,user`` @@ -398,6 +402,11 @@ Options Table Description - bool - false - Require "Signed-off-by" line in the commit message footer. + * - commit + - ai_attribution + - str + - "ignore" + - AI attribution policy. ``"forbid"`` rejects any commit containing known AI tool signatures (Claude Code, Copilot, Codex, Gemini, Cursor, Devin, Aider, Windsurf, Tabby, and generic AI model patterns). ``"ignore"`` disables the check. This feature is a response to the industry-wide discussion on AI disclosure in open source (Linux kernel ``Assisted-by:`` trailer, CPython, VS Code, Apache, Fedora policies). * - branch - conventional_branch - bool diff --git a/docs/what-is-new.rst b/docs/what-is-new.rst index e0a50c65..d9db1cbb 100644 --- a/docs/what-is-new.rst +++ b/docs/what-is-new.rst @@ -3,7 +3,49 @@ What's New This document highlights the major changes and improvements in each version of commit-check. -Version 2.9.1 ā€” Bot Branch Types as Default +Version 2.11.0 ā€” AI Attribution Governance +-------------------------------------------- + +Enforce Your Project's AI Contribution Policy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +commit-check now supports **AI attribution governance** ā€” a neutral enforcement +layer for the industry-wide discussion on AI disclosure in open source. + +Configured under ``[commit]``: + +.. code-block:: toml + + [commit] + # "ignore" (default) | "forbid" + ai_attribution = "forbid" + +When set to ``"forbid"``, any commit containing known AI tool signatures is +rejected. The built-in signature database detects trailers and markers from: + +* **Claude Code** ā€” ``Co-authored-by: Claude``, ``Assisted-by: Claude:...``, + ``šŸ¤– Generated with Claude``, ``Claude-Session:``, ``Claude-Workflow:`` +* **GitHub Copilot** ā€” ``Co-authored-by: Copilot`` +* **OpenAI Codex** ā€” ``Co-authored-by: Codex`` +* **Gemini** ā€” ``Co-authored-by: Gemini`` +* **Cursor** ā€” ``Co-authored-by: Cursor`` +* **Devin** ā€” ``Co-authored-by: Devin`` +* **Aider** ā€” ``Co-authored-by: Aider``, ``Co-authored-by: ... (aider)`` +* **Windsurf** ā€” ``Co-authored-by: Windsurf`` +* **Tabby** ā€” ``Co-authored-by: Tabby`` +* **Generic AI** ā€” ``Assisted-by:`` (Linux kernel style, with tool list), + model names like ``claude-sonnet-4``, ``gpt-4-turbo`` + +The signature database is designed to be extensible ā€” adding a new tool is as +simple as adding a ``KnownAiTool`` entry with the tool's patterns. + +This feature is motivated by ongoing discussions in the CPython core +development community, the Linux kernel's ``Assisted-by:`` trailer standard, +VS Code, Apache, Fedora, and other foundations. + +See `Configuration Documentation `_ for details. + +Version 2.10.0 ā€” Bot Branch Types as Default --------------------------------------------- ``dependabot/`` and ``renovate/`` branches now pass by default diff --git a/tests/ai_signatures_test.py b/tests/ai_signatures_test.py new file mode 100644 index 00000000..4a755e25 --- /dev/null +++ b/tests/ai_signatures_test.py @@ -0,0 +1,308 @@ +"""Tests for commit_check.ai_signatures ā€” the AI tool signature database.""" + +import pytest +from commit_check.ai_signatures import ( + detect_ai_signatures, + has_ai_signature, + ALL_KNOWN_TOOLS, + ALL_PATTERNS, +) + + +class TestDetectAiSignatures: + """Tests for detect_ai_signatures().""" + + @pytest.mark.benchmark + def test_no_signatures_in_clean_commit(self): + """A clean commit message with no AI references returns empty list.""" + message = ( + "feat: add streaming endpoint\n\nSigned-off-by: Alice " + ) + result = detect_ai_signatures(message) + assert result == [] + + @pytest.mark.benchmark + def test_claude_co_author_with_noreply_email(self): + """Co-authored-by: Claude with anthropic noreply is detected.""" + message = ( + "feat: implement feature\n\nCo-authored-by: Claude " + ) + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "Claude Code" for s in result) + + @pytest.mark.benchmark + def test_claude_code_with_github_noreply(self): + """Co-authored-by: Claude with GitHub noreply is detected.""" + message = "feat: implement feature\n\nCo-authored-by: Claude <12345+Claude@users.noreply.github.com>" + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "Claude Code" for s in result) + + @pytest.mark.benchmark + def test_human_claude_with_personal_email_ignored(self): + """A human named Claude with a personal email is NOT detected.""" + message = "feat: add feature\n\nCo-authored-by: Claude Dubois " + result = detect_ai_signatures(message) + claude_hits = [s for s in result if s["tool"] == "Claude Code"] + assert len(claude_hits) == 0 + + @pytest.mark.benchmark + def test_copilot_with_noreply_email(self): + """Co-authored-by: Copilot with GitHub noreply is detected.""" + message = ( + "fix: resolve bug\n\n" + "Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>" + ) + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "GitHub Copilot" for s in result) + + @pytest.mark.benchmark + def test_copilot_bare_name(self): + """Co-authored-by: Copilot (bare, no email) is detected.""" + message = "fix: resolve bug\n\nCo-authored-by: Copilot" + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "GitHub Copilot" for s in result) + + @pytest.mark.benchmark + def test_kernel_format_with_tool_list(self): + """Assisted-by with kernel-style tool list is detected.""" + message = ( + "refactor: clean up API\n\n" + "Assisted-by: Claude:claude-3-opus coccinelle sparse" + ) + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any("Assisted-by" in s["matched_text"] for s in result) + + @pytest.mark.benchmark + def test_kernel_format_simple(self): + """Assisted-by with just tool:model is detected.""" + message = "refactor: clean up API\n\nAssisted-by: Claude:claude-sonnet-4" + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any("Assisted-by" in s["matched_text"] for s in result) + + @pytest.mark.benchmark + def test_multiple_ai_tools_detected(self): + """Multiple AI tool signatures are all detected.""" + message = ( + "feat: implement feature\n\n" + "Co-authored-by: Claude \n" + "Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>" + ) + result = detect_ai_signatures(message) + tools = {s["tool"] for s in result} + assert "Claude Code" in tools + assert "GitHub Copilot" in tools + + @pytest.mark.benchmark + def test_dedup_matched_text(self): + """Duplicate matched text is reported only once.""" + message = "feat: add feature\n\nAssisted-by: Claude:claude-sonnet-4" + result = detect_ai_signatures(message) + matched_texts = [s["matched_text"] for s in result] + assert len(matched_texts) == len(set(matched_texts)) + + @pytest.mark.benchmark + def test_human_co_author_not_detected(self): + """A human Co-authored-by is not flagged.""" + message = "feat: add feature\n\nCo-authored-by: Jane Doe " + result = detect_ai_signatures(message) + # None of the known AI patterns should match a common human name + assert len(result) == 0 + + @pytest.mark.benchmark + def test_claude_session_trailer(self): + """Claude-Session: trailer is detected.""" + message = "feat: update config\n\nClaude-Session: sess_abc123" + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any("Claude-Session" in s["matched_text"] for s in result) + + @pytest.mark.benchmark + def test_emoji_marker_detected(self): + """šŸ¤– Generated with Claude marker is detected.""" + message = "feat: add feature\n\nšŸ¤– Generated with Claude Code" + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any("Claude Code" == s["tool"] for s in result) + + @pytest.mark.benchmark + def test_aider_suffix_pattern(self): + """Co-authored-by with (aider) suffix is detected.""" + message = ( + "feat: add feature\n\nCo-authored-by: Some Dev (aider) " + ) + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "Aider" for s in result) + + @pytest.mark.benchmark + def test_generic_model_name_detected(self): + """Model names like claude-sonnet-4 in Co-authored-by are detected.""" + message = ( + "feat: add feature\n\nCo-authored-by: claude-sonnet-4 " + ) + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "Generic AI" for s in result) + + @pytest.mark.benchmark + def test_generic_gpt_model_detected(self): + """gpt-4-turbo in Co-authored-by is detected.""" + message = "feat: add feature\n\nCo-authored-by: gpt-4-turbo " + result = detect_ai_signatures(message) + assert len(result) >= 1 + assert any(s["tool"] == "Generic AI" for s in result) + + @pytest.mark.benchmark + def test_all_known_tools_have_patterns(self): + """All known tools have at least one pattern.""" + for tool in ALL_KNOWN_TOOLS: + assert len(tool.patterns) >= 1, f"{tool.name} has no patterns" + + @pytest.mark.benchmark + def test_all_patterns_compile(self): + """All patterns in the master registry compile successfully.""" + for regex, tool_name, desc, kind in ALL_PATTERNS: + assert regex is not None, f"{tool_name}: {desc} has None regex" + assert kind in ("trailer", "body_marker"), ( + f"{tool_name}: invalid kind {kind}" + ) + + @pytest.mark.benchmark + def test_kind_field_correct_for_body_marker(self): + """Body markers have kind='body_marker', not 'trailer'.""" + message = "feat: add feature\n\nGenerated by AI" + result = detect_ai_signatures(message) + for r in result: + if r["description"].startswith("``Generated by AI"): + assert r["kind"] == "body_marker", ( + f"Expected body_marker, got {r['kind']}" + ) + + @pytest.mark.benchmark + def test_kind_field_correct_for_trailer(self): + """Trailers have kind='trailer'.""" + message = "feat: add feature\n\nCo-authored-by: Claude " + result = detect_ai_signatures(message) + for r in result: + assert r["kind"] == "trailer", f"Expected trailer, got {r['kind']}" + + +class TestHasAiSignature: + """Tests for has_ai_signature().""" + + @pytest.mark.benchmark + def test_clean_message(self): + """Returns False for a clean commit message.""" + assert has_ai_signature("feat: add feature") is False + + @pytest.mark.benchmark + def test_with_ai_signature(self): + """Returns True when AI signature present.""" + assert ( + has_ai_signature( + "feat: add feature\n\nCo-authored-by: Claude " + ) + is True + ) + + @pytest.mark.benchmark + def test_empty_message(self): + """Returns False for empty message.""" + assert has_ai_signature("") is False + + +class TestHumanNameFalsePositives: + """Human co-authors whose names overlap with AI tool/model tokens. + + The generic model-name pattern requires a hyphenated model suffix + (e.g. ``claude-sonnet-4``), so a bare human first name ā€” even with a + personal email ā€” must never be flagged. + """ + + @pytest.mark.benchmark + @pytest.mark.parametrize( + "trailer", + [ + "Co-authored-by: Claude ", + "Co-authored-by: Gemini Rossi ", + "Co-authored-by: gpt ", + "Co-authored-by: Claude Monet ", + ], + ) + def test_bare_human_name_not_detected(self, trailer): + """A human co-author is not treated as an AI signature.""" + message = f"feat: add feature\n\n{trailer}" + assert detect_ai_signatures(message) == [] + assert has_ai_signature(message) is False + + @pytest.mark.benchmark + @pytest.mark.parametrize( + "trailer", + [ + "Co-authored-by: claude-sonnet-4 ", + "Co-authored-by: gpt-4-turbo ", + "Co-authored-by: gemini-1.5-pro", + ], + ) + def test_model_identifier_still_detected(self, trailer): + """A hyphenated AI model identifier is still caught by Generic AI.""" + message = f"feat: add feature\n\n{trailer}" + assert has_ai_signature(message) is True + + +class TestSignatureDatabase: + """Tests for the structure and completeness of the signature database.""" + + @pytest.mark.benchmark + def test_all_tools_have_unique_names(self): + """All known tools have unique display names.""" + names = [t.name for t in ALL_KNOWN_TOOLS] + assert len(names) == len(set(names)) + + @pytest.mark.benchmark + def test_all_patterns_have_description(self): + """All patterns have a non-empty description.""" + for regex, tool_name, desc, kind in ALL_PATTERNS: + assert desc, f"Pattern for {tool_name} is missing a description" + + @pytest.mark.benchmark + def test_claude_code_variant_detection(self): + """Various Claude Code trailer formats are detected.""" + variants = [ + "Co-authored-by: Claude", + "Co-authored-by: Claude ", + "Co-authored-by: Claude Code ", + "Assisted-by: Claude:claude-sonnet-4-20250514", + "Assisted-by: Claude:claude-3-opus coccinelle sparse", + "Claude-Session: sess_abc123", + "Claude-Workflow: workflow_xyz", + ] + for variant in variants: + message = f"feat: add feature\n\n{variant}" + result = detect_ai_signatures(message) + assert len(result) >= 1, f"Failed to detect: {variant}" + + @pytest.mark.benchmark + def test_generic_ai_catch_all(self): + """Assisted-by with any AI agent is caught by Generic AI.""" + message = "feat: update code\n\nAssisted-by: gpt-4:openai" + result = detect_ai_signatures(message) + assert len(result) >= 1 + + @pytest.mark.benchmark + def test_human_name_not_detected(self): + """Common human names that could overlap with AI tool names.""" + # Devin is both a human name and an AI tool name + message = ( + "feat: add feature\n\nCo-authored-by: Devin Booker " + ) + result = detect_ai_signatures(message) + devin_hits = [s for s in result if s["tool"] == "Devin"] + # With a personal email, Devin should NOT be detected + assert len(devin_hits) == 0 diff --git a/tests/engine_test.py b/tests/engine_test.py index 5261c883..7cea27dd 100644 --- a/tests/engine_test.py +++ b/tests/engine_test.py @@ -21,6 +21,7 @@ BodyValidator, MergeBaseValidator, ForcePushValidator, + AiAttributionValidator, ) from commit_check.rule_builder import ValidationRule @@ -1016,6 +1017,7 @@ def test_validation_engine_validator_map(self): "allow_fixup_commits": CommitTypeValidator, "allow_wip_commits": CommitTypeValidator, "ignore_authors": CommitTypeValidator, + "ai_attribution": AiAttributionValidator, } for check, validator_class in expected_mappings.items(): @@ -1610,3 +1612,93 @@ def test_validation_context_push_upstream_fallback(self): assert ctx.push_upstream_fallback is True ctx2 = ValidationContext() assert ctx2.push_upstream_fallback is False + + +class TestAiAttributionValidator: + """Tests for AiAttributionValidator.""" + + @pytest.mark.benchmark + def test_ignore_policy_always_passes(self): + """ignore policy skips all validation.""" + rule = ValidationRule( + check="ai_attribution", + value="ignore", + ) + validator = AiAttributionValidator(rule) + message = "feat: add feature\n\nCo-authored-by: Claude " + context = ValidationContext(stdin_text=message) + result = validator.validate(context) + assert result == ValidationResult.PASS + + @pytest.mark.benchmark + def test_forbid_policy_rejects_ai_commit(self): + """forbid policy rejects commits with AI signatures.""" + rule = ValidationRule( + check="ai_attribution", + value="forbid", + ) + validator = AiAttributionValidator(rule) + message = "feat: add feature\n\nCo-authored-by: Claude " + context = ValidationContext(stdin_text=message) + result = validator.validate(context) + assert result == ValidationResult.FAIL + + @pytest.mark.benchmark + def test_forbid_policy_allows_clean_commit(self): + """forbid policy allows commits without AI signatures.""" + rule = ValidationRule( + check="ai_attribution", + value="forbid", + ) + validator = AiAttributionValidator(rule) + context = ValidationContext(stdin_text="feat: add feature by hand") + result = validator.validate(context) + assert result == ValidationResult.PASS + + @pytest.mark.benchmark + def test_forbid_policy_multiple_tools(self): + """forbid rejects commits with multiple AI tools.""" + rule = ValidationRule( + check="ai_attribution", + value="forbid", + ) + validator = AiAttributionValidator(rule) + message = ( + "feat: implement feature\n\n" + "Co-authored-by: Claude \n" + "Co-authored-by: Copilot " + ) + context = ValidationContext(stdin_text=message) + result = validator.validate(context) + assert result == ValidationResult.FAIL + + @pytest.mark.benchmark + def test_skip_when_author_ignored(self): + """Validation is skipped when author is in ignore list.""" + rule = ValidationRule( + check="ai_attribution", + value="forbid", + ) + validator = AiAttributionValidator(rule) + message = "feat: add feature\n\nCo-authored-by: Claude" + config = {"commit": {"ignore_authors": ["bot-user"]}} + context = ValidationContext(stdin_text=message, config=config) + + with patch("commit_check.engine.get_commit_info", return_value="bot-user"): + result = validator.validate(context) + assert result == ValidationResult.PASS # Skipped due to ignored author + + @pytest.mark.benchmark + def test_empty_message_passes(self): + """Empty message passes validation.""" + rule = ValidationRule( + check="ai_attribution", + value="forbid", + ) + validator = AiAttributionValidator(rule) + context = ValidationContext(stdin_text="") + result = validator.validate(context) + assert result == ValidationResult.PASS + + result = validator.validate(context) + assert result == ValidationResult.PASS diff --git a/tests/rule_builder_test.py b/tests/rule_builder_test.py index 1a13a069..c4f4763f 100644 --- a/tests/rule_builder_test.py +++ b/tests/rule_builder_test.py @@ -372,3 +372,54 @@ def test_push_rule_unknown_check_returns_none(self): unknown_entry = RuleCatalogEntry(check="unknown_push_check") rule = builder._build_push_rule(unknown_entry) assert rule is None + + +class TestAiAttributionRuleBuilder: + """Tests for AI attribution rule building.""" + + @pytest.mark.benchmark + def test_ai_attribution_ignore_returns_none(self): + """ai_attribution='ignore' (default) returns None.""" + config = {"commit": {"ai_attribution": "ignore"}} + builder = RuleBuilder(config) + entry = RuleCatalogEntry(check="ai_attribution") + rule = builder._build_ai_attribution_rule(entry) + assert rule is None + + @pytest.mark.benchmark + def test_ai_attribution_forbid_creates_rule(self): + """ai_attribution='forbid' creates a validation rule.""" + config = {"commit": {"ai_attribution": "forbid"}} + builder = RuleBuilder(config) + entry = RuleCatalogEntry(check="ai_attribution") + rule = builder._build_ai_attribution_rule(entry) + assert rule is not None + assert rule.check == "ai_attribution" + assert rule.value == "forbid" + + @pytest.mark.benchmark + def test_ai_attribution_require_returns_none(self): + """ai_attribution='require' returns None (only forbid supported).""" + config = {"commit": {"ai_attribution": "require"}} + builder = RuleBuilder(config) + entry = RuleCatalogEntry(check="ai_attribution") + rule = builder._build_ai_attribution_rule(entry) + assert rule is None + + @pytest.mark.benchmark + def test_build_all_rules_forbid_includes_attribution(self): + """forbid mode includes ai_attribution and no other AI rules.""" + config = {"commit": {"ai_attribution": "forbid"}} + builder = RuleBuilder(config) + rules = builder.build_all_rules() + ai_rules = [r for r in rules if r.check.startswith("ai_")] + assert len(ai_rules) == 1 + assert ai_rules[0].check == "ai_attribution" + + @pytest.mark.benchmark + def test_build_all_rules_no_ai_by_default(self): + """build_all_rules does not include AI rules by default.""" + builder = RuleBuilder({}) + rules = builder.build_all_rules() + ai_rules = [r for r in rules if r.check.startswith("ai_")] + assert len(ai_rules) == 0