-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdocs_sync_test.py
More file actions
255 lines (207 loc) · 10.7 KB
/
Copy pathdocs_sync_test.py
File metadata and controls
255 lines (207 loc) · 10.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
"""Anti-drift guard between this site and the commit-check package.
The reference pages describe behaviour that lives in another repository, so
nothing stops the two from diverging except a check that reads both. These
tests import the installed ``commit-check`` and assert that every rule and
option it defines is documented here, with the defaults it actually uses.
Run them against the version the site documents::
pip install commit-check
pytest tests/
"""
from __future__ import annotations
import re
from importlib.metadata import version
from pathlib import Path
from typing import Any
from commit_check.rules_catalog import ALL_RULES
from commit_check.config_merger import get_default_config
DOCS = Path(__file__).parent.parent / "docs"
def _read_doc(name: str) -> str:
"""Read a file from the ``docs`` directory."""
return (DOCS / name).read_text(encoding="utf-8")
#: A pasted failure line, e.g. ``CC003 subject-imperative check failed ==> ...``
_SAMPLE_FAILURE = re.compile(r"(CC\d{3}) (\S+) check failed ==>")
#: A pasted ``--compact`` line, e.g. ``[FAIL] CC003 subject_imperative: ...``.
#: Indented, because these samples sit inside content tabs.
_COMPACT_FAILURE = re.compile(r"^\s*\[FAIL\] (CC\d{3}) ([^:\s]+):", re.M)
def _rule_section(content: str, rule_id: str) -> str:
"""Return just the part of the rules page belonging to one rule."""
_, _, after = content.partition(f"{{ #{rule_id.lower()} }}")
return re.split(r"\{ #cc\d{3} \}", after)[0]
def _stale_samples(pattern: re.Pattern, attribute: str, printer: str) -> list[str]:
"""Find pasted samples that name a rule differently from the tool.
``attribute`` is the field of the catalog entry the format prints, and
``printer`` names that format in the failure message.
An unrecognised rule ID is reported rather than skipped. Skipping it would
mean a typo, or an ID retired upstream, could sit in a sample with every
test still passing — which is the exact failure these guards exist to
catch, so it must not be the one they wave through.
"""
by_id = {entry.rule_id: entry for entry in ALL_RULES}
stale = []
for page in DOCS.rglob("*.md"):
for rule_id, printed in pattern.findall(page.read_text("utf-8")):
where = page.relative_to(DOCS)
entry = by_id.get(rule_id)
if entry is None:
stale.append(f"{where}: {rule_id} is not a rule the package defines")
elif printed != getattr(entry, attribute):
stale.append(
f"{where}: {rule_id} shown as '{printed}', "
f"{printer} prints '{getattr(entry, attribute)}'"
)
return stale
class TestRulesDocumentation:
"""Every rule the package defines stays documented here."""
def test_every_rule_is_documented(self):
"""Each rule ID must have an anchor in the rules reference page."""
content = _read_doc("rules.md")
for entry in ALL_RULES:
anchor = f"{{ #{entry.rule_id.lower()} }}"
assert anchor in content, (
f"{entry.rule_id} ({entry.check}) is missing from docs/rules.md"
)
def test_every_rule_has_a_section_heading(self):
"""Each rule needs a ``name (CCxxx)`` heading, not just an anchor."""
content = _read_doc("rules.md")
for entry in ALL_RULES:
heading = (
f"### {entry.name} ({entry.rule_id}) {{ #{entry.rule_id.lower()} }}"
)
assert heading in content, (
f"docs/rules.md has no section titled '{heading}'"
)
def test_sample_output_matches_what_the_tool_prints(self):
"""Pasted terminal output has to name rules the way the tool does.
These blocks are transcripts, so nothing regenerates them and nothing
else here reads them: the heading and options-table guards both look
at reference tables. When the printed name moved from the config key
to its kebab-case form, six samples across four pages kept showing the
old one and every test still passed.
"""
stale = _stale_samples(_SAMPLE_FAILURE, "name", "the tool")
assert not stale, "sample output is out of date:\n " + "\n ".join(stale)
def test_compact_sample_output_matches_what_the_tool_prints(self):
"""Pasted ``--compact`` output names rules the way that format does.
The two text formats spell a check differently: the default output
prints the kebab-case name, and ``--compact`` prints the config key.
A sample of one therefore cannot be validated against the other, and
the guard above only matches the default format — so the compact
samples were checked by nothing at all. That is the blind spot that
let a pre-2.13 sample sit unnoticed in the troubleshooting page.
If the two formats are ever reconciled (see commit-check#528), this
is what will point at the samples that need rewriting.
"""
stale = _stale_samples(_COMPACT_FAILURE, "check", "--compact")
assert not stale, (
"compact sample output is out of date:\n " + "\n ".join(stale)
)
def test_every_rule_explains_itself(self):
"""Each rule section must answer what it does and why it matters."""
content = _read_doc("rules.md")
for entry in ALL_RULES:
section = _rule_section(content, entry.rule_id)
for required in ("**What it does**", "**Why is this bad?**", "**Options**"):
assert required in section, (
f"{entry.rule_id} ({entry.check}) section is missing {required}"
)
#: A pre-commit revision pin, e.g. ``rev: v2.13.1``.
_REV_PIN = re.compile(r"^\s*rev:\s*v(\d+\.\d+\.\d+)\s*$", re.M)
class TestDocumentedRevisions:
"""The revisions the install snippets pin are the released version.
Copy-pasteable snippets are the most-used thing on the site, and a pin is
invisible once it goes stale: the snippet keeps working, it just installs
an older release than the page around it describes. Nothing else here
reads these — the other guards compare reference tables against the
package — so a release would leave five pages pinned to the version
before it.
"""
def test_pinned_revisions_match_the_released_version(self):
"""Every ``rev:`` outside the blog names the installed version."""
installed = version("commit-check")
stale = []
for page in DOCS.rglob("*.md"):
# Blog posts are dated: they record what was current when they
# were written, and moving their pins forward would falsify them.
if "blog" in page.relative_to(DOCS).parts:
continue
for pinned in _REV_PIN.findall(page.read_text("utf-8")):
if pinned != installed:
stale.append(
f"{page.relative_to(DOCS)}: pins v{pinned}, "
f"the released version is {installed}"
)
assert not stale, "install snippets are out of date:\n " + "\n ".join(stale)
_OPTIONS_ROW = re.compile(
r"^\|\s*(commit|branch|push)\s*" # section
r"\|\s*(\w+)\s*" # option name
r"\|\s*(bool|int|str|list\[str\])\s*" # type
r"\|\s*(.+?)\s*\|", # documented default
re.M,
)
# Markdown code spans use single backticks.
_QUOTED = re.compile(r'^(?:`+(.*?)`+|"(.*?)")')
def _parse_options_table(content: str) -> dict[tuple[str, str], tuple[str, str]]:
"""Map ``(section, option) -> (type, raw default cell)``."""
return {
(section, option): (type_, cell)
for section, option, type_, cell in _OPTIONS_ROW.findall(content)
}
def _documented_default(type_: str, cell: str) -> Any:
"""Turn a documented default cell into a comparable Python value.
Cells carry a human annotation after the value itself (``"" (disabled)``),
so the value is read from the front of the cell and the rest ignored.
"""
cell = cell.strip()
if type_ == "bool":
return cell.startswith("true")
if type_ == "int":
match = re.match(r"-?\d+", cell)
return int(match.group()) if match else None
if type_ == "list[str]":
return re.findall(r'"(.*?)"', cell)
quoted = _QUOTED.match(cell)
if quoted is None:
return cell
backticked, double_quoted = quoted.groups()
return backticked if backticked is not None else double_quoted
class TestDocumentedDefaults:
"""The documented defaults match the ones the package actually uses."""
def test_every_runtime_option_is_documented(self):
"""Every option the runtime defines has a row in the options table."""
documented = _parse_options_table(_read_doc("configuration.md"))
for section, options in get_default_config().items():
for option in options:
assert (section, option) in documented, (
f"[{section}] {option} exists in get_default_config() but "
f"has no row in the options table of docs/configuration.md"
)
def test_no_invented_options_are_documented(self):
"""The options table does not document options that do not exist."""
runtime = get_default_config()
for section, option in _parse_options_table(_read_doc("configuration.md")):
assert option in runtime.get(section, {}), (
f"docs/configuration.md documents [{section}] {option}, which "
f"does not exist in get_default_config()"
)
def test_documented_defaults_match_the_runtime(self):
"""Every documented default equals the value the runtime actually uses."""
documented = _parse_options_table(_read_doc("configuration.md"))
runtime = get_default_config()
for (section, option), (type_, cell) in sorted(documented.items()):
if option not in runtime.get(section, {}):
continue # reported by test_no_invented_options_are_documented
expected = runtime[section][option]
actual = _documented_default(type_, cell)
if isinstance(expected, list):
# Allow-lists: order carries no meaning, membership does.
assert set(actual or []) == set(expected), (
f"docs/configuration.md documents [{section}] {option} "
f"with {sorted(set(actual or []) - set(expected))} that are "
f"not defaults, and is missing "
f"{sorted(set(expected) - set(actual or []))}"
)
else:
assert actual == expected, (
f"docs/configuration.md documents [{section}] {option} as "
f"{cell.strip()!r}, but the runtime default is {expected!r}"
)