Skip to content

Commit 7eba524

Browse files
sepehr-rssepehr-rs
committed
Update CI and add new scripts
1 parent c017cc8 commit 7eba524

3 files changed

Lines changed: 268 additions & 7 deletions

File tree

‎.github/workflows/lint.yml‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,30 @@ on:
1414
jobs:
1515
lint:
1616
runs-on: ubuntu-latest
17-
strategy:
18-
fail-fast: false
19-
continue-on-error: true
2017
steps:
2118
- name: Checkout code
2219
uses: actions/checkout@v4
23-
20+
2421
- name: Set up Python
2522
uses: actions/setup-python@v4
2623
with:
2724
python-version: '3.11'
28-
25+
2926
- name: Install sphinx-lint
3027
run: pip install sphinx-lint
31-
28+
29+
- name: Install gettext tools
30+
run: sudo apt-get install -y gettext
31+
3232
- name: Setup problem matcher
3333
uses: rffontenelle/sphinx-lint-problem-matcher@v1.0.0
34-
34+
3535
- name: Run sphinx-lint
3636
run: sphinx-lint
37+
continue-on-error: true
38+
39+
- name: Check PO file validity
40+
run: find . -name '*.po' -exec msgfmt --check {} \;
41+
42+
- name: Check markup preservation
43+
run: python3 scripts/check_markup.py .

‎scripts/check_markup.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env python3
2+
"""
3+
scripts/check_markup.py
4+
5+
Verify that Sphinx roles, inline literals, and format placeholders in the
6+
English msgid are preserved verbatim in the Persian msgstr. Catches the most
7+
common review slip: translating or mangling `:class:`int``-style markup,
8+
``code`` spans, %s/{0} placeholders, or |substitution| refs.
9+
10+
Usage:
11+
python3 scripts/check_markup.py library/functions.po
12+
python3 scripts/check_markup.py tutorial/*.po
13+
python3 scripts/check_markup.py . # recurse a whole directory
14+
"""
15+
import re
16+
import sys
17+
from pathlib import Path
18+
19+
PATTERNS = [
20+
("sphinx role", re.compile(r":[\w.-]+:`.*?`")),
21+
("literal/code span", re.compile(r"``.*?``")),
22+
("substitution ref", re.compile(r"\|[\w.-]+\|")),
23+
("percent placeholder", re.compile(r"%\(\w+\)[a-zA-Z]|%[a-zA-Z]")),
24+
("brace placeholder", re.compile(r"\{[^{}\s]*\}")),
25+
]
26+
27+
28+
def unescape(raw: str) -> str:
29+
"""Undo PO string escaping (raw includes the surrounding quotes)."""
30+
inner = raw[1:-1]
31+
out = []
32+
i = 0
33+
while i < len(inner):
34+
c = inner[i]
35+
if c == "\\" and i + 1 < len(inner):
36+
nxt = inner[i + 1]
37+
out.append({"n": "\n", "t": "\t", '"': '"', "\\": "\\"}.get(nxt, nxt))
38+
i += 2
39+
else:
40+
out.append(c)
41+
i += 1
42+
return "".join(out)
43+
44+
45+
def parse_po(path: Path):
46+
"""Yield (location, flags, msgid, msgstr) for each entry in a .po file."""
47+
lines = path.read_text(encoding="utf-8").splitlines()
48+
i, n = 0, len(lines)
49+
50+
def read_block(keyword_line):
51+
nonlocal i
52+
parts = [unescape(keyword_line.split(" ", 1)[1].strip())]
53+
i += 1
54+
while i < n and lines[i].strip().startswith('"'):
55+
parts.append(unescape(lines[i].strip()))
56+
i += 1
57+
return "".join(parts)
58+
59+
while i < n:
60+
location, flags = "", []
61+
while i < n and lines[i].startswith("#"):
62+
if lines[i].startswith("#:"):
63+
location = lines[i][2:].strip()
64+
elif lines[i].startswith("#,"):
65+
flags = [f.strip() for f in lines[i][2:].split(",")]
66+
i += 1
67+
if i >= n or not lines[i].startswith("msgid"):
68+
i += 1
69+
continue
70+
71+
msgid = read_block(lines[i])
72+
msgid_plural = read_block(lines[i]) if i < n and lines[i].startswith("msgid_plural") else None
73+
74+
if i < n and lines[i].startswith("msgstr["):
75+
msgstrs = {}
76+
while i < n and lines[i].startswith("msgstr["):
77+
idx = int(lines[i][7:lines[i].index("]")])
78+
msgstrs[idx] = read_block(lines[i])
79+
yield location, flags, msgid, msgstrs.get(0, "")
80+
if msgid_plural is not None and 1 in msgstrs:
81+
yield location, flags, msgid_plural, msgstrs[1]
82+
continue
83+
84+
msgstr = read_block(lines[i]) if i < n and lines[i].startswith("msgstr") else ""
85+
yield location, flags, msgid, msgstr
86+
87+
88+
def check_file(path: Path) -> int:
89+
problems = 0
90+
for location, flags, msgid, msgstr in parse_po(path):
91+
if not msgid or not msgstr:
92+
continue # header entry or still untranslated
93+
for label, pattern in PATTERNS:
94+
expected = pattern.findall(msgid)
95+
if not expected:
96+
continue
97+
missing = [tok for tok in expected if tok not in msgstr]
98+
if missing:
99+
problems += 1
100+
loc = f" ({location})" if location else ""
101+
tag = " [fuzzy]" if "fuzzy" in flags else ""
102+
print(f"{path}{loc}{tag}: missing {label}: {missing}")
103+
print(f" msgid : {msgid[:100]}")
104+
print(f" msgstr: {msgstr[:100]}")
105+
return problems
106+
107+
108+
def main():
109+
if len(sys.argv) < 2:
110+
print(__doc__)
111+
sys.exit(1)
112+
113+
files = []
114+
for arg in sys.argv[1:]:
115+
p = Path(arg)
116+
files.extend(sorted(p.rglob("*.po"))) if p.is_dir() else files.append(p)
117+
118+
total = sum(check_file(f) for f in files)
119+
120+
if total:
121+
print(f"\n{total} markup mismatch(es) found.")
122+
sys.exit(1)
123+
print("No markup mismatches found.")
124+
125+
126+
if __name__ == "__main__":
127+
main()

‎scripts/update_python_version.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env python3
2+
"""
3+
scripts/update_python_version.py
4+
5+
Sync this repo's .po files against a target CPython release tag.
6+
7+
Usage:
8+
python scripts/update_python_version.py v3.14.6
9+
python scripts/update_python_version.py v3.15.0 --keep-src
10+
11+
Run this from anywhere inside the repo (it locates the repo root from
12+
this file's location, assuming it lives in <repo>/scripts/).
13+
"""
14+
import argparse
15+
import shutil
16+
import subprocess
17+
import sys
18+
from pathlib import Path
19+
20+
REPO_ROOT = Path(__file__).resolve().parent.parent
21+
WORKDIR = REPO_ROOT / ".cpython-src" # scratch clone, deleted by default when done
22+
23+
24+
def run(cmd, cwd=None):
25+
print(f"$ {' '.join(str(c) for c in cmd)}")
26+
subprocess.run(cmd, cwd=cwd, check=True)
27+
28+
29+
def fetch_cpython(tag: str) -> None:
30+
if WORKDIR.exists():
31+
shutil.rmtree(WORKDIR)
32+
run([
33+
"git", "clone", "--depth", "1", "--branch", tag,
34+
"https://github.com/python/cpython.git", str(WORKDIR),
35+
])
36+
37+
38+
def build_gettext(tag: str) -> Path:
39+
"""Build .pot templates from the CPython docs at `tag`, return their root dir."""
40+
doc_dir = WORKDIR / "Doc"
41+
venv_dir = doc_dir / "venv"
42+
run([sys.executable, "-m", "venv", str(venv_dir)])
43+
pip = venv_dir / "bin" / "pip"
44+
sphinx_build = venv_dir / "bin" / "sphinx-build"
45+
run([str(pip), "install", "-r", "requirements.txt"], cwd=doc_dir)
46+
run([str(sphinx_build), "-b", "gettext", ".", "build/gettext"], cwd=doc_dir)
47+
return doc_dir / "build" / "gettext"
48+
49+
50+
def merge_all(pot_root: Path) -> None:
51+
updated = new_po = missing_pot = 0
52+
53+
# existing .po files -> merge against matching .pot by relative path
54+
for po_path in sorted(REPO_ROOT.rglob("*.po")):
55+
if ".cpython-src" in po_path.parts or ".git" in po_path.parts:
56+
continue
57+
rel = po_path.relative_to(REPO_ROOT)
58+
pot_path = pot_root / rel.with_suffix(".pot")
59+
if not pot_path.exists():
60+
print(f" ! no matching .pot for {rel} "
61+
f"(page may have been removed/renamed upstream — review manually)")
62+
missing_pot += 1
63+
continue
64+
run(["msgmerge", "--update", "--backup=off", str(po_path), str(pot_path)])
65+
updated += 1
66+
67+
# brand-new .pot files with no .po counterpart yet -> create empty .po via msginit
68+
for pot_path in sorted(pot_root.rglob("*.pot")):
69+
rel = pot_path.relative_to(pot_root)
70+
po_path = REPO_ROOT / rel.with_suffix(".po")
71+
if not po_path.exists():
72+
po_path.parent.mkdir(parents=True, exist_ok=True)
73+
run(["msginit", "--no-translator", "-l", "fa", "-i", str(pot_path), "-o", str(po_path)])
74+
new_po += 1
75+
76+
print(f"\nSummary: {updated} .po files merged, {new_po} new .po files created, "
77+
f"{missing_pot} .po files with no matching upstream source.")
78+
79+
80+
def check_po_files() -> None:
81+
bad = []
82+
for po_path in sorted(REPO_ROOT.rglob("*.po")):
83+
if ".cpython-src" in po_path.parts or ".git" in po_path.parts:
84+
continue
85+
result = subprocess.run(
86+
["msgfmt", "--check", "-o", "/dev/null", str(po_path)],
87+
capture_output=True, text=True,
88+
)
89+
if result.returncode != 0:
90+
bad.append((po_path, result.stderr.strip()))
91+
92+
if bad:
93+
print("\nBroken .po files (fix before committing):")
94+
for path, err in bad:
95+
print(f" {path}:\n {err}")
96+
sys.exit(1)
97+
print("\nAll .po files pass `msgfmt --check`.")
98+
99+
100+
def main() -> None:
101+
parser = argparse.ArgumentParser(description=__doc__)
102+
parser.add_argument("tag", help="CPython git tag to sync against, e.g. v3.14.6")
103+
parser.add_argument("--keep-src", action="store_true",
104+
help="keep the scratch CPython checkout instead of deleting it")
105+
args = parser.parse_args()
106+
107+
print(f"== Fetching CPython {args.tag} ==")
108+
fetch_cpython(args.tag)
109+
110+
print("\n== Building gettext templates ==")
111+
pot_root = build_gettext(args.tag)
112+
113+
print("\n== Merging into this repo's .po files ==")
114+
merge_all(pot_root)
115+
116+
print("\n== Validating .po files ==")
117+
check_po_files()
118+
119+
if not args.keep_src:
120+
shutil.rmtree(WORKDIR, ignore_errors=True)
121+
122+
print(f"\nDone. Review the diff, then commit as something like:\n"
123+
f' git commit -am "Sync translations with CPython {args.tag}"')
124+
125+
126+
if __name__ == "__main__":
127+
main()

0 commit comments

Comments
 (0)