Skip to content

GH-121970: Combine custom Pygments lexers into a package #121976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AA-Turner merged 2 commits into from
Jul 18, 2024
Merged

GH-121970: Combine custom Pygments lexers into a package #121976

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b380c71
GH-121970: Combine custom Pygments lexers into a package
AA-Turner Jul 18, 2024
6e2b6ae
Update Doc/tools/extensions/lexers/asdl_lexer.py
AA-Turner Jul 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
GH-121970: Combine custom Pygments lexers into a package
  • Loading branch information
@AA-Turner
AA-Turner committed Jul 18, 2024
commit b380c7143ccf357a0da581767730475d29a89479
3 changes: 1 addition & 2 deletions Doc/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@
# ---------------------

extensions = [
'asdl_highlight',
'c_annotations',
'escape4chm',
'glossary_search',
'peg_highlight',
'lexers',
'pyspecific',
'sphinx.ext.coverage',
'sphinx.ext.doctest',
Expand Down
15 changes: 15 additions & 0 deletions Doc/tools/extensions/lexers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from .asdl_lexer import ASDLLexer
from .peg_lexer import PEGLexer


def setup(app):
# Used for highlighting Parser/Python.asdl in library/ast.rst
app.add_lexer("asdl", ASDLLexer)
# Used for highlighting Grammar/python.gram in reference/grammar.rst
app.add_lexer("peg", PEGLexer)

return {
"version": "1.0",
"parallel_read_safe": True,
"parallel_write_safe": True,
}
23 changes: 6 additions & 17 deletions Doc/tools/extensions/asdl_highlight.py → Doc/tools/extensions/lexers/asdl_lexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
import sys
from pathlib import Path
from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text

CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent
sys.path.append(str(CPYTHON_ROOT / "Parser"))

from pygments.lexer import RegexLexer, bygroups, include, words
from pygments.token import (Comment, Keyword, Name, Operator,
Punctuation, Text)

from asdl import builtin_types
from sphinx.highlighting import lexers

class ASDLLexer(RegexLexer):
name = "ASDL"
Expand All @@ -34,7 +25,10 @@ class ASDLLexer(RegexLexer):
r"(\w+)(\*\s|\?\s|\s)(\w+)",
bygroups(Name.Builtin.Pseudo, Operator, Name),
),
(words(builtin_types), Name.Builtin),
# Keep in line with ``builtin_types`` from Parser/asdl.py.
# ASDL's 4 builtin types are
# identifier, int, string, constant
('constant|identifier|int|string', Name.Builtin),
Comment thread
AA-Turner marked this conversation as resolved.
(r"attributes", Name.Builtin),
(
_name + _text_ws + "(=)",
Expand All @@ -46,8 +40,3 @@ class ASDLLexer(RegexLexer):
(r".", Text),
],
}


def setup(app):
lexers["asdl"] = ASDLLexer()
return {'version': '1.0', 'parallel_read_safe': True}
7 changes: 0 additions & 7 deletions Doc/tools/extensions/peg_highlight.py → Doc/tools/extensions/lexers/peg_lexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text

from sphinx.highlighting import lexers


class PEGLexer(RegexLexer):
"""Pygments Lexer for PEG grammar (.gram) files
Expand Down Expand Up @@ -79,8 +77,3 @@ class PEGLexer(RegexLexer):
(r".", Text),
],
}


def setup(app):
lexers["peg"] = PEGLexer()
return {"version": "1.0", "parallel_read_safe": True}