diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 8d1927d6b36..c7fa22c7210 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ["3.x"] steps: - uses: actions/checkout@v3 @@ -29,9 +29,13 @@ jobs: pip install mypy pyflakes flake8 - name: Lint with mypy run: | + set -e mypy -p IPython.terminal mypy -p IPython.core.magics + mypy -p IPython.core.guarded_eval + mypy -p IPython.core.completer - name: Lint with pyflakes run: | + set -e flake8 IPython/core/magics/script.py flake8 IPython/core/magics/packaging.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53ccb6f78ed..73968555c4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] deps: [test_extra] # Test all on ubuntu, test ends on macos include: @@ -27,15 +27,15 @@ jobs: python-version: "3.8" deps: test_extra - os: macos-latest - python-version: "3.10" + python-version: "3.11" deps: test_extra # Tests minimal dependencies set - os: ubuntu-latest - python-version: "3.10" + python-version: "3.11" deps: test # Tests latest development Python version - os: ubuntu-latest - python-version: "3.11-dev" + python-version: "3.12-dev" deps: test # Installing optional dependencies stuff takes ages on PyPy - os: ubuntu-latest @@ -77,4 +77,7 @@ jobs: run: | pytest --color=yes -raXxs ${{ startsWith(matrix.python-version, 'pypy') && ' ' || '--cov --cov-report=xml' }} - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 + with: + name: Test + files: /home/runner/work/ipython/ipython/coverage.xml diff --git a/.gitignore b/.gitignore index f4736530e10..3b6963b6317 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ __pycache__ .cache .coverage *.swp -.vscode .pytest_cache .python-version venv*/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5826baf599c..164757fb350 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,8 +66,9 @@ Some guidelines on contributing to IPython: If you're making functional changes, you can clean up the specific pieces of code you're working on. -[Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing -IPython and Pull Requests, but it may make sense to manually perform tests, +[GitHub Actions](https://github.com/ipython/ipython/actions/workflows/test.yml) does +a pretty good job testing IPython and Pull Requests, +but it may make sense to manually perform tests, particularly for PRs that affect `IPython.parallel` or Windows. For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow). @@ -88,3 +89,30 @@ Only a single test (for example **test_alias_lifecycle**) within a single file c ```shell pytest IPython/core/tests/test_alias.py::test_alias_lifecycle ``` + +## Code style + +* Before committing, run `darker -r 60625f241f298b5039cb2debc365db38aa7bb522 ` to apply selective `black` formatting on modified regions using [darker](https://github.com/akaihola/darker). +* For newly added modules or refactors, please enable static typing analysis with `mypy` for the modified module by adding the file path in [`mypy.yml`](https://github.com/ipython/ipython/blob/main/.github/workflows/mypy.yml) workflow. +* As described in the pull requests section, please avoid excessive formatting changes; if a formatting-only commit is necessary, consider adding its hash to [`.git-blame-ignore-revs`](https://github.com/ipython/ipython/blob/main/.git-blame-ignore-revs) file. + +## Documentation + +Sphinx documentation can be built locally using standard sphinx `make` commands. To build HTML documentation from the root of the project, execute: + +```shell +pip install -r docs/requirements.txt # only needed once +make -C docs/ html SPHINXOPTS="-W" +``` + +To force update of the API documentation, precede the `make` command with: + +```shell +python3 docs/autogen_api.py +``` + +Similarly, to force-update the configuration, run: + +```shell +python3 docs/autogen_config.py +``` diff --git a/IPython/__init__.py b/IPython/__init__.py index 03b3116a98a..7d3799ab363 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK """ IPython: tools for interactive and parallel computing in Python. @@ -62,7 +63,7 @@ version_info = release.version_info # list of CVEs that should have been patched in this release. # this is informational and should not be relied upon. -__patched_cves__ = {"CVE-2022-21699"} +__patched_cves__ = {"CVE-2022-21699", "CVE-2023-24816"} def embed_kernel(module=None, local_ns=None, **kwargs): diff --git a/IPython/__main__.py b/IPython/__main__.py index d5123f33a20..8e9f989a82c 100644 --- a/IPython/__main__.py +++ b/IPython/__main__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK # encoding: utf-8 """Terminal-based IPython entry point. """ diff --git a/IPython/core/application.py b/IPython/core/application.py index 26c061661ac..e0a8174f153 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -123,9 +123,8 @@ def load_subconfig(self, fname, path=None, profile=None): return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path) class BaseIPythonApplication(Application): - - name = u'ipython' - description = Unicode(u'IPython: an enhanced interactive Python shell.') + name = "ipython" + description = "IPython: an enhanced interactive Python shell." version = Unicode(release.version) aliases = base_aliases @@ -311,7 +310,7 @@ def _ipython_dir_changed(self, change): except OSError as e: # this will not be EEXIST self.log.error("couldn't create path %s: %s", path, e) - self.log.debug("IPYTHONDIR set to: %s" % new) + self.log.debug("IPYTHONDIR set to: %s", new) def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): """Load the config file. @@ -401,7 +400,7 @@ def init_profile_dir(self): self.log.fatal("Profile %r not found."%self.profile) self.exit(1) else: - self.log.debug(f"Using existing profile dir: {p.location!r}") + self.log.debug("Using existing profile dir: %r", p.location) else: location = self.config.ProfileDir.location # location is fully specified @@ -421,7 +420,7 @@ def init_profile_dir(self): self.log.fatal("Profile directory %r not found."%location) self.exit(1) else: - self.log.debug(f"Using existing profile dir: {p.location!r}") + self.log.debug("Using existing profile dir: %r", p.location) # if profile_dir is specified explicitly, set profile name dir_name = os.path.basename(p.location) if dir_name.startswith('profile_'): @@ -468,7 +467,7 @@ def stage_default_config_file(self): s = self.generate_config_file() config_file = Path(self.profile_dir.location) / self.config_file_name if self.overwrite or not config_file.exists(): - self.log.warning("Generating default config file: %r" % (config_file)) + self.log.warning("Generating default config file: %r", (config_file)) config_file.write_text(s, encoding="utf-8") @catch_config_error diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index 228f705666f..7799a4fc99e 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -116,6 +116,21 @@ def get_code_name(self, raw_code, transformed_code, number): """ return code_name(transformed_code, number) + def format_code_name(self, name): + """Return a user-friendly label and name for a code block. + + Parameters + ---------- + name : str + The name for the code block returned from get_code_name + + Returns + ------- + A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used. + """ + if name in self._filename_map: + return "Cell", "In[%s]" % self._filename_map[name] + def cache(self, transformed_code, number=0, raw_code=None): """Make a name for a block of code, and cache the code. diff --git a/IPython/core/completer.py b/IPython/core/completer.py index fc3aea7b611..f0bbb4e5619 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -50,7 +50,7 @@ It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press ```` to expand it to its latex form. +and press :kbd:`Tab` to expand it to its latex form. .. code:: @@ -59,7 +59,8 @@ Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. +:std:configtrait:`Completer.backslash_combining_completions` option to +``False``. Experimental @@ -95,7 +96,7 @@ ... myvar[1].bi Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` +executing almost any code unlike the deprecated :any:`IPCompleter.greedy` option. Be sure to update :any:`jedi` to the latest stable version or to try the @@ -166,7 +167,7 @@ should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key. The suppression behaviour can is user-configurable via -:any:`IPCompleter.suppress_competing_matchers`. +:std:configtrait:`IPCompleter.suppress_competing_matchers`. """ @@ -178,6 +179,7 @@ from __future__ import annotations import builtins as builtin_mod +import enum import glob import inspect import itertools @@ -186,14 +188,16 @@ import re import string import sys +import tokenize import time import unicodedata import uuid import warnings +from ast import literal_eval +from collections import defaultdict from contextlib import contextmanager from dataclasses import dataclass from functools import cached_property, partial -from importlib import import_module from types import SimpleNamespace from typing import ( Iterable, @@ -204,14 +208,15 @@ Any, Sequence, Dict, - NamedTuple, - Pattern, Optional, TYPE_CHECKING, Set, + Sized, + TypeVar, Literal, ) +from IPython.core.guarded_eval import guarded_eval, EvaluationContext from IPython.core.error import TryNext from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol @@ -231,7 +236,6 @@ Unicode, Dict as DictTrait, Union as UnionTrait, - default, observe, ) from traitlets.config.configurable import Configurable @@ -252,12 +256,13 @@ JEDI_INSTALLED = False -if TYPE_CHECKING or GENERATING_DOCUMENTATION: +if TYPE_CHECKING or GENERATING_DOCUMENTATION and sys.version_info >= (3, 11): from typing import cast - from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias + from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard else: + from typing import Generic - def cast(obj, type_): + def cast(type_, obj): """Workaround for `TypeError: MatcherAPIv2() takes no arguments`""" return obj @@ -266,6 +271,7 @@ def cast(obj, type_): TypedDict = Dict # by extension of `NotRequired` requires 3.11 too Protocol = object # requires Python >=3.8 TypeAlias = Any # requires Python >=3.10 + TypeGuard = Generic # requires Python >=3.10 if GENERATING_DOCUMENTATION: from typing import TypedDict @@ -279,7 +285,7 @@ def cast(obj, type_): # write this). With below range we cover them all, with a density of ~67% # biggest next gap we consider only adds up about 1% density and there are 600 # gaps that would need hard coding. -_UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)] +_UNICODE_RANGES = [(32, 0x323B0), (0xE0001, 0xE01F0)] # Public API __all__ = ["Completer", "IPCompleter"] @@ -296,6 +302,9 @@ def cast(obj, type_): # Completion type reported when no type can be inferred. _UNKNOWN_TYPE = "" +# sentinel value to signal lack of a match +not_found = object() + class ProvisionalCompleterWarning(FutureWarning): """ Exception raise by an experimental feature in this module. @@ -466,8 +475,9 @@ def __init__(self, name): self.complete = name self.type = 'crashed' self.name_with_symbols = name - self.signature = '' - self._origin = 'fake' + self.signature = "" + self._origin = "fake" + self.text = "crashed" def __repr__(self): return '' @@ -503,11 +513,23 @@ class Completion: __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] - def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: - warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) + def __init__( + self, + start: int, + end: int, + text: str, + *, + type: Optional[str] = None, + _origin="", + signature="", + ) -> None: + warnings.warn( + "``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, + stacklevel=2, + ) self.start = start self.end = end @@ -520,7 +542,7 @@ def __repr__(self): return '' % \ (self.start, self.end, self.text, self.type or '?', self.signature or '?') - def __eq__(self, other)->Bool: + def __eq__(self, other) -> bool: """ Equality and hash do not hash the type (as some completer may not be able to infer the type), but are use to (partially) de-duplicate @@ -554,7 +576,7 @@ class SimpleCompletion: __slots__ = ["text", "type"] - def __init__(self, text: str, *, type: str = None): + def __init__(self, text: str, *, type: Optional[str] = None): self.text = text self.type = type @@ -588,14 +610,18 @@ class SimpleMatcherResult(_MatcherResultBase, TypedDict): # in order to get __orig_bases__ for documentation #: List of candidate completions - completions: Sequence[SimpleCompletion] + completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion] class _JediMatcherResult(_MatcherResultBase): """Matching result returned by Jedi (will be processed differently)""" #: list of candidate completions - completions: Iterable[_JediCompletionLike] + completions: Iterator[_JediCompletionLike] + + +AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion] +AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion) @dataclass @@ -642,16 +668,21 @@ def line_with_cursor(self) -> str: class _MatcherAPIv1Base(Protocol): - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol): #: API version matcher_api_version: Optional[Literal[1]] - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... #: Protocol describing Matcher API v1. @@ -666,13 +697,61 @@ class MatcherAPIv2(Protocol): def __call__(self, context: CompletionContext) -> MatcherResult: """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] +def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]: + api_version = _get_matcher_api_version(matcher) + return api_version == 1 + + +def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]: + api_version = _get_matcher_api_version(matcher) + return api_version == 2 + + +def _is_sizable(value: Any) -> TypeGuard[Sized]: + """Determines whether objects is sizable""" + return hasattr(value, "__len__") + + +def _is_iterator(value: Any) -> TypeGuard[Iterator]: + """Determines whether objects is sizable""" + return hasattr(value, "__next__") + + +def has_any_completions(result: MatcherResult) -> bool: + """Check if any result includes any completions.""" + completions = result["completions"] + if _is_sizable(completions): + return len(completions) != 0 + if _is_iterator(completions): + try: + old_iterator = completions + first = next(old_iterator) + result["completions"] = cast( + Iterator[SimpleCompletion], + itertools.chain([first], old_iterator), + ) + return True + except StopIteration: + return False + raise ValueError( + "Completions returned by matcher need to be an Iterator or a Sizable" + ) + + def completion_matcher( - *, priority: float = None, identifier: str = None, api_version: int = 1 + *, + priority: Optional[float] = None, + identifier: Optional[str] = None, + api_version: int = 1, ): """Adds attributes describing the matcher. @@ -684,7 +763,10 @@ def completion_matcher( identifier : Optional[str] identifier of the matcher allowing users to modify the behaviour via traitlets, and also used to for debugging (will be passed as ``origin`` with the completions). - Defaults to matcher function ``__qualname__``. + + Defaults to matcher function's ``__qualname__`` (for example, + ``IPCompleter.file_matcher`` for the built-in matched defined + as a ``file_matcher`` method of the ``IPCompleter`` class). api_version: Optional[int] version of the Matcher API used by this matcher. Currently supported values are 1 and 2. @@ -692,14 +774,14 @@ def completion_matcher( """ def wrapper(func: Matcher): - func.matcher_priority = priority or 0 - func.matcher_identifier = identifier or func.__qualname__ - func.matcher_api_version = api_version + func.matcher_priority = priority or 0 # type: ignore + func.matcher_identifier = identifier or func.__qualname__ # type: ignore + func.matcher_api_version = api_version # type: ignore if TYPE_CHECKING: if api_version == 1: - func = cast(func, MatcherAPIv1) + func = cast(MatcherAPIv1, func) elif api_version == 2: - func = cast(func, MatcherAPIv2) + func = cast(MatcherAPIv2, func) return func return wrapper @@ -886,12 +968,44 @@ def split_line(self, line, cursor_pos=None): class Completer(Configurable): - greedy = Bool(False, - help="""Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. + greedy = Bool( + False, + help="""Activate greedy completion. + + .. deprecated:: 8.8 + Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. + When enabled in IPython 8.8 or newer, changes configuration as follows: + + - ``Completer.evaluation = 'unsafe'`` + - ``Completer.auto_close_dict_keys = True`` + """, + ).tag(config=True) + + evaluation = Enum( + ("forbidden", "minimal", "limited", "unsafe", "dangerous"), + default_value="limited", + help="""Policy for code evaluation under completion. + + Successive options allow to enable more eager evaluation for better + completion suggestions, including for nested dictionaries, nested lists, + or even results of function calls. + Setting ``unsafe`` or higher can lead to evaluation of arbitrary user + code on :kbd:`Tab` with potentially unwanted or dangerous side effects. + + Allowed values are: + + - ``forbidden``: no evaluation of code is permitted, + - ``minimal``: evaluation of literals and access to built-in namespace; + no item/attribute evaluationm no access to locals/globals, + no evaluation of any operations or comparisons. + - ``limited``: access to all namespaces, evaluation of hard-coded methods + (for example: :any:`dict.keys`, :any:`object.__getattr__`, + :any:`object.__getitem__`) on allow-listed objects (for example: + :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``), + - ``unsafe``: evaluation of all methods and function calls but not of + syntax with side-effects like `del x`, + - ``dangerous``: completely arbitrary evaluation. """, ).tag(config=True) @@ -915,6 +1029,18 @@ class Completer(Configurable): "Includes completion of latex commands, unicode names, and expanding " "unicode characters back to latex commands.").tag(config=True) + auto_close_dict_keys = Bool( + False, + help=""" + Enable auto-closing dictionary keys. + + When enabled string keys will be suffixed with a final quote + (matching the opening quote), tuple keys will also receive a + separating comma if needed, and keys which are final will + receive a closing bracket (``]``). + """, + ).tag(config=True) + def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -1013,28 +1139,16 @@ def attr_matches(self, text): with a __getattr__ hook is evaluated. """ + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return [] + expr, attr = m2.group(1, 2) - # Another option, seems to work great. Catches things like ''. - m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) + obj = self._evaluate_expr(expr) - if m: - expr, attr = m.group(1, 3) - elif self.greedy: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return [] - expr, attr = m2.group(1,2) - else: + if obj is not_found: return [] - try: - obj = eval(expr, self.namespace) - except: - try: - obj = eval(expr, self.global_namespace) - except: - return [] - if self.limit_to__all__ and hasattr(obj, '__all__'): words = get__all__entries(obj) else: @@ -1052,8 +1166,31 @@ def attr_matches(self, text): pass # Build match list to return n = len(attr) - return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] + return ["%s.%s" % (expr, w) for w in words if w[:n] == attr] + def _evaluate_expr(self, expr): + obj = not_found + done = False + while not done and expr: + try: + obj = guarded_eval( + expr, + EvaluationContext( + globals=self.global_namespace, + locals=self.namespace, + evaluation=self.evaluation, + ), + ) + done = True + except Exception as e: + if self.debug: + print("Evaluation exception", e) + # trim the expression to remove any invalid prefix + # e.g. user starts `(d[`, so we get `expr = '(d'`, + # where parenthesis is not closed. + # TODO: make this faster by reusing parts of the computation? + expr = expr[1:] + return obj def get__all__entries(obj): """returns the strings in the __all__ attribute""" @@ -1065,8 +1202,82 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str, - extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]: +class _DictKeyState(enum.Flag): + """Represent state of the key match in context of other possible matches. + + - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. + - given `d2 = {('a', 'b'): 1}`: `d2['a', '` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`. + - given `d3 = {('a', 'b'): 1}`: `d3['` will yield `{'a': IN_TUPLE}` as `'a'` can be added. + - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}` + """ + + BASELINE = 0 + END_OF_ITEM = enum.auto() + END_OF_TUPLE = enum.auto() + IN_TUPLE = enum.auto() + + +def _parse_tokens(c): + """Parse tokens even if there is an error.""" + tokens = [] + token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) + while True: + try: + tokens.append(next(token_generator)) + except tokenize.TokenError: + return tokens + except StopIteration: + return tokens + + +def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: + """Match any valid Python numeric literal in a prefix of dictionary keys. + + References: + - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals + - https://docs.python.org/3/library/tokenize.html + """ + if prefix[-1].isspace(): + # if user typed a space we do not have anything to complete + # even if there was a valid number token before + return None + tokens = _parse_tokens(prefix) + rev_tokens = reversed(tokens) + skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE} + number = None + for token in rev_tokens: + if token.type in skip_over: + continue + if number is None: + if token.type == tokenize.NUMBER: + number = token.string + continue + else: + # we did not match a number + return None + if token.type == tokenize.OP: + if token.string == ",": + break + if token.string in {"+", "-"}: + number = token.string + number + else: + return None + return number + + +_INT_FORMATS = { + "0b": bin, + "0o": oct, + "0x": hex, +} + + +def match_dict_keys( + keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], + prefix: str, + delims: str, + extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, +) -> Tuple[str, int, Dict[str, _DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1086,47 +1297,89 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre A tuple of three elements: ``quote``, ``token_start``, ``matched``, with ``quote`` being the quote that need to be used to close current string. ``token_start`` the position where the replacement should start occurring, - ``matches`` a list of replacement/completion - + ``matches`` a dictionary of replacement/completion keys on keys and values + indicating whether the state. """ prefix_tuple = extra_prefix if extra_prefix else () - Nprefix = len(prefix_tuple) + + prefix_tuple_size = sum( + [ + # for pandas, do not count slices as taking space + not isinstance(k, slice) + for k in prefix_tuple + ] + ) + text_serializable_types = (str, bytes, int, float, slice) + def filter_prefix_tuple(key): # Reject too short keys - if len(key) <= Nprefix: + if len(key) <= prefix_tuple_size: return False - # Reject keys with non str/bytes in it + # Reject keys which cannot be serialised to text for k in key: - if not isinstance(k, (str, bytes)): + if not isinstance(k, text_serializable_types): return False # Reject keys that do not match the prefix for k, pt in zip(key, prefix_tuple): - if k != pt: + if k != pt and not isinstance(pt, slice): return False # All checks passed! return True - filtered_keys:List[Union[str,bytes]] = [] - def _add_to_filtered_keys(key): - if isinstance(key, (str, bytes)): - filtered_keys.append(key) + filtered_key_is_final: Dict[ + Union[str, bytes, int, float], _DictKeyState + ] = defaultdict(lambda: _DictKeyState.BASELINE) for k in keys: + # If at least one of the matches is not final, mark as undetermined. + # This can happen with `d = {111: 'b', (111, 222): 'a'}` where + # `111` appears final on first match but is not final on the second. + if isinstance(k, tuple): if filter_prefix_tuple(k): - _add_to_filtered_keys(k[Nprefix]) + key_fragment = k[prefix_tuple_size] + filtered_key_is_final[key_fragment] |= ( + _DictKeyState.END_OF_TUPLE + if len(k) == prefix_tuple_size + 1 + else _DictKeyState.IN_TUPLE + ) + elif prefix_tuple_size > 0: + # we are completing a tuple but this key is not a tuple, + # so we should ignore it + pass else: - _add_to_filtered_keys(k) + if isinstance(k, text_serializable_types): + filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM + + filtered_keys = filtered_key_is_final.keys() if not prefix: - return '', 0, [repr(k) for k in filtered_keys] - quote_match = re.search('["\']', prefix) - assert quote_match is not None # silence mypy - quote = quote_match.group() - try: - prefix_str = eval(prefix + quote, {}) - except Exception: - return '', 0, [] + return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()} + + quote_match = re.search("(?:\"|')", prefix) + is_user_prefix_numeric = False + + if quote_match: + quote = quote_match.group() + valid_prefix = prefix + quote + try: + prefix_str = literal_eval(valid_prefix) + except Exception: + return "", 0, {} + else: + # If it does not look like a string, let's assume + # we are dealing with a number or variable. + number_match = _match_number_in_dict_key_prefix(prefix) + + # We do not want the key matcher to suggest variable names so we yield: + if number_match is None: + # The alternative would be to assume that user forgort the quote + # and if the substring matches, suggest adding it at the start. + return "", 0, {} + + prefix_str = number_match + is_user_prefix_numeric = True + quote = "" pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) @@ -1134,17 +1387,36 @@ def _add_to_filtered_keys(key): token_start = token_match.start() token_prefix = token_match.group() - matched:List[str] = [] + matched: Dict[str, _DictKeyState] = {} + + str_key: Union[str, bytes] + for key in filtered_keys: + if isinstance(key, (int, float)): + # User typed a number but this key is not a number. + if not is_user_prefix_numeric: + continue + str_key = str(key) + if isinstance(key, int): + int_base = prefix_str[:2].lower() + # if user typed integer using binary/oct/hex notation: + if int_base in _INT_FORMATS: + int_format = _INT_FORMATS[int_base] + str_key = int_format(key) + else: + # User typed a string but this key is a number. + if is_user_prefix_numeric: + continue + str_key = key try: - if not key.startswith(prefix_str): + if not str_key.startswith(prefix_str): continue - except (AttributeError, TypeError, UnicodeError): + except (AttributeError, TypeError, UnicodeError) as e: # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue # reformat remainder of key to begin with prefix - rem = key[len(prefix_str):] + rem = str_key[len(prefix_str) :] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') rem_repr = rem_repr[1 + rem_repr.index("'"):-2] @@ -1155,7 +1427,9 @@ def _add_to_filtered_keys(key): rem_repr = rem_repr.replace('"', '\\"') # then reinsert prefix from start of token - matched.append('%s%s' % (token_prefix, rem_repr)) + match = "%s%s" % (token_prefix, rem_repr) + + matched[match] = filtered_key_is_final[key] return quote, token_start, matched @@ -1221,11 +1495,14 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: return line, col -def _safe_isinstance(obj, module, class_name): +def _safe_isinstance(obj, module, class_name, *attrs): """Checks if obj is an instance of module.class_name if loaded """ - return (module in sys.modules and - isinstance(obj, getattr(import_module(module), class_name))) + if module in sys.modules: + m = sys.modules[module] + for attr in [class_name, *attrs]: + m = getattr(m, attr) + return isinstance(obj, m) @context_matcher() @@ -1378,10 +1655,59 @@ def _make_signature(completion)-> str: _CompleteResult = Dict[str, MatcherResult] +DICT_MATCHER_REGEX = re.compile( + r"""(?x) +( # match dict-referring - or any get item object - expression + .+ +) +\[ # open bracket +\s* # and optional whitespace +# Capture any number of serializable objects (e.g. "a", "b", 'c') +# and slices +((?:(?: + (?: # closed string + [uUbB]? # string prefix (r not handled) + (?: + '(?:[^']|(? SimpleMatcherResult: """Utility to help with transition""" @@ -1391,27 +1717,29 @@ def _convert_matcher_v1_result_to_v2( } if fragment is not None: result["matched_fragment"] = fragment - return result + return cast(SimpleMatcherResult, result) class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - __dict_key_regexps: Optional[Dict[bool,Pattern]] = None - @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" - if change['new']: + if change["new"]: + self.evaluation = "unsafe" + self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: + self.evaluation = "limited" + self.auto_close_dict_keys = False self.splitter.delims = DELIMS dict_keys_only = Bool( False, help=""" Whether to show dict key matches only. - + (disables all matchers except for `IPCompleter.dict_key_matcher`). """, ) @@ -1447,14 +1775,18 @@ def _greedy_changed(self, change): If False, only the completion results from the first non-empty completer will be returned. - + As of version 8.6.0, setting the value to ``False`` is an alias for: ``IPCompleter.suppress_competing_matchers = True.``. """, ).tag(config=True) disable_matchers = ListTrait( - Unicode(), help="""List of matchers to disable.""" + Unicode(), + help="""List of matchers to disable. + + The list should contain matcher identifiers (see :any:`completion_matcher`). + """, ).tag(config=True) omit__names = Enum( @@ -1587,7 +1919,7 @@ def __init__( if not self.backslash_combining_completions: for matcher in self._backslash_combining_matchers: - self.disable_matchers.append(matcher.matcher_identifier) + self.disable_matchers.append(_get_matcher_id(matcher)) if not self.merge_completions: self.suppress_competing_matchers = True @@ -1877,7 +2209,7 @@ def _jedi_matcher(self, context: CompletionContext) -> _JediMatcherResult: def _jedi_matches( self, cursor_column: int, cursor_line: int, text: str - ) -> Iterable[_JediCompletionLike]: + ) -> Iterator[_JediCompletionLike]: """ Return a list of :any:`jedi.api.Completion`s object from a ``text`` and cursor position. @@ -1943,16 +2275,24 @@ def _jedi_matches( print("Error detecting if completing a non-finished string :", e, '|') if not try_jedi: - return [] + return iter([]) try: return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1)) except Exception as e: if self.debug: - return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + return iter( + [ + _FakeJediCompletion( + 'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' + % (e) + ) + ] + ) else: - return [] + return iter([]) - def python_matches(self, text:str)->List[str]: + @completion_matcher(api_version=1) + def python_matches(self, text: str) -> Iterable[str]: """Match attributes or global python names""" if "." in text: try: @@ -2129,12 +2469,16 @@ def _get_keys(obj: Any) -> List[Any]: return method() # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): + if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"): try: return list(obj.keys()) except Exception: return [] + elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"): + try: + return list(obj.obj.keys()) + except Exception: + return [] elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ _safe_isinstance(obj, 'numpy', 'void'): return obj.dtype.names or [] @@ -2155,74 +2499,49 @@ def dict_key_matches(self, text: str) -> List[str]: You can use :meth:`dict_key_matcher` instead. """ - if self.__dict_key_regexps is not None: - regexps = self.__dict_key_regexps - else: - dict_key_re_fmt = r'''(?x) - ( # match dict-referring expression wrt greedy setting - %s - ) - \[ # open bracket - \s* # and optional whitespace - # Capture any number of str-like objects (e.g. "a", "b", 'c') - ((?:[uUbB]? # string prefix (r not handled) - (?: - '(?:[^']|(? List[str]: else: leading = text[text_start:completion_start] - # the index of the `[` character - bracket_idx = match.end(1) - # append closing quote and bracket as appropriate # this is *not* appropriate if the opening quote or bracket is outside - # the text given to this method - suf = '' - continuation = self.line_buffer[len(self.text_until_cursor):] - if key_start > text_start and closing_quote: - # quotes were opened inside text, maybe close them - if continuation.startswith(closing_quote): - continuation = continuation[len(closing_quote):] - else: - suf += closing_quote - if bracket_idx > text_start: - # brackets were opened inside text, maybe close them - if not continuation.startswith(']'): - suf += ']' + # the text given to this method, e.g. `d["""a\nt + can_close_quote = False + can_close_bracket = False + + continuation = self.line_buffer[len(self.text_until_cursor) :].strip() - return [leading + k + suf for k in matches] + if continuation.startswith(closing_quote): + # do not close if already closed, e.g. `d['a'` + continuation = continuation[len(closing_quote) :] + else: + can_close_quote = True + + continuation = continuation.strip() + + # e.g. `pandas.DataFrame` has different tuple indexer behaviour, + # handling it is out of scope, so let's avoid appending suffixes. + has_known_tuple_handling = isinstance(obj, dict) + + can_close_bracket = ( + not continuation.startswith("]") and self.auto_close_dict_keys + ) + can_close_tuple_item = ( + not continuation.startswith(",") + and has_known_tuple_handling + and self.auto_close_dict_keys + ) + can_close_quote = can_close_quote and self.auto_close_dict_keys + + # fast path if closing qoute should be appended but not suffix is allowed + if not can_close_quote and not can_close_bracket and closing_quote: + return [leading + k for k in matches] + + results = [] + + end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM + + for k, state_flag in matches.items(): + result = leading + k + if can_close_quote and closing_quote: + result += closing_quote + + if state_flag == end_of_tuple_or_item: + # We do not know which suffix to add, + # e.g. both tuple item and string + # match this item. + pass + + if state_flag in end_of_tuple_or_item and can_close_bracket: + result += "]" + if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item: + result += ", " + results.append(result) + return results @context_matcher() def unicode_name_matcher(self, context: CompletionContext): @@ -2496,17 +2850,23 @@ def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Com jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + def is_non_jedi_result( + result: MatcherResult, identifier: str + ) -> TypeGuard[SimpleMatcherResult]: + return identifier != jedi_matcher_id + results = self._complete( full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column ) + non_jedi_results: Dict[str, SimpleMatcherResult] = { identifier: result for identifier, result in results.items() - if identifier != jedi_matcher_id + if is_non_jedi_result(result, identifier) } jedi_matches = ( - cast(results[jedi_matcher_id], _JediMatcherResult)["completions"] + cast(_JediMatcherResult, results[jedi_matcher_id])["completions"] if jedi_matcher_id in results else () ) @@ -2561,8 +2921,8 @@ def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Com signature="", ) - ordered = [] - sortable = [] + ordered: List[Completion] = [] + sortable: List[Completion] = [] for origin, result in non_jedi_results.items(): matched_text = result["matched_fragment"] @@ -2652,8 +3012,8 @@ def _arrange_and_extract( abort_if_offset_changes: bool, ): - sortable = [] - ordered = [] + sortable: List[AnyMatcherCompletion] = [] + ordered: List[AnyMatcherCompletion] = [] most_recent_fragment = None for identifier, result in results.items(): if identifier in skip_matchers: @@ -2752,11 +3112,11 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, ) # Start with a clean slate of completions - results = {} + results: Dict[str, MatcherResult] = {} jedi_matcher_id = _get_matcher_id(self._jedi_matcher) - suppressed_matchers = set() + suppressed_matchers: Set[str] = set() matchers = { _get_matcher_id(matcher): matcher @@ -2766,7 +3126,6 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, } for matcher_id, matcher in matchers.items(): - api_version = _get_matcher_api_version(matcher) matcher_id = _get_matcher_id(matcher) if matcher_id in self.disable_matchers: @@ -2778,14 +3137,16 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, if matcher_id in suppressed_matchers: continue + result: MatcherResult try: - if api_version == 1: + if _is_matcher_v1(matcher): result = _convert_matcher_v1_result_to_v2( matcher(text), type=_UNKNOWN_TYPE ) - elif api_version == 2: - result = cast(matcher, MatcherAPIv2)(context) + elif _is_matcher_v2(matcher): + result = matcher(context) else: + api_version = _get_matcher_api_version(matcher) raise ValueError(f"Unsupported API version {api_version}") except: # Show the ugly traceback if the matcher causes an @@ -2797,7 +3158,9 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, result["matched_fragment"] = result.get("matched_fragment", context.token) if not suppressed_matchers: - suppression_recommended = result.get("suppress", False) + suppression_recommended: Union[bool, Set[str]] = result.get( + "suppress", False + ) suppression_config = ( self.suppress_competing_matchers.get(matcher_id, None) @@ -2807,13 +3170,15 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, should_suppress = ( (suppression_config is True) or (suppression_recommended and (suppression_config is not False)) - ) and len(result["completions"]) + ) and has_any_completions(result) if should_suppress: - suppression_exceptions = result.get("do_not_suppress", set()) - try: + suppression_exceptions: Set[str] = result.get( + "do_not_suppress", set() + ) + if isinstance(suppression_recommended, Iterable): to_suppress = set(suppression_recommended) - except TypeError: + else: to_suppress = set(matchers) suppressed_matchers = to_suppress - suppression_exceptions @@ -2840,9 +3205,9 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, @staticmethod def _deduplicate( - matches: Sequence[SimpleCompletion], - ) -> Iterable[SimpleCompletion]: - filtered_matches = {} + matches: Sequence[AnyCompletion], + ) -> Iterable[AnyCompletion]: + filtered_matches: Dict[str, AnyCompletion] = {} for match in matches: text = match.text if ( @@ -2854,7 +3219,7 @@ def _deduplicate( return filtered_matches.values() @staticmethod - def _sort(matches: Sequence[SimpleCompletion]): + def _sort(matches: Sequence[AnyCompletion]): return sorted(matches, key=lambda x: completions_sorting_key(x.text)) @context_matcher() diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 578e783ab8e..aba4f904d8d 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -91,7 +91,13 @@ def quiet(self): # some uses of ipshellembed may fail here return False - sio = _io.StringIO(cell) + return self.semicolon_at_end_of_expression(cell) + + @staticmethod + def semicolon_at_end_of_expression(expression): + """Parse Python expression and detects whether last token is ';'""" + + sio = _io.StringIO(expression) tokens = list(tokenize.generate_tokens(sio.readline)) for token in reversed(tokens): diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py new file mode 100644 index 00000000000..3c95213734a --- /dev/null +++ b/IPython/core/guarded_eval.py @@ -0,0 +1,738 @@ +from typing import ( + Any, + Callable, + Dict, + Set, + Sequence, + Tuple, + NamedTuple, + Type, + Literal, + Union, + TYPE_CHECKING, +) +import ast +import builtins +import collections +import operator +import sys +from functools import cached_property +from dataclasses import dataclass, field + +from IPython.utils.docs import GENERATING_DOCUMENTATION +from IPython.utils.decorators import undoc + + +if TYPE_CHECKING or GENERATING_DOCUMENTATION: + from typing_extensions import Protocol +else: + # do not require on runtime + Protocol = object # requires Python >=3.8 + + +@undoc +class HasGetItem(Protocol): + def __getitem__(self, key) -> None: + ... + + +@undoc +class InstancesHaveGetItem(Protocol): + def __call__(self, *args, **kwargs) -> HasGetItem: + ... + + +@undoc +class HasGetAttr(Protocol): + def __getattr__(self, key) -> None: + ... + + +@undoc +class DoesNotHaveGetAttr(Protocol): + pass + + +# By default `__getattr__` is not explicitly implemented on most objects +MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] + + +def _unbind_method(func: Callable) -> Union[Callable, None]: + """Get unbound method for given bound method. + + Returns None if cannot get unbound method, or method is already unbound. + """ + owner = getattr(func, "__self__", None) + owner_class = type(owner) + name = getattr(func, "__name__", None) + instance_dict_overrides = getattr(owner, "__dict__", None) + if ( + owner is not None + and name + and ( + not instance_dict_overrides + or (instance_dict_overrides and name not in instance_dict_overrides) + ) + ): + return getattr(owner_class, name) + return None + + +@undoc +@dataclass +class EvaluationPolicy: + """Definition of evaluation policy.""" + + allow_locals_access: bool = False + allow_globals_access: bool = False + allow_item_access: bool = False + allow_attr_access: bool = False + allow_builtins_access: bool = False + allow_all_operations: bool = False + allow_any_calls: bool = False + allowed_calls: Set[Callable] = field(default_factory=set) + + def can_get_item(self, value, item): + return self.allow_item_access + + def can_get_attr(self, value, attr): + return self.allow_attr_access + + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + if self.allow_all_operations: + return True + + def can_call(self, func): + if self.allow_any_calls: + return True + + if func in self.allowed_calls: + return True + + owner_method = _unbind_method(func) + + if owner_method and owner_method in self.allowed_calls: + return True + + +def _get_external(module_name: str, access_path: Sequence[str]): + """Get value from external module given a dotted access path. + + Raises: + * `KeyError` if module is removed not found, and + * `AttributeError` if acess path does not match an exported object + """ + member_type = sys.modules[module_name] + for attr in access_path: + member_type = getattr(member_type, attr) + return member_type + + +def _has_original_dunder_external( + value, + module_name: str, + access_path: Sequence[str], + method_name: str, +): + if module_name not in sys.modules: + # LBYLB as it is faster + return False + try: + member_type = _get_external(module_name, access_path) + value_type = type(value) + if type(value) == member_type: + return True + if method_name == "__getattribute__": + # we have to short-circuit here due to an unresolved issue in + # `isinstance` implementation: https://bugs.python.org/issue32683 + return False + if isinstance(value, member_type): + method = getattr(value_type, method_name, None) + member_method = getattr(member_type, method_name, None) + if member_method == method: + return True + except (AttributeError, KeyError): + return False + + +def _has_original_dunder( + value, allowed_types, allowed_methods, allowed_external, method_name +): + # note: Python ignores `__getattr__`/`__getitem__` on instances, + # we only need to check at class level + value_type = type(value) + + # strict type check passes → no need to check method + if value_type in allowed_types: + return True + + method = getattr(value_type, method_name, None) + + if method is None: + return None + + if method in allowed_methods: + return True + + for module_name, *access_path in allowed_external: + if _has_original_dunder_external(value, module_name, access_path, method_name): + return True + + return False + + +@undoc +@dataclass +class SelectivePolicy(EvaluationPolicy): + allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) + allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) + + allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) + allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) + + allowed_operations: Set = field(default_factory=set) + allowed_operations_external: Set[Tuple[str, ...]] = field(default_factory=set) + + _operation_methods_cache: Dict[str, Set[Callable]] = field( + default_factory=dict, init=False + ) + + def can_get_attr(self, value, attr): + has_original_attribute = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattribute_methods, + allowed_external=self.allowed_getattr_external, + method_name="__getattribute__", + ) + has_original_attr = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattr_methods, + allowed_external=self.allowed_getattr_external, + method_name="__getattr__", + ) + + accept = False + + # Many objects do not have `__getattr__`, this is fine. + if has_original_attr is None and has_original_attribute: + accept = True + else: + # Accept objects without modifications to `__getattr__` and `__getattribute__` + accept = has_original_attr and has_original_attribute + + if accept: + # We still need to check for overriden properties. + + value_class = type(value) + if not hasattr(value_class, attr): + return True + + class_attr_val = getattr(value_class, attr) + is_property = isinstance(class_attr_val, property) + + if not is_property: + return True + + # Properties in allowed types are ok (although we do not include any + # properties in our default allow list currently). + if type(value) in self.allowed_getattr: + return True # pragma: no cover + + # Properties in subclasses of allowed types may be ok if not changed + for module_name, *access_path in self.allowed_getattr_external: + try: + external_class = _get_external(module_name, access_path) + external_class_attr_val = getattr(external_class, attr) + except (KeyError, AttributeError): + return False # pragma: no cover + return class_attr_val == external_class_attr_val + + return False + + def can_get_item(self, value, item): + """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" + return _has_original_dunder( + value, + allowed_types=self.allowed_getitem, + allowed_methods=self._getitem_methods, + allowed_external=self.allowed_getitem_external, + method_name="__getitem__", + ) + + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + objects = [a] + if b is not None: + objects.append(b) + return all( + [ + _has_original_dunder( + obj, + allowed_types=self.allowed_operations, + allowed_methods=self._operator_dunder_methods(dunder), + allowed_external=self.allowed_operations_external, + method_name=dunder, + ) + for dunder in dunders + for obj in objects + ] + ) + + def _operator_dunder_methods(self, dunder: str) -> Set[Callable]: + if dunder not in self._operation_methods_cache: + self._operation_methods_cache[dunder] = self._safe_get_methods( + self.allowed_operations, dunder + ) + return self._operation_methods_cache[dunder] + + @cached_property + def _getitem_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getitem, "__getitem__") + + @cached_property + def _getattr_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattr__") + + @cached_property + def _getattribute_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattribute__") + + def _safe_get_methods(self, classes, name) -> Set[Callable]: + return { + method + for class_ in classes + for method in [getattr(class_, name, None)] + if method + } + + +class _DummyNamedTuple(NamedTuple): + """Used internally to retrieve methods of named tuple instance.""" + + +class EvaluationContext(NamedTuple): + #: Local namespace + locals: dict + #: Global namespace + globals: dict + #: Evaluation policy identifier + evaluation: Literal[ + "forbidden", "minimal", "limited", "unsafe", "dangerous" + ] = "forbidden" + #: Whether the evalution of code takes place inside of a subscript. + #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. + in_subscript: bool = False + + +class _IdentitySubscript: + """Returns the key itself when item is requested via subscript.""" + + def __getitem__(self, key): + return key + + +IDENTITY_SUBSCRIPT = _IdentitySubscript() +SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" + + +class GuardRejection(Exception): + """Exception raised when guard rejects evaluation attempt.""" + + pass + + +def guarded_eval(code: str, context: EvaluationContext): + """Evaluate provided code in the evaluation context. + + If evaluation policy given by context is set to ``forbidden`` + no evaluation will be performed; if it is set to ``dangerous`` + standard :func:`eval` will be used; finally, for any other, + policy :func:`eval_node` will be called on parsed AST. + """ + locals_ = context.locals + + if context.evaluation == "forbidden": + raise GuardRejection("Forbidden mode") + + # note: not using `ast.literal_eval` as it does not implement + # getitem at all, for example it fails on simple `[0][1]` + + if context.in_subscript: + # syntatic sugar for ellipsis (:) is only available in susbcripts + # so we need to trick the ast parser into thinking that we have + # a subscript, but we need to be able to later recognise that we did + # it so we can ignore the actual __getitem__ operation + if not code: + return tuple() + locals_ = locals_.copy() + locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT + code = SUBSCRIPT_MARKER + "[" + code + "]" + context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}}) + + if context.evaluation == "dangerous": + return eval(code, context.globals, context.locals) + + expression = ast.parse(code, mode="eval") + + return eval_node(expression, context) + + +BINARY_OP_DUNDERS: Dict[Type[ast.operator], Tuple[str]] = { + ast.Add: ("__add__",), + ast.Sub: ("__sub__",), + ast.Mult: ("__mul__",), + ast.Div: ("__truediv__",), + ast.FloorDiv: ("__floordiv__",), + ast.Mod: ("__mod__",), + ast.Pow: ("__pow__",), + ast.LShift: ("__lshift__",), + ast.RShift: ("__rshift__",), + ast.BitOr: ("__or__",), + ast.BitXor: ("__xor__",), + ast.BitAnd: ("__and__",), + ast.MatMult: ("__matmul__",), +} + +COMP_OP_DUNDERS: Dict[Type[ast.cmpop], Tuple[str, ...]] = { + ast.Eq: ("__eq__",), + ast.NotEq: ("__ne__", "__eq__"), + ast.Lt: ("__lt__", "__gt__"), + ast.LtE: ("__le__", "__ge__"), + ast.Gt: ("__gt__", "__lt__"), + ast.GtE: ("__ge__", "__le__"), + ast.In: ("__contains__",), + # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially +} + +UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = { + ast.USub: ("__neg__",), + ast.UAdd: ("__pos__",), + # we have to check both __inv__ and __invert__! + ast.Invert: ("__invert__", "__inv__"), + ast.Not: ("__not__",), +} + + +def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]: + dunder = None + for op, candidate_dunder in dunders.items(): + if isinstance(node_op, op): + dunder = candidate_dunder + return dunder + + +def eval_node(node: Union[ast.AST, None], context: EvaluationContext): + """Evaluate AST node in provided context. + + Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. + + Does not evaluate actions that always have side effects: + + - class definitions (``class sth: ...``) + - function definitions (``def sth: ...``) + - variable assignments (``x = 1``) + - augmented assignments (``x += 1``) + - deletions (``del x``) + + Does not evaluate operations which do not return values: + + - assertions (``assert x``) + - pass (``pass``) + - imports (``import x``) + - control flow: + + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) + - loops (``for`` and `while``) + - exception handling + + The purpose of this function is to guard against unwanted side-effects; + it does not give guarantees on protection from malicious code execution. + """ + policy = EVALUATION_POLICIES[context.evaluation] + if node is None: + return None + if isinstance(node, ast.Expression): + return eval_node(node.body, context) + if isinstance(node, ast.BinOp): + left = eval_node(node.left, context) + right = eval_node(node.right, context) + dunders = _find_dunder(node.op, BINARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, left, right): + return getattr(left, dunders[0])(right) + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Compare): + left = eval_node(node.left, context) + all_true = True + negate = False + for op, right in zip(node.ops, node.comparators): + right = eval_node(right, context) + dunder = None + dunders = _find_dunder(op, COMP_OP_DUNDERS) + if not dunders: + if isinstance(op, ast.NotIn): + dunders = COMP_OP_DUNDERS[ast.In] + negate = True + if isinstance(op, ast.Is): + dunder = "is_" + if isinstance(op, ast.IsNot): + dunder = "is_" + negate = True + if not dunder and dunders: + dunder = dunders[0] + if dunder: + a, b = (right, left) if dunder == "__contains__" else (left, right) + if dunder == "is_" or dunders and policy.can_operate(dunders, a, b): + result = getattr(operator, dunder)(a, b) + if negate: + result = not result + if not result: + all_true = False + left = right + else: + raise GuardRejection( + f"Comparison (`{dunder}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + else: + raise ValueError( + f"Comparison `{dunder}` not supported" + ) # pragma: no cover + return all_true + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Index): + # deprecated since Python 3.9 + return eval_node(node.value, context) # pragma: no cover + if isinstance(node, ast.Tuple): + return tuple(eval_node(e, context) for e in node.elts) + if isinstance(node, ast.List): + return [eval_node(e, context) for e in node.elts] + if isinstance(node, ast.Set): + return {eval_node(e, context) for e in node.elts} + if isinstance(node, ast.Dict): + return dict( + zip( + [eval_node(k, context) for k in node.keys], + [eval_node(v, context) for v in node.values], + ) + ) + if isinstance(node, ast.Slice): + return slice( + eval_node(node.lower, context), + eval_node(node.upper, context), + eval_node(node.step, context), + ) + if isinstance(node, ast.ExtSlice): + # deprecated since Python 3.9 + return tuple([eval_node(dim, context) for dim in node.dims]) # pragma: no cover + if isinstance(node, ast.UnaryOp): + value = eval_node(node.operand, context) + dunders = _find_dunder(node.op, UNARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, value): + return getattr(value, dunders[0])() + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(value), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Subscript): + value = eval_node(node.value, context) + slice_ = eval_node(node.slice, context) + if policy.can_get_item(value, slice_): + return value[slice_] + raise GuardRejection( + "Subscript access (`__getitem__`) for", + type(value), # not joined to avoid calling `repr` + f" not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Name): + if policy.allow_locals_access and node.id in context.locals: + return context.locals[node.id] + if policy.allow_globals_access and node.id in context.globals: + return context.globals[node.id] + if policy.allow_builtins_access and hasattr(builtins, node.id): + # note: do not use __builtins__, it is implementation detail of cPython + return getattr(builtins, node.id) + if not policy.allow_globals_access and not policy.allow_locals_access: + raise GuardRejection( + f"Namespace access not allowed in {context.evaluation} mode" + ) + else: + raise NameError(f"{node.id} not found in locals, globals, nor builtins") + if isinstance(node, ast.Attribute): + value = eval_node(node.value, context) + if policy.can_get_attr(value, node.attr): + return getattr(value, node.attr) + raise GuardRejection( + "Attribute access (`__getattr__`) for", + type(value), # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.IfExp): + test = eval_node(node.test, context) + if test: + return eval_node(node.body, context) + else: + return eval_node(node.orelse, context) + if isinstance(node, ast.Call): + func = eval_node(node.func, context) + if policy.can_call(func) and not node.keywords: + args = [eval_node(arg, context) for arg in node.args] + return func(*args) + raise GuardRejection( + "Call for", + func, # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + raise ValueError("Unhandled node", ast.dump(node)) + + +SUPPORTED_EXTERNAL_GETITEM = { + ("pandas", "core", "indexing", "_iLocIndexer"), + ("pandas", "core", "indexing", "_LocIndexer"), + ("pandas", "DataFrame"), + ("pandas", "Series"), + ("numpy", "ndarray"), + ("numpy", "void"), +} + + +BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { + dict, + str, # type: ignore[arg-type] + bytes, # type: ignore[arg-type] + list, + tuple, + collections.defaultdict, + collections.deque, + collections.OrderedDict, + collections.ChainMap, + collections.UserDict, + collections.UserList, + collections.UserString, # type: ignore[arg-type] + _DummyNamedTuple, + _IdentitySubscript, +} + + +def _list_methods(cls, source=None): + """For use on immutable objects or with methods returning a copy""" + return [getattr(cls, k) for k in (source if source else dir(cls))] + + +dict_non_mutating_methods = ("copy", "keys", "values", "items") +list_non_mutating_methods = ("copy", "index", "count") +set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) + + +dict_keys: Type[collections.abc.KeysView] = type({}.keys()) +method_descriptor: Any = type(list.copy) + +NUMERICS = {int, float, complex} + +ALLOWED_CALLS = { + bytes, + *_list_methods(bytes), + dict, + *_list_methods(dict, dict_non_mutating_methods), + dict_keys.isdisjoint, + list, + *_list_methods(list, list_non_mutating_methods), + set, + *_list_methods(set, set_non_mutating_methods), + frozenset, + *_list_methods(frozenset), + range, + str, + *_list_methods(str), + tuple, + *_list_methods(tuple), + *NUMERICS, + *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)], + collections.deque, + *_list_methods(collections.deque, list_non_mutating_methods), + collections.defaultdict, + *_list_methods(collections.defaultdict, dict_non_mutating_methods), + collections.OrderedDict, + *_list_methods(collections.OrderedDict, dict_non_mutating_methods), + collections.UserDict, + *_list_methods(collections.UserDict, dict_non_mutating_methods), + collections.UserList, + *_list_methods(collections.UserList, list_non_mutating_methods), + collections.UserString, + *_list_methods(collections.UserString, dir(str)), + collections.Counter, + *_list_methods(collections.Counter, dict_non_mutating_methods), + collections.Counter.elements, + collections.Counter.most_common, +} + +BUILTIN_GETATTR: Set[MayHaveGetattr] = { + *BUILTIN_GETITEM, + set, + frozenset, + object, + type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + *NUMERICS, + dict_keys, + method_descriptor, +} + + +BUILTIN_OPERATIONS = {*BUILTIN_GETATTR} + +EVALUATION_POLICIES = { + "minimal": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=False, + allow_globals_access=False, + allow_item_access=False, + allow_attr_access=False, + allowed_calls=set(), + allow_any_calls=False, + allow_all_operations=False, + ), + "limited": SelectivePolicy( + allowed_getitem=BUILTIN_GETITEM, + allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, + allowed_getattr=BUILTIN_GETATTR, + allowed_getattr_external={ + # pandas Series/Frame implements custom `__getattr__` + ("pandas", "DataFrame"), + ("pandas", "Series"), + }, + allowed_operations=BUILTIN_OPERATIONS, + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allowed_calls=ALLOWED_CALLS, + ), + "unsafe": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allow_attr_access=True, + allow_item_access=True, + allow_any_calls=True, + allow_all_operations=True, + ), +} + + +__all__ = [ + "guarded_eval", + "eval_node", + "GuardRejection", + "EvaluationContext", + "_unbind_method", +] diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 21e428b54d4..45ed4e23a19 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -389,6 +389,9 @@ def _import_runner(self, proposal): displayhook_class = Type(DisplayHook) display_pub_class = Type(DisplayPublisher) compiler_class = Type(CachingCompiler) + inspector_class = Type( + oinspect.Inspector, help="Class to use to instantiate the shell inspector" + ).tag(config=True) sphinxify_docstring = Bool(False, help= """ @@ -755,10 +758,12 @@ def init_builtins(self): @observe('colors') def init_inspector(self, changes=None): # Object inspector - self.inspector = oinspect.Inspector(oinspect.InspectColors, - PyColorize.ANSICodeColors, - self.colors, - self.object_info_string_level) + self.inspector = self.inspector_class( + oinspect.InspectColors, + PyColorize.ANSICodeColors, + self.colors, + self.object_info_string_level, + ) def init_io(self): # implemented in subclasses, TerminalInteractiveShell does call @@ -2362,6 +2367,14 @@ def run_line_magic(self, magic_name: str, line, _stack_depth=1): kwargs['local_ns'] = self.get_local_scope(stack_depth) with self.builtin_trap: result = fn(*args, **kwargs) + + # The code below prevents the output from being displayed + # when using magics with decodator @output_can_be_silenced + # when the last Python token in the expression is a ';'. + if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): + if DisplayHook.semicolon_at_end_of_expression(magic_arg_s): + return None + return result def get_local_scope(self, stack_depth): @@ -2415,6 +2428,14 @@ def run_cell_magic(self, magic_name, line, cell): with self.builtin_trap: args = (magic_arg_s, cell) result = fn(*args, **kwargs) + + # The code below prevents the output from being displayed + # when using magics with decodator @output_can_be_silenced + # when the last Python token in the expression is a ';'. + if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): + if DisplayHook.semicolon_at_end_of_expression(cell): + return None + return result def find_line_magic(self, magic_name): @@ -2992,7 +3013,7 @@ def _run_cell( runner = _pseudo_sync_runner try: - return runner(coro) + result = runner(coro) except BaseException as e: info = ExecutionInfo( raw_cell, store_history, silent, shell_futures, cell_id @@ -3000,6 +3021,7 @@ def _run_cell( result = ExecutionResult(info) result.error_in_exec = e self.showtraceback(running_compiled_code=True) + finally: return result def should_run_async( @@ -3138,8 +3160,12 @@ def error_before_exec(value): else: cell = raw_cell + # Do NOT store paste/cpaste magic history + if "get_ipython().run_line_magic(" in cell and "paste" in cell: + store_history = False + # Store raw and processed history - if store_history and raw_cell.strip(" %") != "paste": + if store_history: self.history_manager.store_inputs(self.execution_count, cell, raw_cell) if not silent: self.logger.log(cell, raw_cell) @@ -3191,6 +3217,7 @@ def error_before_exec(value): # Execute the user code interactivity = "none" if silent else self.ast_node_interactivity + has_raised = await self.run_ast_nodes(code_ast.body, cell_name, interactivity=interactivity, compiler=compiler, result=result) diff --git a/IPython/core/logger.py b/IPython/core/logger.py index e3cb233cfa4..99e7ce29185 100644 --- a/IPython/core/logger.py +++ b/IPython/core/logger.py @@ -198,7 +198,16 @@ def log_write(self, data, kind='input'): odata = u'\n'.join([u'#[Out]# %s' % s for s in data.splitlines()]) write(u'%s\n' % odata) - self.logfile.flush() + try: + self.logfile.flush() + except OSError: + print("Failed to flush the log file.") + print( + f"Please check that {self.logfname} exists and have the right permissions." + ) + print( + "Also consider turning off the log with `%logstop` to avoid this warning." + ) def logstop(self): """Fully stop logging and close log file. diff --git a/IPython/core/magic.py b/IPython/core/magic.py index cedba619378..4f9e4e548f7 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -257,7 +257,8 @@ def mark(func, *a, **kw): return magic_deco -MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand' +MAGIC_NO_VAR_EXPAND_ATTR = "_ipython_magic_no_var_expand" +MAGIC_OUTPUT_CAN_BE_SILENCED = "_ipython_magic_output_can_be_silenced" def no_var_expand(magic_func): @@ -276,6 +277,16 @@ def no_var_expand(magic_func): return magic_func +def output_can_be_silenced(magic_func): + """Mark a magic function so its output may be silenced. + + The output is silenced if the Python code used as a parameter of + the magic ends in a semicolon, not counting a Python comment that can + follow it. + """ + setattr(magic_func, MAGIC_OUTPUT_CAN_BE_SILENCED, True) + return magic_func + # Create the actual decorators for public use # These three are used to decorate methods in class definitions diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index af69b02676d..7dfa84ce2d5 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -297,7 +297,10 @@ def page(self, parameter_s=''): oname = args and args or '_' info = self.shell._ofind(oname) if info['found']: - txt = (raw and str or pformat)( info['obj'] ) + if raw: + txt = str(info["obj"]) + else: + txt = pformat(info["obj"]) page.page(txt) else: print('Object `%s` not found' % oname) diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index f442ba15259..9e1cb38c254 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -68,92 +68,22 @@ def config(self, s): To view what is configurable on a given class, just pass the class name:: - In [2]: %config IPCompleter - IPCompleter(Completer) options - ---------------------------- - IPCompleter.backslash_combining_completions= - Enable unicode completions, e.g. \\alpha . Includes completion of latex - commands, unicode names, and expanding unicode characters back to latex - commands. - Current: True - IPCompleter.debug= - Enable debug for the Completer. Mostly print extra information for - experimental jedi integration. + In [2]: %config LoggingMagics + LoggingMagics(Magics) options + --------------------------- + LoggingMagics.quiet= + Suppress output of log state when logging is enabled Current: False - IPCompleter.disable_matchers=... - List of matchers to disable. - Current: [] - IPCompleter.greedy= - Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - Current: False - IPCompleter.jedi_compute_type_timeout= - Experimental: restrict time (in milliseconds) during which Jedi can compute types. - Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt - performance by preventing jedi to build its cache. - Current: 400 - IPCompleter.limit_to__all__= - DEPRECATED as of version 5.0. - Instruct the completer to use __all__ for the completion - Specifically, when completing on ``object.``. - When True: only those names in obj.__all__ will be included. - When False [default]: the __all__ attribute is ignored - Current: False - IPCompleter.merge_completions= - Whether to merge completion results into a single list - If False, only the completion results from the first non-empty - completer will be returned. - As of version 8.6.0, setting the value to ``False`` is an alias for: - ``IPCompleter.suppress_competing_matchers = True.``. - Current: True - IPCompleter.omit__names= - Instruct the completer to omit private method names - Specifically, when completing on ``object.``. - When 2 [default]: all names that start with '_' will be excluded. - When 1: all 'magic' names (``__foo__``) will be excluded. - When 0: nothing will be excluded. - Choices: any of [0, 1, 2] - Current: 2 - IPCompleter.profile_completions= - If True, emit profiling data for completion subsystem using cProfile. - Current: False - IPCompleter.profiler_output_dir= - Template for path at which to output profile data for completions. - Current: '.completion_profiles' - IPCompleter.suppress_competing_matchers= - Whether to suppress completions from other *Matchers*. - When set to ``None`` (default) the matchers will attempt to auto-detect - whether suppression of other matchers is desirable. For example, at the - beginning of a line followed by `%` we expect a magic completion to be the - only applicable option, and after ``my_dict['`` we usually expect a - completion with an existing dictionary key. - If you want to disable this heuristic and see completions from all matchers, - set ``IPCompleter.suppress_competing_matchers = False``. To disable the - heuristic for specific matchers provide a dictionary mapping: - ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': - False}``. - Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions - to the set of matchers with the highest priority; this is equivalent to - ``IPCompleter.merge_completions`` and can be beneficial for performance, but - will sometimes omit relevant candidates from matchers further down the - priority list. - Current: None - IPCompleter.use_jedi= - Experimental: Use Jedi to generate autocompletions. Default to True if jedi - is installed. - Current: True but the real use is in setting values:: - In [3]: %config IPCompleter.greedy = True + In [3]: %config LoggingMagics.quiet = True and these values are read from the user_ns if they are variables:: - In [4]: feeling_greedy=False + In [4]: feeling_quiet=False - In [5]: %config IPCompleter.greedy = feeling_greedy + In [5]: %config LoggingMagics.quiet = feeling_quiet """ from traitlets.config.loader import Config diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index da7f780b9cb..7b558d5bc6a 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -37,6 +37,7 @@ magics_class, needs_local_scope, no_var_expand, + output_can_be_silenced, on_off, ) from IPython.testing.skipdoctest import skip_doctest @@ -1194,6 +1195,7 @@ def timeit(self, line='', cell=None, local_ns=None): @no_var_expand @needs_local_scope @line_cell_magic + @output_can_be_silenced def time(self,line='', cell=None, local_ns=None): """Time execution of a Python statement or expression. diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index f3fa246567b..f64f1bce6ae 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -468,9 +468,9 @@ def set_env(self, parameter_s): string. Usage:\\ - %set_env var val: set value for var - %set_env var=val: set value for var - %set_env var=$val: set value for var, using python expansion if possible + :``%set_env var val``: set value for var + :``%set_env var=val``: set value for var + :``%set_env var=$val``: set value for var, using python expansion if possible """ split = '=' if '=' in parameter_s else ' ' bits = parameter_s.split(split, 1) diff --git a/IPython/core/magics/pylab.py b/IPython/core/magics/pylab.py index 0f3fff62faf..2a69453ac98 100644 --- a/IPython/core/magics/pylab.py +++ b/IPython/core/magics/pylab.py @@ -54,7 +54,7 @@ def matplotlib(self, line=''): If you are using the inline matplotlib backend in the IPython Notebook you can set which figure formats are enabled using the following:: - In [1]: from IPython.display import set_matplotlib_formats + In [1]: from matplotlib_inline.backend_inline import set_matplotlib_formats In [2]: set_matplotlib_formats('pdf', 'svg') @@ -65,9 +65,9 @@ def matplotlib(self, line=''): In [3]: %config InlineBackend.print_figure_kwargs = {'bbox_inches':None} - In addition, see the docstring of - `IPython.display.set_matplotlib_formats` and - `IPython.display.set_matplotlib_close` for more information on + In addition, see the docstrings of + `matplotlib_inline.backend_inline.set_matplotlib_formats` and + `matplotlib_inline.backend_inline.set_matplotlib_close` for more information on changing additional behaviors of the inline backend. Examples diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index 9fd2fc6c0dd..e0615c0ca85 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -210,7 +210,7 @@ def in_thread(coro): async def _handle_stream(stream, stream_arg, file_object): while True: - line = (await stream.readline()).decode("utf8") + line = (await stream.readline()).decode("utf8", errors="replace") if not line: break if stream_arg: diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index f1c454b2604..bcaa95c97fa 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -16,6 +16,7 @@ import ast import inspect from inspect import signature +import html import linecache import warnings import os @@ -530,8 +531,8 @@ def _mime_format(self, text:str, formatter=None) -> dict: """ defaults = { - 'text/plain': text, - 'text/html': '
' + text + '
' + "text/plain": text, + "text/html": f"
{html.escape(text)}
", } if formatter is None: @@ -542,66 +543,66 @@ def _mime_format(self, text:str, formatter=None) -> dict: if not isinstance(formatted, dict): # Handle the deprecated behavior of a formatter returning # a string instead of a mime bundle. - return { - 'text/plain': formatted, - 'text/html': '
' + formatted + '
' - } + return {"text/plain": formatted, "text/html": f"
{formatted}
"} else: return dict(defaults, **formatted) def format_mime(self, bundle): - - text_plain = bundle['text/plain'] - - text = '' - heads, bodies = list(zip(*text_plain)) - _len = max(len(h) for h in heads) - - for head, body in zip(heads, bodies): - body = body.strip('\n') - delim = '\n' if '\n' in body else ' ' - text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n' - - bundle['text/plain'] = text + """Format a mimebundle being created by _make_info_unformatted into a real mimebundle""" + # Format text/plain mimetype + if isinstance(bundle["text/plain"], (list, tuple)): + # bundle['text/plain'] is a list of (head, formatted body) pairs + lines = [] + _len = max(len(h) for h, _ in bundle["text/plain"]) + + for head, body in bundle["text/plain"]: + body = body.strip("\n") + delim = "\n" if "\n" in body else " " + lines.append( + f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}" + ) + + bundle["text/plain"] = "\n".join(lines) + + # Format the text/html mimetype + if isinstance(bundle["text/html"], (list, tuple)): + # bundle['text/html'] is a list of (head, formatted body) pairs + bundle["text/html"] = "\n".join( + (f"

{head}

\n{body}" for (head, body) in bundle["text/html"]) + ) return bundle - def _get_info( - self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + def _append_info_field( + self, bundle, title: str, key: str, info, omit_sections, formatter ): - """Retrieve an info dict and format it. - - Parameters - ---------- - obj : any - Object to inspect and return info from - oname : str (default: ''): - Name of the variable pointing to `obj`. - formatter : callable - info - already computed information - detail_level : integer - Granularity of detail level, if set to 1, give more information. - omit_sections : container[str] - Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) - """ - - info = self.info(obj, oname=oname, info=info, detail_level=detail_level) - - _mime = { - 'text/plain': [], - 'text/html': '', + """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted""" + if title in omit_sections or key in omit_sections: + return + field = info[key] + if field is not None: + formatted_field = self._mime_format(field, formatter) + bundle["text/plain"].append((title, formatted_field["text/plain"])) + bundle["text/html"].append((title, formatted_field["text/html"])) + + def _make_info_unformatted(self, obj, info, formatter, detail_level, omit_sections): + """Assemble the mimebundle as unformatted lists of information""" + bundle = { + "text/plain": [], + "text/html": [], } - def append_field(bundle, title:str, key:str, formatter=None): - if title in omit_sections or key in omit_sections: - return - field = info[key] - if field is not None: - formatted_field = self._mime_format(field, formatter) - bundle['text/plain'].append((title, formatted_field['text/plain'])) - bundle['text/html'] += '

' + title + '

\n' + formatted_field['text/html'] + '\n' + # A convenience function to simplify calls below + def append_field(bundle, title: str, key: str, formatter=None): + self._append_info_field( + bundle, + title=title, + key=key, + info=info, + omit_sections=omit_sections, + formatter=formatter, + ) def code_formatter(text): return { @@ -609,57 +610,82 @@ def code_formatter(text): 'text/html': pylight(text) } - if info['isalias']: - append_field(_mime, 'Repr', 'string_form') + if info["isalias"]: + append_field(bundle, "Repr", "string_form") elif info['ismagic']: if detail_level > 0: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) - append_field(_mime, 'File', 'file') + append_field(bundle, "Docstring", "docstring", formatter) + append_field(bundle, "File", "file") elif info['isclass'] or is_simple_callable(obj): # Functions, methods, classes - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Init signature', 'init_definition', code_formatter) - append_field(_mime, 'Docstring', 'docstring', formatter) - if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Init signature", "init_definition", code_formatter) + append_field(bundle, "Docstring", "docstring", formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Init docstring', 'init_docstring', formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) - append_field(_mime, 'File', 'file') - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'Subclasses', 'subclasses') + append_field(bundle, "File", "file") + append_field(bundle, "Type", "type_name") + append_field(bundle, "Subclasses", "subclasses") else: # General Python objects - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Call signature', 'call_def', code_formatter) - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'String form', 'string_form') + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Call signature", "call_def", code_formatter) + append_field(bundle, "Type", "type_name") + append_field(bundle, "String form", "string_form") # Namespace - if info['namespace'] != 'Interactive': - append_field(_mime, 'Namespace', 'namespace') + if info["namespace"] != "Interactive": + append_field(bundle, "Namespace", "namespace") - append_field(_mime, 'Length', 'length') - append_field(_mime, 'File', 'file') + append_field(bundle, "Length", "length") + append_field(bundle, "File", "file") # Source or docstring, depending on detail level and whether # source found. - if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) + append_field(bundle, "Docstring", "docstring", formatter) + + append_field(bundle, "Class docstring", "class_docstring", formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) + append_field(bundle, "Call docstring", "call_docstring", formatter) + return bundle - append_field(_mime, 'Class docstring', 'class_docstring', formatter) - append_field(_mime, 'Init docstring', 'init_docstring', formatter) - append_field(_mime, 'Call docstring', 'call_docstring', formatter) + def _get_info( + self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + ): + """Retrieve an info dict and format it. + + Parameters + ---------- + obj : any + Object to inspect and return info from + oname : str (default: ''): + Name of the variable pointing to `obj`. + formatter : callable + info + already computed information + detail_level : integer + Granularity of detail level, if set to 1, give more information. + omit_sections : container[str] + Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) + """ - return self.format_mime(_mime) + info = self.info(obj, oname=oname, info=info, detail_level=detail_level) + bundle = self._make_info_unformatted( + obj, info, formatter, detail_level=detail_level, omit_sections=omit_sections + ) + return self.format_mime(bundle) def pinfo( self, diff --git a/IPython/core/release.py b/IPython/core/release.py index d8eeae46901..79164040b5a 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -16,7 +16,7 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 6 +_version_minor = 10 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 180f8b1b01f..29325a0ad2b 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -278,7 +278,7 @@ def init_extensions(self): ) for ext in extensions: try: - self.log.info("Loading IPython extension: %s" % ext) + self.log.info("Loading IPython extension: %s", ext) self.shell.extension_manager.load_extension(ext) except: if self.reraise_ipython_extension_failures: diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index fd72cf7d57a..7783798eb36 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -24,6 +24,7 @@ provisionalcompleter, match_dict_keys, _deduplicate_completions, + _match_number_in_dict_key_prefix, completion_matcher, SimpleCompletion, CompletionContext, @@ -98,7 +99,7 @@ def test_unicode_range(): assert len_exp == len_test, message # fail if new unicode symbols have been added. - assert len_exp <= 138552, message + assert len_exp <= 143041, message @contextmanager @@ -112,6 +113,17 @@ def greedy_completion(): ip.Completer.greedy = greedy_original +@contextmanager +def evaluation_policy(evaluation: str): + ip = get_ipython() + evaluation_original = ip.Completer.evaluation + try: + ip.Completer.evaluation = evaluation + yield + finally: + ip.Completer.evaluation = evaluation_original + + @contextmanager def custom_matchers(matchers): ip = get_ipython() @@ -170,7 +182,6 @@ def check_line_split(splitter, test_specs): out = splitter.split_line(line, cursor_pos) assert out == split - def test_line_split(): """Basic line splitter test with default specs.""" sp = completer.CompletionSplitter() @@ -522,10 +533,10 @@ class Z: def test_greedy_completions(self): """ - Test the capability of the Greedy completer. + Test the capability of the Greedy completer. Most of the test here does not really show off the greedy completer, for proof - each of the text below now pass with Jedi. The greedy completer is capable of more. + each of the text below now pass with Jedi. The greedy completer is capable of more. See the :any:`test_dict_key_completion_contexts` @@ -841,18 +852,45 @@ def test_match_dict_keys(self): """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" - keys = ["foo", b"far"] - assert match_dict_keys(keys, "b'", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, "b'f", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, 'b"', delims=delims) == ('"', 2, ["far"]) - assert match_dict_keys(keys, 'b"f', delims=delims) == ('"', 2, ["far"]) + def match(*args, **kwargs): + quote, offset, matches = match_dict_keys(*args, delims=delims, **kwargs) + return quote, offset, list(matches) - assert match_dict_keys(keys, "'", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, "'f", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"]) - assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"]) - - match_dict_keys + keys = ["foo", b"far"] + assert match(keys, "b'") == ("'", 2, ["far"]) + assert match(keys, "b'f") == ("'", 2, ["far"]) + assert match(keys, 'b"') == ('"', 2, ["far"]) + assert match(keys, 'b"f') == ('"', 2, ["far"]) + + assert match(keys, "'") == ("'", 1, ["foo"]) + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, '"') == ('"', 1, ["foo"]) + assert match(keys, '"f') == ('"', 1, ["foo"]) + + # Completion on first item of tuple + keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")] + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, "33") == ("", 0, ["3333"]) + + # Completion on numbers + keys = [ + 0xDEADBEEF, + 1111, + 1234, + "1999", + 0b10101, + 22, + ] # 0xDEADBEEF = 3735928559; 0b10101 = 21 + assert match(keys, "0xdead") == ("", 0, ["0xdeadbeef"]) + assert match(keys, "1") == ("", 0, ["1111", "1234"]) + assert match(keys, "2") == ("", 0, ["21", "22"]) + assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"]) + + # Should yield on variables + assert match(keys, "a_variable") == ("", 0, []) + + # Should pass over invalid literals + assert match(keys, "'' ''") == ("", 0, []) def test_match_dict_keys_tuple(self): """ @@ -860,28 +898,94 @@ def test_match_dict_keys_tuple(self): does return what expected, and does not crash. """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" - + keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')] + def match(*args, extra=None, **kwargs): + quote, offset, matches = match_dict_keys( + *args, delims=delims, extra_prefix=extra, **kwargs + ) + return quote, offset, list(matches) + # Completion on first key == "foo" - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"]) - assert match_dict_keys(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"]) - assert match_dict_keys(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) - assert match_dict_keys(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["bar", "oof"]) + assert match(keys, '"', extra=("foo",)) == ('"', 1, ["bar", "oof"]) + assert match(keys, "'o", extra=("foo",)) == ("'", 1, ["oof"]) + assert match(keys, '"o', extra=("foo",)) == ('"', 1, ["oof"]) + assert match(keys, "b'", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"', extra=("foo",)) == ('"', 2, ["bar"]) + assert match(keys, "b'b", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"b', extra=("foo",)) == ('"', 2, ["bar"]) # No Completion - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) + assert match(keys, "'", extra=("no_foo",)) == ("'", 1, []) + assert match(keys, "'", extra=("fo",)) == ("'", 1, []) + + keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")] + assert match(keys, "'foo", extra=("foo1",)) == ("'", 1, ["foo2"]) + assert match(keys, "'foo", extra=("foo1", "foo2")) == ("'", 1, ["foo3"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3")) == ("'", 1, ["foo4"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3", "foo4")) == ( + "'", + 1, + [], + ) + + keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)] + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["2222"]) + assert match(keys, "", extra=("foo",)) == ("", 0, ["1111", "'2222'"]) + assert match(keys, "'", extra=(3333,)) == ("'", 1, ["bar"]) + assert match(keys, "", extra=(3333,)) == ("", 0, ["'bar'", "4444"]) + assert match(keys, "'", extra=("3333",)) == ("'", 1, []) + assert match(keys, "33") == ("", 0, ["3333"]) - keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')] - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2", "foo2"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + def test_dict_key_completion_closures(self): + ip = get_ipython() + complete = ip.Completer.complete + ip.Completer.auto_close_dict_keys = True + + ip.user_ns["d"] = { + # tuple only + ("aa", 11): None, + # tuple and non-tuple + ("bb", 22): None, + "bb": None, + # non-tuple only + "cc": None, + # numeric tuple only + (77, "x"): None, + # numeric tuple and non-tuple + (88, "y"): None, + 88: None, + # numeric non-tuple only + 99: None, + } + + _, matches = complete(line_buffer="d[") + # should append `, ` if matches a tuple only + self.assertIn("'aa', ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("'bb'", matches) + # should append `]` if matches and item only + self.assertIn("'cc']", matches) + + # should append `, ` if matches a tuple only + self.assertIn("77, ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("88", matches) + # should append `]` if matches and item only + self.assertIn("99]", matches) + + _, matches = complete(line_buffer="d['aa', ") + # should restrict matches to those matching tuple prefix + self.assertIn("11]", matches) + self.assertNotIn("'bb'", matches) + self.assertNotIn("'bb', ", matches) + self.assertNotIn("'bb']", matches) + self.assertNotIn("'cc'", matches) + self.assertNotIn("'cc', ", matches) + self.assertNotIn("'cc']", matches) + ip.Completer.auto_close_dict_keys = False def test_dict_key_completion_string(self): """Test dictionary key completion for string keys""" @@ -1038,6 +1142,35 @@ def test_dict_key_completion_string(self): self.assertNotIn("foo", matches) self.assertNotIn("bar", matches) + def test_dict_key_completion_numbers(self): + ip = get_ipython() + complete = ip.Completer.complete + + ip.user_ns["d"] = { + 0xDEADBEEF: None, # 3735928559 + 1111: None, + 1234: None, + "1999": None, + 0b10101: None, # 21 + 22: None, + } + _, matches = complete(line_buffer="d[1") + self.assertIn("1111", matches) + self.assertIn("1234", matches) + self.assertNotIn("1999", matches) + self.assertNotIn("'1999'", matches) + + _, matches = complete(line_buffer="d[0xdead") + self.assertIn("0xdeadbeef", matches) + + _, matches = complete(line_buffer="d[2") + self.assertIn("21", matches) + self.assertIn("22", matches) + + _, matches = complete(line_buffer="d[0b101") + self.assertIn("0b10101", matches) + self.assertIn("0b10110", matches) + def test_dict_key_completion_contexts(self): """Test expression contexts in which dict key completion occurs""" ip = get_ipython() @@ -1050,6 +1183,7 @@ class C: ip.user_ns["C"] = C ip.user_ns["get"] = lambda: d + ip.user_ns["nested"] = {"x": d} def assert_no_completion(**kwargs): _, matches = complete(**kwargs) @@ -1075,6 +1209,13 @@ def assert_completion(**kwargs): assert_completion(line_buffer="(d[") assert_completion(line_buffer="C.data[") + # nested dict completion + assert_completion(line_buffer="nested['x'][") + + with evaluation_policy("minimal"): + with pytest.raises(AssertionError): + assert_completion(line_buffer="nested['x'][") + # greedy flag def assert_completion(**kwargs): _, matches = complete(**kwargs) @@ -1162,12 +1303,22 @@ def test_struct_array_key_completion(self): _, matches = complete(line_buffer="d['") self.assertIn("my_head", matches) self.assertIn("my_data", matches) - # complete on a nested level - with greedy_completion(): + + def completes_on_nested(): ip.user_ns["d"] = numpy.zeros(2, dtype=dt) _, matches = complete(line_buffer="d[1]['my_head']['") self.assertTrue(any(["my_dt" in m for m in matches])) self.assertTrue(any(["my_df" in m for m in matches])) + # complete on a nested level + with greedy_completion(): + completes_on_nested() + + with evaluation_policy("limited"): + completes_on_nested() + + with evaluation_policy("minimal"): + with pytest.raises(AssertionError): + completes_on_nested() @dec.skip_without("pandas") def test_dataframe_key_completion(self): @@ -1180,6 +1331,17 @@ def test_dataframe_key_completion(self): _, matches = complete(line_buffer="d['") self.assertIn("hello", matches) self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[:, '") + self.assertIn("hello", matches) + self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[1:, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1:-1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[::, '") + self.assertIn("hello", matches) def test_dict_key_completion_invalids(self): """Smoke test cases dict key completion can't handle""" @@ -1396,6 +1558,65 @@ def configure(suppression_config): configure({"b_matcher": True}) _("do not suppress", ["completion_b"]) + configure(True) + _("do not suppress", ["completion_a"]) + + def test_matcher_suppression_with_iterator(self): + @completion_matcher(identifier="matcher_returning_iterator") + def matcher_returning_iterator(text): + return iter(["completion_iter"]) + + @completion_matcher(identifier="matcher_returning_list") + def matcher_returning_list(text): + return ["completion_list"] + + with custom_matchers([matcher_returning_iterator, matcher_returning_list]): + ip = get_ipython() + c = ip.Completer + + def _(text, expected): + c.use_jedi = False + s, matches = c.complete(text) + self.assertEqual(expected, matches) + + def configure(suppression_config): + cfg = Config() + cfg.IPCompleter.suppress_competing_matchers = suppression_config + c.update_config(cfg) + + configure(False) + _("---", ["completion_iter", "completion_list"]) + + configure(True) + _("---", ["completion_iter"]) + + configure(None) + _("--", ["completion_iter", "completion_list"]) + + def test_matcher_suppression_with_jedi(self): + ip = get_ipython() + c = ip.Completer + c.use_jedi = True + + def configure(suppression_config): + cfg = Config() + cfg.IPCompleter.suppress_competing_matchers = suppression_config + c.update_config(cfg) + + def _(): + with provisionalcompleter(): + matches = [completion.text for completion in c.completions("dict.", 5)] + self.assertIn("keys", matches) + + configure(False) + _() + + configure(True) + _() + + configure(None) + _() + def test_matcher_disabling(self): @completion_matcher(identifier="a_matcher") def a_matcher(text): @@ -1444,3 +1665,38 @@ def _(expected): _(["completion_b"]) a_matcher.matcher_priority = 3 _(["completion_a"]) + + +@pytest.mark.parametrize( + "input, expected", + [ + ["1.234", "1.234"], + # should match signed numbers + ["+1", "+1"], + ["-1", "-1"], + ["-1.0", "-1.0"], + ["-1.", "-1."], + ["+1.", "+1."], + [".1", ".1"], + # should not match non-numbers + ["1..", None], + ["..", None], + [".1.", None], + # should match after comma + [",1", "1"], + [", 1", "1"], + [", .1", ".1"], + [", +.1", "+.1"], + # should not match after trailing spaces + [".1 ", None], + # some complex cases + ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"], + ["0xdeadbeef", "0xdeadbeef"], + ["0b_1110_0101", "0b_1110_0101"], + # should not match if in an operation + ["1 + 1", None], + [", 1 + 1", None], + ], +) +def test_match_numeric_literal_for_dict_key(input, expected): + assert _match_number_in_dict_key_prefix(input) == expected diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py new file mode 100644 index 00000000000..905cf3ab8e3 --- /dev/null +++ b/IPython/core/tests/test_guarded_eval.py @@ -0,0 +1,570 @@ +from contextlib import contextmanager +from typing import NamedTuple +from functools import partial +from IPython.core.guarded_eval import ( + EvaluationContext, + GuardRejection, + guarded_eval, + _unbind_method, +) +from IPython.testing import decorators as dec +import pytest + + +def create_context(evaluation: str, **kwargs): + return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation) + + +forbidden = partial(create_context, "forbidden") +minimal = partial(create_context, "minimal") +limited = partial(create_context, "limited") +unsafe = partial(create_context, "unsafe") +dangerous = partial(create_context, "dangerous") + +LIMITED_OR_HIGHER = [limited, unsafe, dangerous] +MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] + + +@contextmanager +def module_not_installed(module: str): + import sys + + try: + to_restore = sys.modules[module] + del sys.modules[module] + except KeyError: + to_restore = None + try: + yield + finally: + sys.modules[module] = to_restore + + +def test_external_not_installed(): + """ + Because attribute check requires checking if object is not of allowed + external type, this tests logic for absence of external module. + """ + + class Custom: + def __init__(self): + self.test = 1 + + def __getattr__(self, key): + return key + + with module_not_installed("pandas"): + context = limited(x=Custom()) + with pytest.raises(GuardRejection): + guarded_eval("x.test", context) + + +@dec.skip_without("pandas") +def test_external_changed_api(monkeypatch): + """Check that the execution rejects if external API changed paths""" + import pandas as pd + + series = pd.Series([1], index=["a"]) + + with monkeypatch.context() as m: + m.delattr(pd, "Series") + context = limited(data=series) + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + +@dec.skip_without("pandas") +def test_pandas_series_iloc(): + import pandas as pd + + series = pd.Series([1], index=["a"]) + context = limited(data=series) + assert guarded_eval("data.iloc[0]", context) == 1 + + +def test_rejects_custom_properties(): + class BadProperty: + @property + def iloc(self): + return [None] + + series = BadProperty() + context = limited(data=series) + + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + +@dec.skip_without("pandas") +def test_accepts_non_overriden_properties(): + import pandas as pd + + class GoodProperty(pd.Series): + pass + + series = GoodProperty([1], index=["a"]) + context = limited(data=series) + + assert guarded_eval("data.iloc[0]", context) == 1 + + +@dec.skip_without("pandas") +def test_pandas_series(): + import pandas as pd + + context = limited(data=pd.Series([1], index=["a"])) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(KeyError): + guarded_eval('data["c"]', context) + + +@dec.skip_without("pandas") +def test_pandas_bad_series(): + import pandas as pd + + class BadItemSeries(pd.Series): + def __getitem__(self, key): + return "CUSTOM_ITEM" + + class BadAttrSeries(pd.Series): + def __getattr__(self, key): + return "CUSTOM_ATTR" + + bad_series = BadItemSeries([1], index=["a"]) + context = limited(data=bad_series) + + with pytest.raises(GuardRejection): + guarded_eval('data["a"]', context) + with pytest.raises(GuardRejection): + guarded_eval('data["c"]', context) + + # note: here result is a bit unexpected because + # pandas `__getattr__` calls `__getitem__`; + # FIXME - special case to handle it? + assert guarded_eval("data.a", context) == "CUSTOM_ITEM" + + context = unsafe(data=bad_series) + assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM" + + bad_attr_series = BadAttrSeries([1], index=["a"]) + context = limited(data=bad_attr_series) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(GuardRejection): + guarded_eval("data.a", context) + + +@dec.skip_without("pandas") +def test_pandas_dataframe_loc(): + import pandas as pd + from pandas.testing import assert_series_equal + + data = pd.DataFrame([{"a": 1}]) + context = limited(data=data) + assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"]) + + +def test_named_tuple(): + class GoodNamedTuple(NamedTuple): + a: str + pass + + class BadNamedTuple(NamedTuple): + a: str + + def __getitem__(self, key): + return None + + good = GoodNamedTuple(a="x") + bad = BadNamedTuple(a="x") + + context = limited(data=good) + assert guarded_eval("data[0]", context) == "x" + + context = limited(data=bad) + with pytest.raises(GuardRejection): + guarded_eval("data[0]", context) + + +def test_dict(): + context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) + assert guarded_eval('data["a"]', context) == 1 + assert guarded_eval('data["b"]', context) == {"x": 2} + assert guarded_eval('data["b"]["x"]', context) == 2 + assert guarded_eval('data["x", "y"]', context) == 3 + + assert guarded_eval("data.keys", context) + + +def test_set(): + context = limited(data={"a", "b"}) + assert guarded_eval("data.difference", context) + + +def test_list(): + context = limited(data=[1, 2, 3]) + assert guarded_eval("data[1]", context) == 2 + assert guarded_eval("data.copy", context) + + +def test_dict_literal(): + context = limited() + assert guarded_eval("{}", context) == {} + assert guarded_eval('{"a": 1}', context) == {"a": 1} + + +def test_list_literal(): + context = limited() + assert guarded_eval("[]", context) == [] + assert guarded_eval('[1, "a"]', context) == [1, "a"] + + +def test_set_literal(): + context = limited() + assert guarded_eval("set()", context) == set() + assert guarded_eval('{"a"}', context) == {"a"} + + +def test_evaluates_if_expression(): + context = limited() + assert guarded_eval("2 if True else 3", context) == 2 + assert guarded_eval("4 if False else 5", context) == 5 + + +def test_object(): + obj = object() + context = limited(obj=obj) + assert guarded_eval("obj.__dir__", context) == obj.__dir__ + + +@pytest.mark.parametrize( + "code,expected", + [ + ["int.numerator", int.numerator], + ["float.is_integer", float.is_integer], + ["complex.real", complex.real], + ], +) +def test_number_attributes(code, expected): + assert guarded_eval(code, limited()) == expected + + +def test_method_descriptor(): + context = limited() + assert guarded_eval("list.copy.__name__", context) == "copy" + + +@pytest.mark.parametrize( + "data,good,bad,expected", + [ + [[1, 2, 3], "data.index(2)", "data.append(4)", 1], + [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True], + ], +) +def test_evaluates_calls(data, good, bad, expected): + context = limited(data=data) + assert guarded_eval(good, context) == expected + + with pytest.raises(GuardRejection): + guarded_eval(bad, context) + + +@pytest.mark.parametrize( + "code,expected", + [ + ["(1\n+\n1)", 2], + ["list(range(10))[-1:]", [9]], + ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_complex_cases(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1", 1], + ["1.0", 1.0], + ["0xdeedbeef", 0xDEEDBEEF], + ["True", True], + ["None", None], + ["{}", {}], + ["[]", []], + ], +) +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_evaluates_literals(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["-5", -5], + ["+5", +5], + ["~5", -6], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_unary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1 + 1", 2], + ["3 - 1", 2], + ["2 * 3", 6], + ["5 // 2", 2], + ["5 / 2", 2.5], + ["5**2", 25], + ["2 >> 1", 1], + ["2 << 1", 4], + ["1 | 2", 3], + ["1 & 1", 1], + ["1 & 2", 0], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_binary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["2 > 1", True], + ["2 < 1", False], + ["2 <= 1", False], + ["2 <= 2", True], + ["1 >= 2", False], + ["2 >= 2", True], + ["2 == 2", True], + ["1 == 2", False], + ["1 != 2", True], + ["1 != 1", False], + ["1 < 4 < 3", False], + ["(1 < 4) < 3", True], + ["4 > 3 > 2 > 1", True], + ["4 > 3 > 2 > 9", False], + ["1 < 2 < 3 < 4", True], + ["9 < 2 < 3 < 4", False], + ["1 < 2 > 1 > 0 > -1 < 1", True], + ["1 in [1] in [[1]]", True], + ["1 in [1] in [[2]]", False], + ["1 in [1]", True], + ["0 in [1]", False], + ["1 not in [1]", False], + ["0 not in [1]", True], + ["True is True", True], + ["False is False", True], + ["True is False", False], + ["True is not True", False], + ["False is not True", True], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_comparisons(code, expected, context): + assert guarded_eval(code, context()) == expected + + +def test_guards_comparisons(): + class GoodEq(int): + pass + + class BadEq(int): + def __eq__(self, other): + assert False + + context = limited(bad=BadEq(1), good=GoodEq(1)) + + with pytest.raises(GuardRejection): + guarded_eval("bad == 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad != 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 == bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 != bad", context) + + assert guarded_eval("good == 1", context) is True + assert guarded_eval("good != 1", context) is False + assert guarded_eval("1 == good", context) is True + assert guarded_eval("1 != good", context) is False + + +def test_guards_unary_operations(): + class GoodOp(int): + pass + + class BadOpInv(int): + def __inv__(self, other): + assert False + + class BadOpInverse(int): + def __inv__(self, other): + assert False + + context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1)) + + with pytest.raises(GuardRejection): + guarded_eval("~bad1", context) + + with pytest.raises(GuardRejection): + guarded_eval("~bad2", context) + + +def test_guards_binary_operations(): + class GoodOp(int): + pass + + class BadOp(int): + def __add__(self, other): + assert False + + context = limited(good=GoodOp(1), bad=BadOp(1)) + + with pytest.raises(GuardRejection): + guarded_eval("1 + bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad + 1", context) + + assert guarded_eval("good + 1", context) == 2 + assert guarded_eval("1 + good", context) == 2 + + +def test_guards_attributes(): + class GoodAttr(float): + pass + + class BadAttr1(float): + def __getattr__(self, key): + assert False + + class BadAttr2(float): + def __getattribute__(self, key): + assert False + + context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5)) + + with pytest.raises(GuardRejection): + guarded_eval("bad1.as_integer_ratio", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad2.as_integer_ratio", context) + + assert guarded_eval("good.as_integer_ratio()", context) == (1, 2) + + +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_access_builtins(context): + assert guarded_eval("round", context()) == round + + +def test_access_builtins_fails(): + context = limited() + with pytest.raises(NameError): + guarded_eval("this_is_not_builtin", context) + + +def test_rejects_forbidden(): + context = forbidden() + with pytest.raises(GuardRejection): + guarded_eval("1", context) + + +def test_guards_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal" + ) + + with pytest.raises(GuardRejection): + guarded_eval("local_a", context) + + with pytest.raises(GuardRejection): + guarded_eval("global_b", context) + + +def test_access_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited" + ) + assert guarded_eval("local_a", context) == "a" + assert guarded_eval("global_b", context) == "b" + + +@pytest.mark.parametrize( + "code", + ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"], +) +@pytest.mark.parametrize("context", [minimal(), limited(), unsafe()]) +def test_rejects_side_effect_syntax(code, context): + with pytest.raises(SyntaxError): + guarded_eval(code, context) + + +def test_subscript(): + context = EvaluationContext( + locals={}, globals={}, evaluation="limited", in_subscript=True + ) + empty_slice = slice(None, None, None) + assert guarded_eval("", context) == tuple() + assert guarded_eval(":", context) == empty_slice + assert guarded_eval("1:2:3", context) == slice(1, 2, 3) + assert guarded_eval(':, "a"', context) == (empty_slice, "a") + + +def test_unbind_method(): + class X(list): + def index(self, k): + return "CUSTOM" + + x = X() + assert _unbind_method(x.index) is X.index + assert _unbind_method([].index) is list.index + assert _unbind_method(list.index) is None + + +def test_assumption_instance_attr_do_not_matter(): + """This is semi-specified in Python documentation. + + However, since the specification says 'not guaranted + to work' rather than 'is forbidden to work', future + versions could invalidate this assumptions. This test + is meant to catch such a change if it ever comes true. + """ + + class T: + def __getitem__(self, k): + return "a" + + def __getattr__(self, k): + return "a" + + def f(self): + return "b" + + t = T() + t.__getitem__ = f + t.__getattr__ = f + assert t[1] == "a" + assert t[1] == "a" + + +def test_assumption_named_tuples_share_getitem(): + """Check assumption on named tuples sharing __getitem__""" + from typing import NamedTuple + + class A(NamedTuple): + pass + + class B(NamedTuple): + pass + + assert A.__getitem__ == B.__getitem__ diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 982bd5a3b22..920d911b4c9 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -17,6 +17,7 @@ import sys import tempfile import unittest +import pytest from unittest import mock from os.path import join @@ -635,10 +636,23 @@ def test_control_c(self, *mocks): ) self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT) - def test_magic_warnings(self): - for magic_cmd in ("pip", "conda", "cd"): - with self.assertWarnsRegex(Warning, "You executed the system command"): - ip.system_raw(magic_cmd) + +@pytest.mark.parametrize("magic_cmd", ["pip", "conda", "cd"]) +def test_magic_warnings(magic_cmd): + if sys.platform == "win32": + to_mock = "os.system" + expected_arg, expected_kwargs = magic_cmd, dict() + else: + to_mock = "subprocess.call" + expected_arg, expected_kwargs = magic_cmd, dict( + shell=True, executable=os.environ.get("SHELL", None) + ) + + with mock.patch(to_mock, return_value=0) as mock_sub: + with pytest.warns(Warning, match=r"You executed the system command"): + ip.system_raw(magic_cmd) + mock_sub.assert_called_once_with(expected_arg, **expected_kwargs) + # TODO: Exit codes are currently ignored on Windows. class TestSystemPipedExitCode(ExitCodeChecks): @@ -1089,9 +1103,12 @@ def test_run_cell_asyncio_run(): def test_should_run_async(): - assert not ip.should_run_async("a = 5") - assert ip.should_run_async("await x") - assert ip.should_run_async("import asyncio; await asyncio.sleep(1)") + assert not ip.should_run_async("a = 5", transformed_cell="a = 5") + assert ip.should_run_async("await x", transformed_cell="await x") + assert ip.should_run_async( + "import asyncio; await asyncio.sleep(1)", + transformed_cell="import asyncio; await asyncio.sleep(1)", + ) def test_set_custom_completer(): @@ -1110,3 +1127,49 @@ def foo(*args, **kwargs): # clean up ip.Completer.custom_matchers.pop() + + +class TestShowTracebackAttack(unittest.TestCase): + """Test that the interactive shell is resilient against the client attack of + manipulating the showtracebacks method. These attacks shouldn't result in an + unhandled exception in the kernel.""" + + def setUp(self): + self.orig_showtraceback = interactiveshell.InteractiveShell.showtraceback + + def tearDown(self): + interactiveshell.InteractiveShell.showtraceback = self.orig_showtraceback + + def test_set_show_tracebacks_none(self): + """Test the case of the client setting showtracebacks to None""" + + result = ip.run_cell( + """ + import IPython.core.interactiveshell + IPython.core.interactiveshell.InteractiveShell.showtraceback = None + + assert False, "This should not raise an exception" + """ + ) + print(result) + + assert result.result is None + assert isinstance(result.error_in_exec, TypeError) + assert str(result.error_in_exec) == "'NoneType' object is not callable" + + def test_set_show_tracebacks_noop(self): + """Test the case of the client setting showtracebacks to a no op lambda""" + + result = ip.run_cell( + """ + import IPython.core.interactiveshell + IPython.core.interactiveshell.InteractiveShell.showtraceback = lambda *args, **kwargs: None + + assert False, "This should not raise an exception" + """ + ) + print(result) + + assert result.result is None + assert isinstance(result.error_in_exec, AssertionError) + assert str(result.error_in_exec) == "This should not raise an exception" diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 509dd66dd28..e64b959322b 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -416,6 +416,65 @@ def test_time(): with tt.AssertPrints("hihi", suppress=False): ip.run_cell("f('hi')") + +# ';' at the end of %time prevents instruction value to be printed. +# This tests fix for #13837. +def test_time_no_output_with_semicolon(): + ip = get_ipython() + + # Test %time cases + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456") + + with tt.AssertNotPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456;") + + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456 # Comment") + + with tt.AssertNotPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456; # Comment") + + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456 # ;Comment") + + # Test %%time cases + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456\n\n\n") + + with tt.AssertNotPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456;\n\n\n") + + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456 # Comment\n\n\n") + + with tt.AssertNotPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456; # Comment\n\n\n") + + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456 # ;Comment\n\n\n") + + def test_time_last_not_expression(): ip.run_cell("%%time\n" "var_1 = 1\n" diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index e83e2b4a0c1..18eff270829 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -173,7 +173,7 @@ def _format_traceback_lines(lines, Colors, has_colors: bool, lvals): def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): """ - Format filename lines with `In [n]` if it's the nth code cell or `File *.py` if it's a module. + Format filename lines with custom formatting from caching compiler or `File *.py` by default Parameters ---------- @@ -184,23 +184,29 @@ def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): ColorScheme's normal coloring to be used. """ ipinst = get_ipython() - - if ipinst is not None and file in ipinst.compile._filename_map: - file = "[%s]" % ipinst.compile._filename_map[file] + if ( + ipinst is not None + and (data := ipinst.compile.format_code_name(file)) is not None + ): + label, name = data if lineno is None: - tpl_link = f"Cell {ColorFilename}In {{file}}{ColorNormal}" + tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}" else: - tpl_link = f"Cell {ColorFilename}In {{file}}, line {{lineno}}{ColorNormal}" + tpl_link = ( + f"{{label}} {ColorFilename}{{name}}, line {{lineno}}{ColorNormal}" + ) else: - file = util_path.compress_user( + label = "File" + name = util_path.compress_user( py3compat.cast_unicode(file, util_path.fs_encoding) ) if lineno is None: - tpl_link = f"File {ColorFilename}{{file}}{ColorNormal}" + tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}" else: - tpl_link = f"File {ColorFilename}{{file}}:{{lineno}}{ColorNormal}" + # can we make this the more friendly ", line {{lineno}}", or do we need to preserve the formatting with the colon? + tpl_link = f"{{label}} {ColorFilename}{{name}}:{{lineno}}{ColorNormal}" - return tpl_link.format(file=file, lineno=lineno) + return tpl_link.format(label=label, name=name, lineno=lineno) #--------------------------------------------------------------------------- # Module classes diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 88637fbab9c..2c3c9db212d 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -367,7 +367,8 @@ class TestEnum(Enum): self.shell.run_code("assert func2() == 'changed'") self.shell.run_code("t = Test(); assert t.new_func() == 'changed'") self.shell.run_code("assert number == 1") - self.shell.run_code("assert TestEnum.B.value == 'added'") + if sys.version_info < (3, 12): + self.shell.run_code("assert TestEnum.B.value == 'added'") # ----------- TEST IMPORT FROM MODULE -------------------------- diff --git a/IPython/lib/latextools.py b/IPython/lib/latextools.py index 693060ba55c..f2aa5728844 100644 --- a/IPython/lib/latextools.py +++ b/IPython/lib/latextools.py @@ -160,38 +160,37 @@ def latex_to_png_dvipng(s, wrap, color='Black', scale=1.0): with workdir.joinpath(tmpfile).open("w", encoding="utf8") as f: f.writelines(genelatex(s, wrap)) - with open(os.devnull, 'wb') as devnull: - subprocess.check_call( - ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile], - cwd=workdir, - stdout=devnull, - stderr=devnull, - startupinfo=startupinfo, - ) - - resolution = round(150*scale) - subprocess.check_call( - [ - "dvipng", - "-T", - "tight", - "-D", - str(resolution), - "-z", - "9", - "-bg", - "Transparent", - "-o", - outfile, - dvifile, - "-fg", - color, - ], - cwd=workdir, - stdout=devnull, - stderr=devnull, - startupinfo=startupinfo, - ) + subprocess.check_call( + ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile], + cwd=workdir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + startupinfo=startupinfo, + ) + + resolution = round(150 * scale) + subprocess.check_call( + [ + "dvipng", + "-T", + "tight", + "-D", + str(resolution), + "-z", + "9", + "-bg", + "Transparent", + "-o", + outfile, + dvifile, + "-fg", + color, + ], + cwd=workdir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + startupinfo=startupinfo, + ) with workdir.joinpath(outfile).open("rb") as f: return f.read() diff --git a/IPython/lib/tests/test_lexers.py b/IPython/lib/tests/test_lexers.py index efa00d601ea..000b8fe6fd9 100644 --- a/IPython/lib/tests/test_lexers.py +++ b/IPython/lib/tests/test_lexers.py @@ -4,11 +4,14 @@ # Distributed under the terms of the Modified BSD License. from unittest import TestCase +from pygments import __version__ as pygments_version from pygments.token import Token from pygments.lexers import BashLexer from .. import lexers +pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) + class TestLexers(TestCase): """Collection of lexers tests""" @@ -18,25 +21,26 @@ def setUp(self): def testIPythonLexer(self): fragment = '!echo $HOME\n' - tokens = [ + bash_tokens = [ (Token.Operator, '!'), ] - tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) + ipylex_token = list(self.lexer.get_tokens(fragment)) + assert bash_tokens[:-1] == ipylex_token[:-1] - fragment_2 = '!' + fragment + fragment_2 = "!" + fragment tokens_2 = [ (Token.Operator, '!!'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = '\t %%!\n' + fragment[1:] tokens_2 = [ (Token.Text, '\t '), (Token.Operator, '%%!'), (Token.Text, '\n'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = 'x = ' + fragment tokens_2 = [ @@ -44,8 +48,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = ' + fragment tokens_2 = [ @@ -54,8 +58,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = %sx ' + fragment[1:] tokens_2 = [ @@ -67,8 +71,10 @@ def testIPythonLexer(self): (Token.Operator, '%'), (Token.Keyword, 'sx'), (Token.Text, ' '), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + if tokens_2[7] == (Token.Text, " ") and pyg214: # pygments 2.14+ + tokens_2[7] = (Token.Text.Whitespace, " ") + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'f = %R function () {}\n' tokens_2 = [ @@ -80,7 +86,7 @@ def testIPythonLexer(self): (Token.Keyword, 'R'), (Token.Text, ' function () {}\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '\t%%xyz\n$foo\n' tokens_2 = [ @@ -89,7 +95,7 @@ def testIPythonLexer(self): (Token.Keyword, 'xyz'), (Token.Text, '\n$foo\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '%system?\n' tokens_2 = [ @@ -98,7 +104,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x != y\n' tokens_2 = [ @@ -109,7 +115,7 @@ def testIPythonLexer(self): (Token.Name, 'y'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = ' ?math.sin\n' tokens_2 = [ @@ -118,7 +124,7 @@ def testIPythonLexer(self): (Token.Text, 'math.sin'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment = ' *int*?\n' tokens = [ @@ -126,7 +132,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment)) fragment = '%%writefile -a foo.py\nif a == b:\n pass' tokens = [ @@ -145,7 +151,9 @@ def testIPythonLexer(self): (Token.Keyword, 'pass'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + if tokens[10] == (Token.Text, "\n") and pyg214: # pygments 2.14+ + tokens[10] = (Token.Text.Whitespace, "\n") + assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1] fragment = '%%timeit\nmath.sin(0)' tokens = [ @@ -173,4 +181,4 @@ def testIPythonLexer(self): (Token.Punctuation, '>'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment)) diff --git a/IPython/lib/tests/test_pygments.py b/IPython/lib/tests/test_pygments.py new file mode 100644 index 00000000000..877b4221ffe --- /dev/null +++ b/IPython/lib/tests/test_pygments.py @@ -0,0 +1,26 @@ +from typing import List + +import pytest +import pygments.lexers +import pygments.lexer + +from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer + +#: the human-readable names of the IPython lexers with ``entry_points`` +EXPECTED_LEXER_NAMES = [ + cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer] +] + + +@pytest.fixture +def all_pygments_lexer_names() -> List[str]: + """Get all lexer names registered in pygments.""" + return {l[0] for l in pygments.lexers.get_all_lexers()} + + +@pytest.mark.parametrize("expected_lexer", EXPECTED_LEXER_NAMES) +def test_pygments_entry_points( + expected_lexer: str, all_pygments_lexer_names: List[str] +) -> None: + """Check whether the ``entry_points`` for ``pygments.lexers`` are correct.""" + assert expected_lexer in all_pygments_lexer_names diff --git a/IPython/py.typed b/IPython/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 9e3c7b2276a..c428e7917fd 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -981,8 +981,9 @@ def setup(self): self.shell.warning_is_error = warning_is_error # setup bookmark for saving figures directory - self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir, - store_history=False) + self.shell.process_input_line( + 'bookmark ipy_savedir "%s"' % savefig_dir, store_history=False + ) self.shell.clear_cout() return rgxin, rgxout, promptin, promptout diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index c867b553f2e..b5fc148ab1c 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -4,6 +4,7 @@ import os import sys from warnings import warn +from typing import Union as UnionType from IPython.core.async_helpers import get_asyncio_loop from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC @@ -49,6 +50,10 @@ from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook from .ptutils import IPythonPTCompleter, IPythonPTLexer from .shortcuts import create_ipython_shortcuts +from .shortcuts.auto_suggest import ( + NavigableAutoSuggestFromHistory, + AppendAutoSuggestionInAnyLine, +) PTK3 = ptk_version.startswith('3.') @@ -183,7 +188,10 @@ class TerminalInteractiveShell(InteractiveShell): 'menus, decrease for short and wide.' ).tag(config=True) - pt_app = None + pt_app: UnionType[PromptSession, None] = None + auto_suggest: UnionType[ + AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None + ] = None debugger_history = None debugger_history_file = Unicode( @@ -376,18 +384,27 @@ def _displayhook_class_default(self): ).tag(config=True) autosuggestions_provider = Unicode( - "AutoSuggestFromHistory", + "NavigableAutoSuggestFromHistory", help="Specifies from which source automatic suggestions are provided. " - "Can be set to `'AutoSuggestFromHistory`' or `None` to disable" - "automatic suggestions. Default is `'AutoSuggestFromHistory`'.", + "Can be set to ``'NavigableAutoSuggestFromHistory'`` (:kbd:`up` and " + ":kbd:`down` swap suggestions), ``'AutoSuggestFromHistory'``, " + " or ``None`` to disable automatic suggestions. " + "Default is `'NavigableAutoSuggestFromHistory`'.", allow_none=True, ).tag(config=True) def _set_autosuggestions(self, provider): + # disconnect old handler + if self.auto_suggest and isinstance( + self.auto_suggest, NavigableAutoSuggestFromHistory + ): + self.auto_suggest.disconnect() if provider is None: self.auto_suggest = None elif provider == "AutoSuggestFromHistory": self.auto_suggest = AutoSuggestFromHistory() + elif provider == "NavigableAutoSuggestFromHistory": + self.auto_suggest = NavigableAutoSuggestFromHistory() else: raise ValueError("No valid provider.") if self.pt_app: @@ -462,6 +479,8 @@ def prompt(): tempfile_suffix=".py", **self._extra_prompt_options() ) + if isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory): + self.auto_suggest.connect(self.pt_app) def _make_style_from_name_or_cls(self, name_or_cls): """ @@ -560,23 +579,39 @@ def get_message(): get_message = get_message() options = { - 'complete_in_thread': False, - 'lexer':IPythonPTLexer(), - 'reserve_space_for_menu':self.space_for_menu, - 'message': get_message, - 'prompt_continuation': ( - lambda width, lineno, is_soft_wrap: - PygmentsTokens(self.prompts.continuation_prompt_tokens(width))), - 'multiline': True, - 'complete_style': self.pt_complete_style, - + "complete_in_thread": False, + "lexer": IPythonPTLexer(), + "reserve_space_for_menu": self.space_for_menu, + "message": get_message, + "prompt_continuation": ( + lambda width, lineno, is_soft_wrap: PygmentsTokens( + self.prompts.continuation_prompt_tokens(width) + ) + ), + "multiline": True, + "complete_style": self.pt_complete_style, + "input_processors": [ # Highlight matching brackets, but only when this setting is # enabled, and only when the DEFAULT_BUFFER has the focus. - 'input_processors': [ConditionalProcessor( - processor=HighlightMatchingBracketProcessor(chars='[](){}'), - filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() & - Condition(lambda: self.highlight_matching_brackets))], - } + ConditionalProcessor( + processor=HighlightMatchingBracketProcessor(chars="[](){}"), + filter=HasFocus(DEFAULT_BUFFER) + & ~IsDone() + & Condition(lambda: self.highlight_matching_brackets), + ), + # Show auto-suggestion in lines other than the last line. + ConditionalProcessor( + processor=AppendAutoSuggestionInAnyLine(), + filter=HasFocus(DEFAULT_BUFFER) + & ~IsDone() + & Condition( + lambda: isinstance( + self.auto_suggest, NavigableAutoSuggestFromHistory + ) + ), + ), + ], + } if not PTK3: options['inputhook'] = self.inputhook @@ -647,7 +682,7 @@ def init_alias(self): self.alias_manager.soft_define_alias(cmd, cmd) - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super(TerminalInteractiveShell, self).__init__(*args, **kwargs) self._set_autosuggestions(self.autosuggestions_provider) self.init_prompt_toolkit_cli() diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index df4648b8914..6280bce3b20 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -156,7 +156,7 @@ def make_report(self,traceback): flags.update(frontend_flags) aliases = dict(base_aliases) -aliases.update(shell_aliases) +aliases.update(shell_aliases) # type: ignore[arg-type] #----------------------------------------------------------------------------- # Main classes and functions @@ -180,7 +180,7 @@ def start(self): class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): name = u'ipython' description = usage.cl_usage - crash_handler_class = IPAppCrashHandler + crash_handler_class = IPAppCrashHandler # typing: ignore[assignment] examples = _examples flags = flags diff --git a/IPython/terminal/magics.py b/IPython/terminal/magics.py index 66d532511b3..cea53e4a248 100644 --- a/IPython/terminal/magics.py +++ b/IPython/terminal/magics.py @@ -147,7 +147,7 @@ def cpaste(self, parameter_s=''): sentinel = opts.get('s', u'--') block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet)) - self.store_or_execute(block, name, store_history=False) + self.store_or_execute(block, name, store_history=True) @line_magic def paste(self, parameter_s=''): diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts/__init__.py similarity index 56% rename from IPython/terminal/shortcuts.py rename to IPython/terminal/shortcuts/__init__.py index 7d6de8b3b04..7ec9a2871a6 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -6,25 +6,38 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -import warnings +import os +import re import signal import sys -import re -import os -from typing import Callable - +import warnings +from typing import Callable, Dict, Union from prompt_toolkit.application.current import get_app from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER -from prompt_toolkit.filters import (has_focus, has_selection, Condition, - vi_insert_mode, emacs_insert_mode, has_completions, vi_mode) -from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline +from prompt_toolkit.filters import Condition, emacs_insert_mode, has_completions +from prompt_toolkit.filters import has_focus as has_focus_impl +from prompt_toolkit.filters import ( + has_selection, + has_suggestion, + vi_insert_mode, + vi_mode, +) from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.key_binding.bindings import named_commands as nc +from prompt_toolkit.key_binding.bindings.completion import ( + display_completions_like_readline, +) from prompt_toolkit.key_binding.vi_state import InputMode, ViState +from prompt_toolkit.layout.layout import FocusableElement +from IPython.terminal.shortcuts import auto_match as match +from IPython.terminal.shortcuts import auto_suggest from IPython.utils.decorators import undoc +__all__ = ["create_ipython_shortcuts"] + + @undoc @Condition def cursor_in_leading_ws(): @@ -32,75 +45,114 @@ def cursor_in_leading_ws(): return (not before) or before.isspace() -# Needed for to accept autosuggestions in vi insert mode -def _apply_autosuggest(event): - """ - Apply autosuggestion if at end of line. - """ - b = event.current_buffer - d = b.document - after_cursor = d.text[d.cursor_position :] - lines = after_cursor.split("\n") - end_of_current_line = lines[0].strip() - suggestion = b.suggestion - if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): - b.insert_text(suggestion.text) - else: - nc.end_of_line(event) +def has_focus(value: FocusableElement): + """Wrapper around has_focus adding a nice `__name__` to tester function""" + tester = has_focus_impl(value).func + tester.__name__ = f"is_focused({value})" + return Condition(tester) -def create_ipython_shortcuts(shell): - """Set up the prompt_toolkit keyboard shortcuts for IPython""" + +@undoc +@Condition +def has_line_below() -> bool: + document = get_app().current_buffer.document + return document.cursor_position_row < len(document.lines) - 1 + + +@undoc +@Condition +def has_line_above() -> bool: + document = get_app().current_buffer.document + return document.cursor_position_row != 0 + + +def create_ipython_shortcuts(shell, for_all_platforms: bool = False) -> KeyBindings: + """Set up the prompt_toolkit keyboard shortcuts for IPython. + + Parameters + ---------- + shell: InteractiveShell + The current IPython shell Instance + for_all_platforms: bool (default false) + This parameter is mostly used in generating the documentation + to create the shortcut binding for all the platforms, and export + them. + + Returns + ------- + KeyBindings + the keybinding instance for prompt toolkit. + + """ + # Warning: if possible, do NOT define handler functions in the locals + # scope of this function, instead define functions in the global + # scope, or a separate module, and include a user-friendly docstring + # describing the action. kb = KeyBindings() insert_mode = vi_insert_mode | emacs_insert_mode - if getattr(shell, 'handle_return', None): + if getattr(shell, "handle_return", None): return_handler = shell.handle_return(shell) else: return_handler = newline_or_execute_outer(shell) - kb.add('enter', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - ))(return_handler) + kb.add("enter", filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode))( + return_handler + ) + @Condition + def ebivim(): + return shell.emacs_bindings_in_vi_insert_mode + + @kb.add( + "escape", + "enter", + filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim), + ) def reformat_and_execute(event): - reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell) + """Reformat code and execute it""" + reformat_text_before_cursor( + event.current_buffer, event.current_buffer.document, shell + ) event.current_buffer.validate_and_handle() - kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - ))(reformat_and_execute) - kb.add("c-\\")(quit) - kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)) - )(previous_history_or_previous_completion) + kb.add("c-p", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))( + previous_history_or_previous_completion + ) - kb.add('c-n', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)) - )(next_history_or_next_completion) + kb.add("c-n", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))( + next_history_or_next_completion + ) - kb.add('c-g', filter=(has_focus(DEFAULT_BUFFER) & has_completions) - )(dismiss_completion) + kb.add("c-g", filter=(has_focus(DEFAULT_BUFFER) & has_completions))( + dismiss_completion + ) - kb.add('c-c', filter=has_focus(DEFAULT_BUFFER))(reset_buffer) + kb.add("c-c", filter=has_focus(DEFAULT_BUFFER))(reset_buffer) - kb.add('c-c', filter=has_focus(SEARCH_BUFFER))(reset_search_buffer) + kb.add("c-c", filter=has_focus(SEARCH_BUFFER))(reset_search_buffer) - supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP')) - kb.add('c-z', filter=supports_suspend)(suspend_to_bg) + supports_suspend = Condition(lambda: hasattr(signal, "SIGTSTP")) + kb.add("c-z", filter=supports_suspend)(suspend_to_bg) # Ctrl+I == Tab - kb.add('tab', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - & cursor_in_leading_ws - ))(indent_buffer) - kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode) - )(newline_autoindent_outer(shell.input_transformer_manager)) + kb.add( + "tab", + filter=( + has_focus(DEFAULT_BUFFER) + & ~has_selection + & insert_mode + & cursor_in_leading_ws + ), + )(indent_buffer) + kb.add("c-o", filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode))( + newline_autoindent_outer(shell.input_transformer_manager) + ) - kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor) + kb.add("f2", filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor) @Condition def auto_match(): @@ -119,10 +171,10 @@ def all_quotes_paired(quote, buf): return paired focused_insert = (vi_insert_mode | emacs_insert_mode) & has_focus(DEFAULT_BUFFER) - _preceding_text_cache = {} - _following_text_cache = {} + _preceding_text_cache: Dict[Union[str, Callable], Condition] = {} + _following_text_cache: Dict[Union[str, Callable], Condition] = {} - def preceding_text(pattern): + def preceding_text(pattern: Union[str, Callable]): if pattern in _preceding_text_cache: return _preceding_text_cache[pattern] @@ -131,7 +183,8 @@ def preceding_text(pattern): def _preceding_text(): app = get_app() before_cursor = app.current_buffer.document.current_line_before_cursor - return bool(pattern(before_cursor)) + # mypy can't infer if(callable): https://github.com/python/mypy/issues/3603 + return bool(pattern(before_cursor)) # type: ignore[operator] else: m = re.compile(pattern) @@ -141,6 +194,8 @@ def _preceding_text(): before_cursor = app.current_buffer.document.current_line_before_cursor return bool(m.match(before_cursor)) + _preceding_text.__name__ = f"preceding_text({pattern!r})" + condition = Condition(_preceding_text) _preceding_text_cache[pattern] = condition return condition @@ -156,6 +211,8 @@ def _following_text(): app = get_app() return bool(m.match(app.current_buffer.document.current_line_after_cursor)) + _following_text.__name__ = f"following_text({pattern!r})" + condition = Condition(_following_text) _following_text_cache[pattern] = condition return condition @@ -173,151 +230,104 @@ def not_inside_unclosed_string(): return not ('"' in s or "'" in s) # auto match - @kb.add("(", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("()") - event.current_buffer.cursor_left() - - @kb.add("[", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("[]") - event.current_buffer.cursor_left() + for key, cmd in match.auto_match_parens.items(): + kb.add(key, filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))( + cmd + ) - @kb.add("{", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("{}") - event.current_buffer.cursor_left() + # raw string + for key, cmd in match.auto_match_parens_raw_string.items(): + kb.add( + key, + filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"), + )(cmd) - @kb.add( + kb.add( '"', filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(lambda line: all_quotes_paired('"', line)) & following_text(r"[,)}\]]|$"), - ) - def _(event): - event.current_buffer.insert_text('""') - event.current_buffer.cursor_left() + )(match.double_quote) - @kb.add( + kb.add( "'", filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(lambda line: all_quotes_paired("'", line)) & following_text(r"[,)}\]]|$"), - ) - def _(event): - event.current_buffer.insert_text("''") - event.current_buffer.cursor_left() + )(match.single_quote) - @kb.add( + kb.add( '"', filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(r'^.*""$'), - ) - def _(event): - event.current_buffer.insert_text('""""') - event.current_buffer.cursor_left(3) + )(match.docstring_double_quotes) - @kb.add( + kb.add( "'", filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(r"^.*''$"), - ) - def _(event): - event.current_buffer.insert_text("''''") - event.current_buffer.cursor_left(3) + )(match.docstring_single_quotes) - # raw string - @kb.add( - "(", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + # just move cursor + kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))( + match.skip_over ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("()" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) - - @kb.add( - "[", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]"))( + match.skip_over ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("[]" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) - - @kb.add( - "{", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}"))( + match.skip_over + ) + kb.add('"', filter=focused_insert & auto_match & following_text('^"'))( + match.skip_over + ) + kb.add("'", filter=focused_insert & auto_match & following_text("^'"))( + match.skip_over ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("{}" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) - - # just move cursor - @kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)")) - @kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]")) - @kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}")) - @kb.add('"', filter=focused_insert & auto_match & following_text('^"')) - @kb.add("'", filter=focused_insert & auto_match & following_text("^'")) - def _(event): - event.current_buffer.cursor_right() - @kb.add( + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\($") & auto_match & following_text(r"^\)"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\[$") & auto_match & following_text(r"^\]"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\{$") & auto_match & following_text(r"^\}"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text('.*"$') & auto_match & following_text('^"'), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*'$") & auto_match & following_text(r"^'"), - ) - def _(event): - event.current_buffer.delete() - event.current_buffer.delete_before_cursor() + )(match.delete_pair) if shell.display_completions == "readlinelike": kb.add( @@ -330,41 +340,65 @@ def _(event): ), )(display_completions_like_readline) - if sys.platform == "win32": + if sys.platform == "win32" or for_all_platforms: kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste) - @Condition - def ebivim(): - return shell.emacs_bindings_in_vi_insert_mode - focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode - @kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode)) - def _(event): - _apply_autosuggest(event) - - @kb.add("c-e", filter=focused_insert_vi & ebivim) - def _(event): - _apply_autosuggest(event) - - @kb.add("c-f", filter=focused_insert_vi) - def _(event): - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - b.insert_text(suggestion.text) - else: - nc.forward_char(event) + # autosuggestions + @Condition + def navigable_suggestions(): + return isinstance( + shell.auto_suggest, auto_suggest.NavigableAutoSuggestFromHistory + ) - @kb.add("escape", "f", filter=focused_insert_vi & ebivim) - def _(event): - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - t = re.split(r"(\S+\s+)", suggestion.text) - b.insert_text(next((x for x in t if x), "")) - else: - nc.forward_word(event) + kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))( + auto_suggest.accept_in_vi_insert_mode + ) + kb.add("c-e", filter=focused_insert_vi & ebivim)( + auto_suggest.accept_in_vi_insert_mode + ) + kb.add("c-f", filter=focused_insert_vi)(auto_suggest.accept) + kb.add("escape", "f", filter=focused_insert_vi & ebivim)(auto_suggest.accept_word) + kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_token + ) + kb.add( + "escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER) & emacs_insert_mode + )(auto_suggest.discard) + kb.add( + "up", + filter=navigable_suggestions + & ~has_line_above + & has_suggestion + & has_focus(DEFAULT_BUFFER), + )(auto_suggest.swap_autosuggestion_up(shell.auto_suggest)) + kb.add( + "down", + filter=navigable_suggestions + & ~has_line_below + & has_suggestion + & has_focus(DEFAULT_BUFFER), + )(auto_suggest.swap_autosuggestion_down(shell.auto_suggest)) + kb.add( + "up", filter=has_line_above & navigable_suggestions & has_focus(DEFAULT_BUFFER) + )(auto_suggest.up_and_update_hint) + kb.add( + "down", + filter=has_line_below & navigable_suggestions & has_focus(DEFAULT_BUFFER), + )(auto_suggest.down_and_update_hint) + kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_character + ) + kb.add("c-left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_and_move_cursor_left + ) + kb.add("c-down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_and_keep_cursor + ) + kb.add("backspace", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.backspace_and_resume_hint + ) # Simple Control keybindings key_cmd_dict = { @@ -415,14 +449,13 @@ def set_input_mode(self, mode): self._input_mode = mode if shell.editing_mode == "vi" and shell.modal_cursor: - ViState._input_mode = InputMode.INSERT - ViState.input_mode = property(get_input_mode, set_input_mode) - + ViState._input_mode = InputMode.INSERT # type: ignore + ViState.input_mode = property(get_input_mode, set_input_mode) # type: ignore return kb def reformat_text_before_cursor(buffer, document, shell): - text = buffer.delete_before_cursor(len(document.text[:document.cursor_position])) + text = buffer.delete_before_cursor(len(document.text[: document.cursor_position])) try: formatted_text = shell.reformat_handler(text) buffer.insert_text(formatted_text) @@ -431,7 +464,6 @@ def reformat_text_before_cursor(buffer, document, shell): def newline_or_execute_outer(shell): - def newline_or_execute(event): """When the user presses return, insert a newline or execute the code.""" b = event.current_buffer @@ -450,34 +482,38 @@ def newline_or_execute(event): if d.line_count == 1: check_text = d.text else: - check_text = d.text[:d.cursor_position] + check_text = d.text[: d.cursor_position] status, indent = shell.check_complete(check_text) - + # if all we have after the cursor is whitespace: reformat current text # before cursor - after_cursor = d.text[d.cursor_position:] + after_cursor = d.text[d.cursor_position :] reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) reformatted = True - if not (d.on_last_line or - d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() - ): + if not ( + d.on_last_line + or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() + ): if shell.autoindent: - b.insert_text('\n' + indent) + b.insert_text("\n" + indent) else: - b.insert_text('\n') + b.insert_text("\n") return - if (status != 'incomplete') and b.accept_handler: + if (status != "incomplete") and b.accept_handler: if not reformatted: reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: - b.insert_text('\n' + indent) + b.insert_text("\n" + indent) else: - b.insert_text('\n') + b.insert_text("\n") + + newline_or_execute.__qualname__ = "newline_or_execute" + return newline_or_execute @@ -500,12 +536,14 @@ def next_history_or_next_completion(event): def dismiss_completion(event): + """Dismiss completion""" b = event.current_buffer if b.complete_state: b.cancel_completion() def reset_buffer(event): + """Reset buffer""" b = event.current_buffer if b.complete_state: b.cancel_completion() @@ -514,16 +552,22 @@ def reset_buffer(event): def reset_search_buffer(event): + """Reset search buffer""" if event.current_buffer.document.text: event.current_buffer.reset() else: event.app.layout.focus(DEFAULT_BUFFER) + def suspend_to_bg(event): + """Suspend to background""" event.app.suspend_to_background() + def quit(event): """ + Quit application with ``SIGQUIT`` if supported or ``sys.exit`` otherwise. + On platforms that support SIGQUIT, send SIGQUIT to the current process. On other platforms, just exit the process with a message. """ @@ -533,8 +577,11 @@ def quit(event): else: sys.exit("Quit") + def indent_buffer(event): - event.current_buffer.insert_text(' ' * 4) + """Indent buffer""" + event.current_buffer.insert_text(" " * 4) + @undoc def newline_with_copy_margin(event): @@ -546,9 +593,12 @@ def newline_with_copy_margin(event): Preserve margin and cursor position when using Control-O to insert a newline in EMACS mode """ - warnings.warn("`newline_with_copy_margin(event)` is deprecated since IPython 6.0. " - "see `newline_autoindent_outer(shell)(event)` for a replacement.", - DeprecationWarning, stacklevel=2) + warnings.warn( + "`newline_with_copy_margin(event)` is deprecated since IPython 6.0. " + "see `newline_autoindent_outer(shell)(event)` for a replacement.", + DeprecationWarning, + stacklevel=2, + ) b = event.current_buffer cursor_start_pos = b.document.cursor_position_col @@ -559,6 +609,7 @@ def newline_with_copy_margin(event): pos_diff = cursor_start_pos - cursor_end_pos b.cursor_right(count=pos_diff) + def newline_autoindent_outer(inputsplitter) -> Callable[..., None]: """ Return a function suitable for inserting a indented newline after the cursor. @@ -570,28 +621,33 @@ def newline_autoindent_outer(inputsplitter) -> Callable[..., None]: """ def newline_autoindent(event): - """insert a newline after the cursor indented appropriately.""" + """Insert a newline after the cursor indented appropriately.""" b = event.current_buffer d = b.document if b.complete_state: b.cancel_completion() - text = d.text[:d.cursor_position] + '\n' + text = d.text[: d.cursor_position] + "\n" _, indent = inputsplitter.check_complete(text) - b.insert_text('\n' + (' ' * (indent or 0)), move_cursor=False) + b.insert_text("\n" + (" " * (indent or 0)), move_cursor=False) + + newline_autoindent.__qualname__ = "newline_autoindent" return newline_autoindent def open_input_in_editor(event): + """Open code from input in external editor""" event.app.current_buffer.open_in_editor() -if sys.platform == 'win32': +if sys.platform == "win32": from IPython.core.error import TryNext - from IPython.lib.clipboard import (ClipboardEmpty, - win32_clipboard_get, - tkinter_clipboard_get) + from IPython.lib.clipboard import ( + ClipboardEmpty, + tkinter_clipboard_get, + win32_clipboard_get, + ) @undoc def win_paste(event): @@ -605,3 +661,10 @@ def win_paste(event): except ClipboardEmpty: return event.current_buffer.insert_text(text.replace("\t", " " * 4)) + +else: + + @undoc + def win_paste(event): + """Stub used when auto-generating shortcuts for documentation""" + pass diff --git a/IPython/terminal/shortcuts/auto_match.py b/IPython/terminal/shortcuts/auto_match.py new file mode 100644 index 00000000000..46cb1bd8754 --- /dev/null +++ b/IPython/terminal/shortcuts/auto_match.py @@ -0,0 +1,104 @@ +""" +Utilities function for keybinding with prompt toolkit. + +This will be bound to specific key press and filter modes, +like whether we are in edit mode, and whether the completer is open. +""" +import re +from prompt_toolkit.key_binding import KeyPressEvent + + +def parenthesis(event: KeyPressEvent): + """Auto-close parenthesis""" + event.current_buffer.insert_text("()") + event.current_buffer.cursor_left() + + +def brackets(event: KeyPressEvent): + """Auto-close brackets""" + event.current_buffer.insert_text("[]") + event.current_buffer.cursor_left() + + +def braces(event: KeyPressEvent): + """Auto-close braces""" + event.current_buffer.insert_text("{}") + event.current_buffer.cursor_left() + + +def double_quote(event: KeyPressEvent): + """Auto-close double quotes""" + event.current_buffer.insert_text('""') + event.current_buffer.cursor_left() + + +def single_quote(event: KeyPressEvent): + """Auto-close single quotes""" + event.current_buffer.insert_text("''") + event.current_buffer.cursor_left() + + +def docstring_double_quotes(event: KeyPressEvent): + """Auto-close docstring (double quotes)""" + event.current_buffer.insert_text('""""') + event.current_buffer.cursor_left(3) + + +def docstring_single_quotes(event: KeyPressEvent): + """Auto-close docstring (single quotes)""" + event.current_buffer.insert_text("''''") + event.current_buffer.cursor_left(3) + + +def raw_string_parenthesis(event: KeyPressEvent): + """Auto-close parenthesis in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) if matches else "" + event.current_buffer.insert_text("()" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def raw_string_bracket(event: KeyPressEvent): + """Auto-close bracker in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) if matches else "" + event.current_buffer.insert_text("[]" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def raw_string_braces(event: KeyPressEvent): + """Auto-close braces in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) if matches else "" + event.current_buffer.insert_text("{}" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def skip_over(event: KeyPressEvent): + """Skip over automatically added parenthesis. + + (rather than adding another parenthesis)""" + event.current_buffer.cursor_right() + + +def delete_pair(event: KeyPressEvent): + """Delete auto-closed parenthesis""" + event.current_buffer.delete() + event.current_buffer.delete_before_cursor() + + +auto_match_parens = {"(": parenthesis, "[": brackets, "{": braces} +auto_match_parens_raw_string = { + "(": raw_string_parenthesis, + "[": raw_string_bracket, + "{": raw_string_braces, +} diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py new file mode 100644 index 00000000000..3bfd6d54b83 --- /dev/null +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -0,0 +1,378 @@ +import re +import tokenize +from io import StringIO +from typing import Callable, List, Optional, Union, Generator, Tuple, Sequence + +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.key_binding import KeyPressEvent +from prompt_toolkit.key_binding.bindings import named_commands as nc +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory, Suggestion +from prompt_toolkit.document import Document +from prompt_toolkit.history import History +from prompt_toolkit.shortcuts import PromptSession +from prompt_toolkit.layout.processors import ( + Processor, + Transformation, + TransformationInput, +) + +from IPython.utils.tokenutil import generate_tokens + + +def _get_query(document: Document): + return document.lines[document.cursor_position_row] + + +class AppendAutoSuggestionInAnyLine(Processor): + """ + Append the auto suggestion to lines other than the last (appending to the + last line is natively supported by the prompt toolkit). + """ + + def __init__(self, style: str = "class:auto-suggestion") -> None: + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + is_last_line = ti.lineno == ti.document.line_count - 1 + is_active_line = ti.lineno == ti.document.cursor_position_row + + if not is_last_line and is_active_line: + buffer = ti.buffer_control.buffer + + if buffer.suggestion and ti.document.is_cursor_at_the_end_of_line: + suggestion = buffer.suggestion.text + else: + suggestion = "" + + return Transformation(fragments=ti.fragments + [(self.style, suggestion)]) + else: + return Transformation(fragments=ti.fragments) + + +class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory): + """ + A subclass of AutoSuggestFromHistory that allow navigation to next/previous + suggestion from history. To do so it remembers the current position, but it + state need to carefully be cleared on the right events. + """ + + def __init__( + self, + ): + self.skip_lines = 0 + self._connected_apps = [] + + def reset_history_position(self, _: Buffer): + self.skip_lines = 0 + + def disconnect(self): + for pt_app in self._connected_apps: + text_insert_event = pt_app.default_buffer.on_text_insert + text_insert_event.remove_handler(self.reset_history_position) + + def connect(self, pt_app: PromptSession): + self._connected_apps.append(pt_app) + # note: `on_text_changed` could be used for a bit different behaviour + # on character deletion (i.e. reseting history position on backspace) + pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position) + pt_app.default_buffer.on_cursor_position_changed.add_handler(self._dismiss) + + def get_suggestion( + self, buffer: Buffer, document: Document + ) -> Optional[Suggestion]: + text = _get_query(document) + + if text.strip(): + for suggestion, _ in self._find_next_match( + text, self.skip_lines, buffer.history + ): + return Suggestion(suggestion) + + return None + + def _dismiss(self, buffer, *args, **kwargs): + buffer.suggestion = None + + def _find_match( + self, text: str, skip_lines: float, history: History, previous: bool + ) -> Generator[Tuple[str, float], None, None]: + """ + text : str + Text content to find a match for, the user cursor is most of the + time at the end of this text. + skip_lines : float + number of items to skip in the search, this is used to indicate how + far in the list the user has navigated by pressing up or down. + The float type is used as the base value is +inf + history : History + prompt_toolkit History instance to fetch previous entries from. + previous : bool + Direction of the search, whether we are looking previous match + (True), or next match (False). + + Yields + ------ + Tuple with: + str: + current suggestion. + float: + will actually yield only ints, which is passed back via skip_lines, + which may be a +inf (float) + + + """ + line_number = -1 + for string in reversed(list(history.get_strings())): + for line in reversed(string.splitlines()): + line_number += 1 + if not previous and line_number < skip_lines: + continue + # do not return empty suggestions as these + # close the auto-suggestion overlay (and are useless) + if line.startswith(text) and len(line) > len(text): + yield line[len(text) :], line_number + if previous and line_number >= skip_lines: + return + + def _find_next_match( + self, text: str, skip_lines: float, history: History + ) -> Generator[Tuple[str, float], None, None]: + return self._find_match(text, skip_lines, history, previous=False) + + def _find_previous_match(self, text: str, skip_lines: float, history: History): + return reversed( + list(self._find_match(text, skip_lines, history, previous=True)) + ) + + def up(self, query: str, other_than: str, history: History) -> None: + for suggestion, line_number in self._find_next_match( + query, self.skip_lines, history + ): + # if user has history ['very.a', 'very', 'very.b'] and typed 'very' + # we want to switch from 'very.b' to 'very.a' because a) if the + # suggestion equals current text, prompt-toolkit aborts suggesting + # b) user likely would not be interested in 'very' anyways (they + # already typed it). + if query + suggestion != other_than: + self.skip_lines = line_number + break + else: + # no matches found, cycle back to beginning + self.skip_lines = 0 + + def down(self, query: str, other_than: str, history: History) -> None: + for suggestion, line_number in self._find_previous_match( + query, self.skip_lines, history + ): + if query + suggestion != other_than: + self.skip_lines = line_number + break + else: + # no matches found, cycle to end + for suggestion, line_number in self._find_previous_match( + query, float("Inf"), history + ): + if query + suggestion != other_than: + self.skip_lines = line_number + break + + +# Needed for to accept autosuggestions in vi insert mode +def accept_in_vi_insert_mode(event: KeyPressEvent): + """Apply autosuggestion if at end of line.""" + buffer = event.current_buffer + d = buffer.document + after_cursor = d.text[d.cursor_position :] + lines = after_cursor.split("\n") + end_of_current_line = lines[0].strip() + suggestion = buffer.suggestion + if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): + buffer.insert_text(suggestion.text) + else: + nc.end_of_line(event) + + +def accept(event: KeyPressEvent): + """Accept autosuggestion""" + buffer = event.current_buffer + suggestion = buffer.suggestion + if suggestion: + buffer.insert_text(suggestion.text) + else: + nc.forward_char(event) + + +def discard(event: KeyPressEvent): + """Discard autosuggestion""" + buffer = event.current_buffer + buffer.suggestion = None + + +def accept_word(event: KeyPressEvent): + """Fill partial autosuggestion by word""" + buffer = event.current_buffer + suggestion = buffer.suggestion + if suggestion: + t = re.split(r"(\S+\s+)", suggestion.text) + buffer.insert_text(next((x for x in t if x), "")) + else: + nc.forward_word(event) + + +def accept_character(event: KeyPressEvent): + """Fill partial autosuggestion by character""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion and suggestion.text: + b.insert_text(suggestion.text[0]) + + +def accept_and_keep_cursor(event: KeyPressEvent): + """Accept autosuggestion and keep cursor in place""" + buffer = event.current_buffer + old_position = buffer.cursor_position + suggestion = buffer.suggestion + if suggestion: + buffer.insert_text(suggestion.text) + buffer.cursor_position = old_position + + +def accept_and_move_cursor_left(event: KeyPressEvent): + """Accept autosuggestion and move cursor left in place""" + accept_and_keep_cursor(event) + nc.backward_char(event) + + +def _update_hint(buffer: Buffer): + if buffer.auto_suggest: + suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + + +def backspace_and_resume_hint(event: KeyPressEvent): + """Resume autosuggestions after deleting last character""" + current_buffer = event.current_buffer + + def resume_hinting(buffer: Buffer): + _update_hint(buffer) + current_buffer.on_text_changed.remove_handler(resume_hinting) + + current_buffer.on_text_changed.add_handler(resume_hinting) + nc.backward_delete_char(event) + + +def up_and_update_hint(event: KeyPressEvent): + """Go up and update hint""" + current_buffer = event.current_buffer + + current_buffer.auto_up(count=event.arg) + _update_hint(current_buffer) + + +def down_and_update_hint(event: KeyPressEvent): + """Go down and update hint""" + current_buffer = event.current_buffer + + current_buffer.auto_down(count=event.arg) + _update_hint(current_buffer) + + +def accept_token(event: KeyPressEvent): + """Fill partial autosuggestion by token""" + b = event.current_buffer + suggestion = b.suggestion + + if suggestion: + prefix = _get_query(b.document) + text = prefix + suggestion.text + + tokens: List[Optional[str]] = [None, None, None] + substrings = [""] + i = 0 + + for token in generate_tokens(StringIO(text).readline): + if token.type == tokenize.NEWLINE: + index = len(text) + else: + index = text.index(token[1], len(substrings[-1])) + substrings.append(text[:index]) + tokenized_so_far = substrings[-1] + if tokenized_so_far.startswith(prefix): + if i == 0 and len(tokenized_so_far) > len(prefix): + tokens[0] = tokenized_so_far[len(prefix) :] + substrings.append(tokenized_so_far) + i += 1 + tokens[i] = token[1] + if i == 2: + break + i += 1 + + if tokens[0]: + to_insert: str + insert_text = substrings[-2] + if tokens[1] and len(tokens[1]) == 1: + insert_text = substrings[-1] + to_insert = insert_text[len(prefix) :] + b.insert_text(to_insert) + return + + nc.forward_word(event) + + +Provider = Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] + + +def _swap_autosuggestion( + buffer: Buffer, + provider: NavigableAutoSuggestFromHistory, + direction_method: Callable, +): + """ + We skip most recent history entry (in either direction) if it equals the + current autosuggestion because if user cycles when auto-suggestion is shown + they most likely want something else than what was suggested (otherwise + they would have accepted the suggestion). + """ + suggestion = buffer.suggestion + if not suggestion: + return + + query = _get_query(buffer.document) + current = query + suggestion.text + + direction_method(query=query, other_than=current, history=buffer.history) + + new_suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = new_suggestion + + +def swap_autosuggestion_up(provider: Provider): + def swap_autosuggestion_up(event: KeyPressEvent): + """Get next autosuggestion from history.""" + if not isinstance(provider, NavigableAutoSuggestFromHistory): + return + + return _swap_autosuggestion( + buffer=event.current_buffer, provider=provider, direction_method=provider.up + ) + + swap_autosuggestion_up.__name__ = "swap_autosuggestion_up" + return swap_autosuggestion_up + + +def swap_autosuggestion_down( + provider: Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] +): + def swap_autosuggestion_down(event: KeyPressEvent): + """Get previous autosuggestion from history.""" + if not isinstance(provider, NavigableAutoSuggestFromHistory): + return + + return _swap_autosuggestion( + buffer=event.current_buffer, + provider=provider, + direction_method=provider.down, + ) + + swap_autosuggestion_down.__name__ = "swap_autosuggestion_down" + return swap_autosuggestion_down diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 68dbe372d91..01008d78369 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -7,11 +7,25 @@ import unittest import os +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory + from IPython.core.inputtransformer import InputTransformer from IPython.testing import tools as tt from IPython.utils.capture import capture_output from IPython.terminal.ptutils import _elide, _adjust_completion_text_based_on_context +from IPython.terminal.shortcuts.auto_suggest import NavigableAutoSuggestFromHistory + + +class TestAutoSuggest(unittest.TestCase): + def test_changing_provider(self): + ip = get_ipython() + ip.autosuggestions_provider = None + self.assertEqual(ip.auto_suggest, None) + ip.autosuggestions_provider = "AutoSuggestFromHistory" + self.assertIsInstance(ip.auto_suggest, AutoSuggestFromHistory) + ip.autosuggestions_provider = "NavigableAutoSuggestFromHistory" + self.assertIsInstance(ip.auto_suggest, NavigableAutoSuggestFromHistory) class TestElide(unittest.TestCase): @@ -24,10 +38,10 @@ def test_elide(self): ) test_string = os.sep.join(["", 10 * "a", 10 * "b", 10 * "c", ""]) - expect_stirng = ( + expect_string = ( os.sep + "a" + "\N{HORIZONTAL ELLIPSIS}" + "b" + os.sep + 10 * "c" ) - self.assertEqual(_elide(test_string, ""), expect_stirng) + self.assertEqual(_elide(test_string, ""), expect_string) def test_elide_typed_normal(self): self.assertEqual( diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py new file mode 100644 index 00000000000..309205d4f54 --- /dev/null +++ b/IPython/terminal/tests/test_shortcuts.py @@ -0,0 +1,318 @@ +import pytest +from IPython.terminal.shortcuts.auto_suggest import ( + accept, + accept_in_vi_insert_mode, + accept_token, + accept_character, + accept_word, + accept_and_keep_cursor, + discard, + NavigableAutoSuggestFromHistory, + swap_autosuggestion_up, + swap_autosuggestion_down, +) + +from prompt_toolkit.history import InMemoryHistory +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.document import Document +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory + +from unittest.mock import patch, Mock + + +def make_event(text, cursor, suggestion): + event = Mock() + event.current_buffer = Mock() + event.current_buffer.suggestion = Mock() + event.current_buffer.text = text + event.current_buffer.cursor_position = cursor + event.current_buffer.suggestion.text = suggestion + event.current_buffer.document = Document(text=text, cursor_position=cursor) + return event + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):"), + ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):"), + ], +) +def test_accept(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + accept(event) + assert buffer.insert_text.called + assert buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion", + [ + ("", "def out(tag: str, n=50):"), + ("def ", "out(tag: str, n=50):"), + ], +) +def test_discard(text, suggestion): + event = make_event(text, len(text), suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + discard(event) + assert not buffer.insert_text.called + assert buffer.suggestion is None + + +@pytest.mark.parametrize( + "text, cursor, suggestion, called", + [ + ("123456", 6, "123456789", True), + ("123456", 3, "123456789", False), + ("123456 \n789", 6, "123456789", True), + ], +) +def test_autosuggest_at_EOL(text, cursor, suggestion, called): + """ + test that autosuggest is only applied at end of line. + """ + + event = make_event(text, cursor, suggestion) + event.current_buffer.insert_text = Mock() + accept_in_vi_insert_mode(event) + if called: + event.current_buffer.insert_text.assert_called() + else: + event.current_buffer.insert_text.assert_not_called() + # event.current_buffer.document.get_end_of_line_position.assert_called() + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def "), + ("d", "ef out(tag: str, n=50):", "ef "), + ("de ", "f out(tag: str, n=50):", "f "), + ("def", " out(tag: str, n=50):", " "), + ("def ", "out(tag: str, n=50):", "out("), + ("def o", "ut(tag: str, n=50):", "ut("), + ("def ou", "t(tag: str, n=50):", "t("), + ("def out", "(tag: str, n=50):", "("), + ("def out(", "tag: str, n=50):", "tag: "), + ("def out(t", "ag: str, n=50):", "ag: "), + ("def out(ta", "g: str, n=50):", "g: "), + ("def out(tag", ": str, n=50):", ": "), + ("def out(tag:", " str, n=50):", " "), + ("def out(tag: ", "str, n=50):", "str, "), + ("def out(tag: s", "tr, n=50):", "tr, "), + ("def out(tag: st", "r, n=50):", "r, "), + ("def out(tag: str", ", n=50):", ", n"), + ("def out(tag: str,", " n=50):", " n"), + ("def out(tag: str, ", "n=50):", "n="), + ("def out(tag: str, n", "=50):", "="), + ("def out(tag: str, n=", "50):", "50)"), + ("def out(tag: str, n=5", "0):", "0)"), + ("def out(tag: str, n=50", "):", "):"), + ("def out(tag: str, n=50)", ":", ":"), + ], +) +def test_autosuggest_token(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_token(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "d"), + ("d", "ef out(tag: str, n=50):", "e"), + ("de ", "f out(tag: str, n=50):", "f"), + ("def", " out(tag: str, n=50):", " "), + ], +) +def test_accept_character(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_character(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def "), + ("d", "ef out(tag: str, n=50):", "ef "), + ("de", "f out(tag: str, n=50):", "f "), + ("def", " out(tag: str, n=50):", " "), + # (this is why we also have accept_token) + ("def ", "out(tag: str, n=50):", "out(tag: "), + ], +) +def test_accept_word(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_word(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion, expected, cursor", + [ + ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):", 0), + ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):", 4), + ], +) +def test_accept_and_keep_cursor(text, suggestion, expected, cursor): + event = make_event(text, cursor, suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + accept_and_keep_cursor(event) + assert buffer.insert_text.called + assert buffer.insert_text.call_args[0] == (expected,) + assert buffer.cursor_position == cursor + + +def test_autosuggest_token_empty(): + full = "def out(tag: str, n=50):" + event = make_event(full, len(full), "") + event.current_buffer.insert_text = Mock() + + with patch( + "prompt_toolkit.key_binding.bindings.named_commands.forward_word" + ) as forward_word: + accept_token(event) + assert not event.current_buffer.insert_text.called + assert forward_word.called + + +def test_other_providers(): + """Ensure that swapping autosuggestions does not break with other providers""" + provider = AutoSuggestFromHistory() + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + event = Mock() + event.current_buffer = Buffer() + assert up(event) is None + assert down(event) is None + + +async def test_navigable_provider(): + provider = NavigableAutoSuggestFromHistory() + history = InMemoryHistory(history_strings=["very_a", "very", "very_b", "very_c"]) + buffer = Buffer(history=history) + + async for _ in history.load(): + pass + + buffer.cursor_position = 5 + buffer.text = "very" + + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + + event = Mock() + event.current_buffer = buffer + + def get_suggestion(): + suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + return suggestion + + assert get_suggestion().text == "_c" + + # should go up + up(event) + assert get_suggestion().text == "_b" + + # should skip over 'very' which is identical to buffer content + up(event) + assert get_suggestion().text == "_a" + + # should cycle back to beginning + up(event) + assert get_suggestion().text == "_c" + + # should cycle back through end boundary + down(event) + assert get_suggestion().text == "_a" + + down(event) + assert get_suggestion().text == "_b" + + down(event) + assert get_suggestion().text == "_c" + + down(event) + assert get_suggestion().text == "_a" + + +async def test_navigable_provider_multiline_entries(): + provider = NavigableAutoSuggestFromHistory() + history = InMemoryHistory(history_strings=["very_a\nvery_b", "very_c"]) + buffer = Buffer(history=history) + + async for _ in history.load(): + pass + + buffer.cursor_position = 5 + buffer.text = "very" + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + + event = Mock() + event.current_buffer = buffer + + def get_suggestion(): + suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + return suggestion + + assert get_suggestion().text == "_c" + + up(event) + assert get_suggestion().text == "_b" + + up(event) + assert get_suggestion().text == "_a" + + down(event) + assert get_suggestion().text == "_b" + + down(event) + assert get_suggestion().text == "_c" + + +def create_session_mock(): + session = Mock() + session.default_buffer = Buffer() + return session + + +def test_navigable_provider_connection(): + provider = NavigableAutoSuggestFromHistory() + provider.skip_lines = 1 + + session_1 = create_session_mock() + provider.connect(session_1) + + assert provider.skip_lines == 1 + session_1.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 0 + + session_2 = create_session_mock() + provider.connect(session_2) + provider.skip_lines = 2 + + assert provider.skip_lines == 2 + session_2.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 0 + + provider.skip_lines = 3 + provider.disconnect() + session_1.default_buffer.on_text_insert.fire() + session_2.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 3 diff --git a/IPython/tests/test_shortcuts.py b/IPython/tests/test_shortcuts.py deleted file mode 100644 index 42edb92ba58..00000000000 --- a/IPython/tests/test_shortcuts.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest -from IPython.terminal.shortcuts import _apply_autosuggest - -from unittest.mock import Mock - - -def make_event(text, cursor, suggestion): - event = Mock() - event.current_buffer = Mock() - event.current_buffer.suggestion = Mock() - event.current_buffer.cursor_position = cursor - event.current_buffer.suggestion.text = suggestion - event.current_buffer.document = Mock() - event.current_buffer.document.get_end_of_line_position = Mock(return_value=0) - event.current_buffer.document.text = text - event.current_buffer.document.cursor_position = cursor - return event - - -@pytest.mark.parametrize( - "text, cursor, suggestion, called", - [ - ("123456", 6, "123456789", True), - ("123456", 3, "123456789", False), - ("123456 \n789", 6, "123456789", True), - ], -) -def test_autosuggest_at_EOL(text, cursor, suggestion, called): - """ - test that autosuggest is only applied at end of line. - """ - - event = make_event(text, cursor, suggestion) - event.current_buffer.insert_text = Mock() - _apply_autosuggest(event) - if called: - event.current_buffer.insert_text.assert_called() - else: - event.current_buffer.insert_text.assert_not_called() - # event.current_buffer.document.get_end_of_line_position.assert_called() diff --git a/IPython/utils/io.py b/IPython/utils/io.py index 2eac5e66cba..cef4319f92c 100644 --- a/IPython/utils/io.py +++ b/IPython/utils/io.py @@ -18,11 +18,6 @@ from IPython.utils.decorators import undoc from .capture import CapturedIO, capture_output -# setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr -devnull = open(os.devnull, "w", encoding="utf-8") -atexit.register(devnull.close) - - class Tee(object): """A class to duplicate an output stream to stdout/err. diff --git a/IPython/utils/terminal.py b/IPython/utils/terminal.py index 161a9ae6042..b09cfe0d22d 100644 --- a/IPython/utils/terminal.py +++ b/IPython/utils/terminal.py @@ -91,30 +91,14 @@ def _restore_term_title_xterm(): _set_term_title = _set_term_title_xterm _restore_term_title = _restore_term_title_xterm elif sys.platform == 'win32': - try: - import ctypes - - SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW - SetConsoleTitleW.argtypes = [ctypes.c_wchar_p] - - def _set_term_title(title): - """Set terminal title using ctypes to access the Win32 APIs.""" - SetConsoleTitleW(title) - except ImportError: - def _set_term_title(title): - """Set terminal title using the 'title' command.""" - global ignore_termtitle - - try: - # Cannot be on network share when issuing system commands - curr = os.getcwd() - os.chdir("C:") - ret = os.system("title " + title) - finally: - os.chdir(curr) - if ret: - # non-zero return code signals error, don't try again - ignore_termtitle = True + import ctypes + + SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW + SetConsoleTitleW.argtypes = [ctypes.c_wchar_p] + + def _set_term_title(title): + """Set terminal title using ctypes to access the Win32 APIs.""" + SetConsoleTitleW(title) def set_term_title(title): diff --git a/MANIFEST.in b/MANIFEST.in index c70c57d346f..970adeef334 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include LICENSE include setupbase.py include MANIFEST.in include pytest.ini +include py.typed include mypy.ini include .mailmap include .flake8 diff --git a/README.rst b/README.rst index 0371848061e..b004792e0e9 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :target: https://pypi.python.org/pypi/ipython .. image:: https://github.com/ipython/ipython/actions/workflows/test.yml/badge.svg - :target: https://github.com/ipython/ipython/actions/workflows/test.yml) + :target: https://github.com/ipython/ipython/actions/workflows/test.yml .. image:: https://www.codetriage.com/ipython/ipython/badges/users.svg :target: https://www.codetriage.com/ipython/ipython/ diff --git a/docs/autogen_shortcuts.py b/docs/autogen_shortcuts.py index db7fe8d4917..f8fd17bb51f 100755 --- a/docs/autogen_shortcuts.py +++ b/docs/autogen_shortcuts.py @@ -1,45 +1,98 @@ +from dataclasses import dataclass +from inspect import getsource from pathlib import Path +from typing import cast, Callable, List, Union +from html import escape as html_escape +import re + +from prompt_toolkit.keys import KEY_ALIASES +from prompt_toolkit.key_binding import KeyBindingsBase +from prompt_toolkit.filters import Filter, Condition +from prompt_toolkit.shortcuts import PromptSession from IPython.terminal.shortcuts import create_ipython_shortcuts -def name(c): - s = c.__class__.__name__ - if s == '_Invert': - return '(Not: %s)' % name(c.filter) - if s in log_filters.keys(): - return '(%s: %s)' % (log_filters[s], ', '.join(name(x) for x in c.filters)) - return log_filters[s] if s in log_filters.keys() else s +@dataclass +class Shortcut: + #: a sequence of keys (each element on the list corresponds to pressing one or more keys) + keys_sequence: list[str] + filter: str -def sentencize(s): - """Extract first sentence - """ - s = s.replace('\n', ' ').strip().split('.') - s = s[0] if len(s) else s - try: - return " ".join(s.split()) - except AttributeError: - return s +@dataclass +class Handler: + description: str + identifier: str -def most_common(lst, n=3): - """Most common elements occurring more then `n` times - """ - from collections import Counter - c = Counter(lst) - return [k for (k, v) in c.items() if k and v > n] +@dataclass +class Binding: + handler: Handler + shortcut: Shortcut -def multi_filter_str(flt): - """Yield readable conditional filter - """ - assert hasattr(flt, 'filters'), 'Conditional filter required' - yield name(flt) +class _NestedFilter(Filter): + """Protocol reflecting non-public prompt_toolkit's `_AndList` and `_OrList`.""" + + filters: List[Filter] + + +class _Invert(Filter): + """Protocol reflecting non-public prompt_toolkit's `_Invert`.""" + + filter: Filter + + +conjunctions_labels = {"_AndList": "and", "_OrList": "or"} +ATOMIC_CLASSES = {"Never", "Always", "Condition"} + + +def format_filter( + filter_: Union[Filter, _NestedFilter, Condition, _Invert], + is_top_level=True, + skip=None, +) -> str: + """Create easily readable description of the filter.""" + s = filter_.__class__.__name__ + if s == "Condition": + func = cast(Condition, filter_).func + name = func.__name__ + if name == "": + source = getsource(func) + return source.split("=")[0].strip() + return func.__name__ + elif s == "_Invert": + operand = cast(_Invert, filter_).filter + if operand.__class__.__name__ in ATOMIC_CLASSES: + return f"not {format_filter(operand, is_top_level=False)}" + return f"not ({format_filter(operand, is_top_level=False)})" + elif s in conjunctions_labels: + filters = cast(_NestedFilter, filter_).filters + conjunction = conjunctions_labels[s] + glue = f" {conjunction} " + result = glue.join(format_filter(x, is_top_level=False) for x in filters) + if len(filters) > 1 and not is_top_level: + result = f"({result})" + return result + elif s in ["Never", "Always"]: + return s.lower() + else: + raise ValueError(f"Unknown filter type: {filter_}") + + +def sentencize(s) -> str: + """Extract first sentence""" + s = re.split(r"\.\W", s.replace("\n", " ").strip()) + s = s[0] if len(s) else "" + if not s.endswith("."): + s += "." + try: + return " ".join(s.split()) + except AttributeError: + return s -log_filters = {'_AndList': 'And', '_OrList': 'Or'} -log_invert = {'_Invert'} class _DummyTerminal: """Used as a buffer to get prompt_toolkit bindings @@ -48,49 +101,121 @@ class _DummyTerminal: input_transformer_manager = None display_completions = None editing_mode = "emacs" + auto_suggest = None -ipy_bindings = create_ipython_shortcuts(_DummyTerminal()).bindings - -dummy_docs = [] # ignore bindings without proper documentation - -common_docs = most_common([kb.handler.__doc__ for kb in ipy_bindings]) -if common_docs: - dummy_docs.extend(common_docs) +def create_identifier(handler: Callable): + parts = handler.__module__.split(".") + name = handler.__name__ + package = parts[0] + if len(parts) > 1: + final_module = parts[-1] + return f"{package}:{final_module}.{name}" + else: + return f"{package}:{name}" + + +def bindings_from_prompt_toolkit(prompt_bindings: KeyBindingsBase) -> List[Binding]: + """Collect bindings to a simple format that does not depend on prompt-toolkit internals""" + bindings: List[Binding] = [] + + for kb in prompt_bindings.bindings: + bindings.append( + Binding( + handler=Handler( + description=kb.handler.__doc__ or "", + identifier=create_identifier(kb.handler), + ), + shortcut=Shortcut( + keys_sequence=[ + str(k.value) if hasattr(k, "value") else k for k in kb.keys + ], + filter=format_filter(kb.filter, skip={"has_focus_filter"}), + ), + ) + ) + return bindings + + +INDISTINGUISHABLE_KEYS = {**KEY_ALIASES, **{v: k for k, v in KEY_ALIASES.items()}} + + +def format_prompt_keys(keys: str, add_alternatives=True) -> str: + """Format prompt toolkit key with modifier into an RST representation.""" + + def to_rst(key): + escaped = key.replace("\\", "\\\\") + return f":kbd:`{escaped}`" + + keys_to_press: list[str] + + prefixes = { + "c-s-": [to_rst("ctrl"), to_rst("shift")], + "s-c-": [to_rst("ctrl"), to_rst("shift")], + "c-": [to_rst("ctrl")], + "s-": [to_rst("shift")], + } + + for prefix, modifiers in prefixes.items(): + if keys.startswith(prefix): + remainder = keys[len(prefix) :] + keys_to_press = [*modifiers, to_rst(remainder)] + break + else: + keys_to_press = [to_rst(keys)] -dummy_docs = list(set(dummy_docs)) + result = " + ".join(keys_to_press) -single_filter = {} -multi_filter = {} -for kb in ipy_bindings: - doc = kb.handler.__doc__ - if not doc or doc in dummy_docs: - continue + if keys in INDISTINGUISHABLE_KEYS and add_alternatives: + alternative = INDISTINGUISHABLE_KEYS[keys] - shortcut = ' '.join([k if isinstance(k, str) else k.name for k in kb.keys]) - shortcut += shortcut.endswith('\\') and '\\' or '' - if hasattr(kb.filter, 'filters'): - flt = ' '.join(multi_filter_str(kb.filter)) - multi_filter[(shortcut, flt)] = sentencize(doc) - else: - single_filter[(shortcut, name(kb.filter))] = sentencize(doc) + result = ( + result + + " (or " + + format_prompt_keys(alternative, add_alternatives=False) + + ")" + ) + return result if __name__ == '__main__': here = Path(__file__).parent dest = here / "source" / "config" / "shortcuts" - def sort_key(item): - k, v = item - shortcut, flt = k - return (str(shortcut), str(flt)) - - for filters, output_filename in [ - (single_filter, "single_filtered"), - (multi_filter, "multi_filtered"), - ]: - with (dest / "{}.csv".format(output_filename)).open( - "w", encoding="utf-8" - ) as csv: - for (shortcut, flt), v in sorted(filters.items(), key=sort_key): - csv.write(":kbd:`{}`\t{}\t{}\n".format(shortcut, flt, v)) + ipy_bindings = create_ipython_shortcuts(_DummyTerminal(), for_all_platforms=True) + + session = PromptSession(key_bindings=ipy_bindings) + prompt_bindings = session.app.key_bindings + + assert prompt_bindings + # Ensure that we collected the default shortcuts + assert len(prompt_bindings.bindings) > len(ipy_bindings.bindings) + + bindings = bindings_from_prompt_toolkit(prompt_bindings) + + def sort_key(binding: Binding): + return binding.handler.identifier, binding.shortcut.filter + + filters = [] + with (dest / "table.tsv").open("w", encoding="utf-8") as csv: + for binding in sorted(bindings, key=sort_key): + sequence = ", ".join( + [format_prompt_keys(keys) for keys in binding.shortcut.keys_sequence] + ) + if binding.shortcut.filter == "always": + condition_label = "-" + else: + # we cannot fit all the columns as the filters got too complex over time + condition_label = "ⓘ" + + csv.write( + "\t".join( + [ + sequence, + sentencize(binding.handler.description) + + f" :raw-html:`
` `{binding.handler.identifier}`", + f':raw-html:`{condition_label}`', + ] + ) + + "\n" + ) diff --git a/docs/environment.yml b/docs/environment.yml index afb5eff0051..9961253138a 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -3,14 +3,14 @@ channels: - conda-forge - defaults dependencies: - - python=3.8 - - setuptools>=18.5 + - python=3.10 + - setuptools - sphinx>=4.2 - - sphinx_rtd_theme>=1.0 + - sphinx_rtd_theme - numpy - - nose - testpath - matplotlib + - pip - pip: - docrepr - prompt_toolkit diff --git a/docs/source/_images/autosuggest.gif b/docs/source/_images/autosuggest.gif new file mode 100644 index 00000000000..ee105489432 Binary files /dev/null and b/docs/source/_images/autosuggest.gif differ diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000000..156db8c24b0 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,7 @@ +/* + Needed to revert problematic lack of wrapping in sphinx_rtd_theme, see: + https://github.com/readthedocs/sphinx_rtd_theme/issues/117 +*/ +.wy-table-responsive table.shortcuts td, .wy-table-responsive table.shortcuts th { + white-space: normal!important; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index d04d4637ba7..325b3ed008f 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -183,8 +183,7 @@ def filter(self, record): # Exclude these glob-style patterns when looking for source files. They are # relative to the source/ directory. -exclude_patterns = [] - +exclude_patterns = ["**.ipynb_checkpoints"] # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True @@ -211,7 +210,6 @@ def filter(self, record): # given in html_static_path. # html_style = 'default.css' - # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None @@ -327,6 +325,10 @@ def filter(self, record): modindex_common_prefix = ['IPython.'] +def setup(app): + app.add_css_file("theme_overrides.css") + + # Cleanup # ------- # delete release info to avoid pickling errors from sphinx diff --git a/docs/source/config/custommagics.rst b/docs/source/config/custommagics.rst index 99d4068773c..0a37b858a4c 100644 --- a/docs/source/config/custommagics.rst +++ b/docs/source/config/custommagics.rst @@ -139,13 +139,26 @@ Accessing user namespace and local scope ======================================== When creating line magics, you may need to access surrounding scope to get user -variables (e.g when called inside functions). IPython provide the +variables (e.g when called inside functions). IPython provides the ``@needs_local_scope`` decorator that can be imported from ``IPython.core.magics``. When decorated with ``@needs_local_scope`` a magic will be passed ``local_ns`` as an argument. As a convenience ``@needs_local_scope`` can also be applied to cell magics even if cell magics cannot appear at local scope context. +Silencing the magic output +========================== + +Sometimes it may be useful to define a magic that can be silenced the same way +that non-magic expressions can, i.e., by appending a semicolon at the end of the Python +code to be executed. That can be achieved by decorating the magic function with +the decorator ``@output_can_be_silenced`` that can be imported from +``IPython.core.magics``. When this decorator is used, IPython will parse the Python +code used by the magic and, if the last token is a ``;``, the output created by the +magic will not show up on the screen. If you want to see an example of this decorator +in action, take a look on the ``time`` magic defined in +``IPython.core.magics.execution.py``. + Complete Example ================ diff --git a/docs/source/config/integrating.rst b/docs/source/config/integrating.rst index 07429ef1792..23cc1e58875 100644 --- a/docs/source/config/integrating.rst +++ b/docs/source/config/integrating.rst @@ -128,7 +128,6 @@ More powerful methods Displays the object as a side effect; the return value is ignored. If this is defined, all other display methods are ignored. - This method is ignored in the REPL. Metadata diff --git a/docs/source/config/shortcuts/index.rst b/docs/source/config/shortcuts/index.rst index 4103d92a7bd..e361ec26c5d 100755 --- a/docs/source/config/shortcuts/index.rst +++ b/docs/source/config/shortcuts/index.rst @@ -4,28 +4,23 @@ IPython shortcuts Available shortcuts in an IPython terminal. -.. warning:: +.. note:: - This list is automatically generated, and may not hold all available - shortcuts. In particular, it may depend on the version of ``prompt_toolkit`` - installed during the generation of this page. + This list is automatically generated. Key bindings defined in ``prompt_toolkit`` may differ + between installations depending on the ``prompt_toolkit`` version. -Single Filtered shortcuts -========================= - -.. csv-table:: - :header: Shortcut,Filter,Description - :widths: 30, 30, 100 - :delim: tab - :file: single_filtered.csv +* Comma-separated keys, e.g. :kbd:`Esc`, :kbd:`f`, indicate a sequence which can be activated by pressing the listed keys in succession. +* Plus-separated keys, e.g. :kbd:`Esc` + :kbd:`f` indicate a combination which requires pressing all keys simultaneously. +* Hover over the ⓘ icon in the filter column to see when the shortcut is active.g +.. role:: raw-html(raw) + :format: html -Multi Filtered shortcuts -======================== .. csv-table:: - :header: Shortcut,Filter,Description - :widths: 30, 30, 100 + :header: Shortcut,Description and identifier,Filter :delim: tab - :file: multi_filtered.csv + :class: shortcuts + :file: table.tsv + :widths: 20 75 5 diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index eee7af0aa48..7fecb799d15 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -2,6 +2,155 @@ 8.x Series ============ + +.. _version 8.10.0: + +IPython 8.10 +------------ + +Out of schedule release of IPython with minor fixes to patch a potential CVE-2023-24816. +This is a really low severity CVE that you most likely are not affected by unless: + + - You are on windows. + - You have a custom build of Python without ``_ctypes`` + - You cd or start IPython or Jupyter in untrusted directory which names may be + valid shell commands. + +You can read more on `the advisory +`__. + +In addition to fixing this CVE we also fix a couple of outstanding bugs and issues. + +As usual you can find the full list of PRs on GitHub under `the 8.10 milestone +`__. + +In Particular: + + - bump minimum numpy to `>=1.21` version following NEP29. :ghpull:`13930` + - fix for compatibility with MyPy 1.0. :ghpull:`13933` + - fix nbgrader stalling when IPython's ``showtraceback`` function is + monkeypatched. :ghpull:`13934` + + + +As this release also contains those minimal changes in addition to fixing the +CVE I decided to bump the minor version anyway. + +This will not affect the normal release schedule, so IPython 8.11 is due in +about 2 weeks. + +.. _version 8.9.0: + +IPython 8.9.0 +------------- + +Second release of IPython in 2023, last Friday of the month, we are back on +track. This is a small release with a few bug-fixes, and improvements, mostly +with respect to terminal shortcuts. + + +The biggest improvement for 8.9 is a drastic amelioration of the +auto-suggestions sponsored by D.E. Shaw and implemented by the more and more +active contributor `@krassowski `. + +- ``right`` accepts a single character from suggestion +- ``ctrl+right`` accepts a semantic token (macos default shortcuts take + precedence and need to be disabled to make this work) +- ``backspace`` deletes a character and resumes hinting autosuggestions +- ``ctrl-left`` accepts suggestion and moves cursor left one character. +- ``backspace`` deletes a character and resumes hinting autosuggestions +- ``down`` moves to suggestion to later in history when no lines are present below the cursors. +- ``up`` moves to suggestion from earlier in history when no lines are present above the cursor. + +This is best described by the Gif posted by `@krassowski +`, and in the PR itself :ghpull:`13888`. + +.. image:: ../_images/autosuggest.gif + +Please report any feedback in order for us to improve the user experience. +In particular we are also working on making the shortcuts configurable. + +If you are interested in better terminal shortcuts, I also invite you to +participate in issue `13879 +`__. + + +As we follow `NEP29 +`__, next version of +IPython will officially stop supporting numpy 1.20, and will stop supporting +Python 3.8 after April release. + +As usual you can find the full list of PRs on GitHub under `the 8.9 milestone +`__. + + +Thanks to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. + +.. _version 8.8.0: + +IPython 8.8.0 +------------- + +First release of IPython in 2023 as there was no release at the end of +December. + +This is an unusually big release (relatively speaking) with more than 15 Pull +Requests merged. + +Of particular interest are: + + - :ghpull:`13852` that replaces the greedy completer and improves + completion, in particular for dictionary keys. + - :ghpull:`13858` that adds ``py.typed`` to ``setup.cfg`` to make sure it is + bundled in wheels. + - :ghpull:`13869` that implements tab completions for IPython options in the + shell when using `argcomplete `. I + believe this also needs a recent version of Traitlets. + - :ghpull:`13865` makes the ``inspector`` class of `InteractiveShell` + configurable. + - :ghpull:`13880` that removes minor-version entrypoints as the minor version + entry points that would be included in the wheel would be the one of the + Python version that was used to build the ``whl`` file. + +In no particular order, the rest of the changes update the test suite to be +compatible with Pygments 2.14, various docfixes, testing on more recent python +versions and various updates. + +As usual you can find the full list of PRs on GitHub under `the 8.8 milestone +`__. + +Many thanks to @krassowski for the many PRs and @jasongrout for reviewing and +merging contributions. + +Thanks to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. + +.. _version 8.7.0: + +IPython 8.7.0 +------------- + + +Small release of IPython with a couple of bug fixes and new features for this +month. Next month is the end of year, it is unclear if there will be a release +close to the new year's eve, or if the next release will be at the end of January. + +Here are a few of the relevant fixes, +as usual you can find the full list of PRs on GitHub under `the 8.7 milestone +`__. + + + - :ghpull:`13834` bump the minimum prompt toolkit to 3.0.11. + - IPython shipped with the ``py.typed`` marker now, and we are progressively + adding more types. :ghpull:`13831` + - :ghpull:`13817` add configuration of code blacks formatting. + + +Thanks to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. + + .. _version 8.6.0: IPython 8.6.0 @@ -9,29 +158,29 @@ IPython 8.6.0 Back to a more regular release schedule (at least I try), as Friday is already over by more than 24h hours. This is a slightly bigger release with a -few new features that contain no less then 25 PRs. +few new features that contain no less than 25 PRs. We'll notably found a couple of non negligible changes: The ``install_ext`` and related functions have been removed after being deprecated for years. You can use pip to install extensions. ``pip`` did not -exists when ``install_ext`` was introduced. You can still load local extensions +exist when ``install_ext`` was introduced. You can still load local extensions without installing them. Just set your ``sys.path`` for example. :ghpull:`13744` -IPython now have extra entry points that that the major *and minor* version of -python. For some of you this mean that you can do a quick ``ipython3.10`` to +IPython now has extra entry points that use the major *and minor* version of +python. For some of you this means that you can do a quick ``ipython3.10`` to launch IPython from the Python 3.10 interpreter, while still using Python 3.11 as your main Python. :ghpull:`13743` -The completer matcher API have been improved. See :ghpull:`13745`. This should +The completer matcher API has been improved. See :ghpull:`13745`. This should improve the type inference and improve dict keys completions in many use case. -Tanks ``@krassowski`` for all the works, and the D.E. Shaw group for sponsoring +Thanks ``@krassowski`` for all the work, and the D.E. Shaw group for sponsoring it. The color of error nodes in tracebacks can now be customized. See -:ghpull:`13756`. This is a private attribute until someone find the time to -properly add a configuration option. Note that with Python 3.11 that also show -the relevant nodes in traceback, it would be good to leverage this informations +:ghpull:`13756`. This is a private attribute until someone finds the time to +properly add a configuration option. Note that with Python 3.11 that also shows +the relevant nodes in traceback, it would be good to leverage this information (plus the "did you mean" info added on attribute errors). But that's likely work I won't have time to do before long, so contributions welcome. @@ -40,11 +189,11 @@ As we follow NEP 29, we removed support for numpy 1.19 :ghpull:`13760`. The ``open()`` function present in the user namespace by default will now refuse to open the file descriptors 0,1,2 (stdin, out, err), to avoid crashing IPython. -This mostly occurs in teaching context when incorrect values get passed around. +This mostly occurs in teaching context when incorrect values get passed around. The ``?``, ``??``, and corresponding ``pinfo``, ``pinfo2`` magics can now find -objects insides arrays. That is to say, the following now works:: +objects inside arrays. That is to say, the following now works:: >>> def my_func(*arg, **kwargs):pass @@ -53,7 +202,7 @@ objects insides arrays. That is to say, the following now works:: If ``container`` define a custom ``getitem``, this __will__ trigger the custom -method. So don't put side effects in your ``getitems``. Thanks the D.E. Shaw +method. So don't put side effects in your ``getitems``. Thanks to the D.E. Shaw group for the request and sponsoring the work. @@ -79,17 +228,17 @@ an bug fixes. Many thanks to everybody who contributed PRs for your patience in review and merges. -Here is a non exhaustive list of changes that have been implemented for IPython +Here is a non-exhaustive list of changes that have been implemented for IPython 8.5.0. As usual you can find the full list of issues and PRs tagged with `the 8.5 milestone `__. - - Added shortcut for accepting auto suggestion. The End key shortcut for + - Added a shortcut for accepting auto suggestion. The End key shortcut for accepting auto-suggestion This binding works in Vi mode too, provided ``TerminalInteractiveShell.emacs_bindings_in_vi_insert_mode`` is set to be ``True`` :ghpull:`13566`. - - No popup in window for latex generation w hen generating latex (e.g. via + - No popup in window for latex generation when generating latex (e.g. via `_latex_repr_`) no popup window is shows under Windows. :ghpull:`13679` - Fixed error raised when attempting to tab-complete an input string with @@ -112,7 +261,7 @@ Here is a non exhaustive list of changes that have been implemented for IPython - Fix paste magic on wayland. :ghpull:`13671` - show maxlen in deque's repr. :ghpull:`13648` -Restore line numbers for Input +Restore line numbers for Input ------------------------------ Line number information in tracebacks from input are restored. @@ -205,12 +354,12 @@ IPython 8.3.0 - :ghpull:`13600`, ``pre_run_*``-hooks will now have a ``cell_id`` attribute on - the info object when frontend provide it. This has been backported to 7.33 + the info object when frontend provides it. This has been backported to 7.33 - :ghpull:`13624`, fixed :kbd:`End` key being broken after accepting an auto-suggestion. - - :ghpull:`13657` fix issue where history from different sessions would be mixed. + - :ghpull:`13657` fixed an issue where history from different sessions would be mixed. .. _version 8.2.0: @@ -228,8 +377,8 @@ IPython 8.2 mostly bring bugfixes to IPython. - Fixes to ``ultratb`` ipdb support when used outside of IPython. :ghpull:`13498` -I am still trying to fix and investigate :ghissue:`13598`, which seem to be -random, and would appreciate help if you find reproducible minimal case. I've +I am still trying to fix and investigate :ghissue:`13598`, which seems to be +random, and would appreciate help if you find a reproducible minimal case. I've tried to make various changes to the codebase to mitigate it, but a proper fix will be difficult without understanding the cause. @@ -243,7 +392,7 @@ Thanks to the `D. E. Shaw group `__ for sponsoring work on IPython and related libraries. .. _version 8.1.1: - + IPython 8.1.1 ------------- @@ -258,7 +407,7 @@ IPython 8.1.0 ------------- IPython 8.1 is the first minor release after 8.0 and fixes a number of bugs and -Update a few behavior that were problematic with the 8.0 as with many new major +updates a few behaviors that were problematic with the 8.0 as with many new major release. Note that beyond the changes listed here, IPython 8.1.0 also contains all the @@ -309,8 +458,8 @@ We want to remind users that IPython is part of the Jupyter organisations, and thus governed by a Code of Conduct. Some of the behavior we have seen on GitHub is not acceptable. Abuse and non-respectful comments on discussion will not be tolerated. -Many thanks to all the contributors to this release, many of the above fixed issue and -new features where done by first time contributors, showing there is still +Many thanks to all the contributors to this release, many of the above fixed issues and +new features were done by first time contributors, showing there is still plenty of easy contribution possible in IPython . You can find all individual contributions to this milestone `on github `__. @@ -371,16 +520,16 @@ IPython 8.0 IPython 8.0 is bringing a large number of new features and improvements to both the user of the terminal and of the kernel via Jupyter. The removal of compatibility -with older version of Python is also the opportunity to do a couple of +with an older version of Python is also the opportunity to do a couple of performance improvements in particular with respect to startup time. The 8.x branch started diverging from its predecessor around IPython 7.12 (January 2020). This release contains 250+ pull requests, in addition to many of the features -and backports that have made it to the 7.x branch. Please see the +and backports that have made it to the 7.x branch. Please see the `8.0 milestone `__ for the full list of pull requests. -Please feel free to send pull requests to updates those notes after release, +Please feel free to send pull requests to update those notes after release, I have likely forgotten a few things reviewing 250+ PRs. Dependencies changes/downstream packaging @@ -395,8 +544,8 @@ looking for help to do so. - minimal Python is now 3.8 - ``nose`` is not a testing requirement anymore - ``pytest`` replaces nose. - - ``iptest``/``iptest3`` cli entrypoints do not exists anymore. - - minimum officially support ``numpy`` version has been bumped, but this should + - ``iptest``/``iptest3`` cli entrypoints do not exist anymore. + - the minimum officially ​supported ``numpy`` version has been bumped, but this should not have much effect on packaging. @@ -417,7 +566,7 @@ deprecation warning: - Please add **since which version** something is deprecated. As a side note, it is much easier to conditionally compare version -numbers rather than using ``try/except`` when functionality changes with a version. +numbers rather than using ``try/except`` when functionality changes with a version. I won't list all the removed features here, but modules like ``IPython.kernel``, which was just a shim module around ``ipykernel`` for the past 8 years, have been @@ -449,7 +598,7 @@ by mypy. Featured changes ---------------- -Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. +Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. Please note as well that many features have been added in the 7.x branch as well (and hence why you want to read the 7.x what's new notes), in particular features contributed by QuantStack (with respect to debugger protocol and Xeus @@ -497,7 +646,7 @@ The error traceback is now correctly formatted, showing the cell number in which ZeroDivisionError: division by zero -The ``stack_data`` package has been integrated, which provides smarter information in the traceback; +The ``stack_data`` package has been integrated, which provides smarter information in the traceback; in particular it will highlight the AST node where an error occurs which can help to quickly narrow down errors. For example in the following snippet:: @@ -537,7 +686,7 @@ and IPython 8.0 is capable of telling you where the index error occurs:: ----> 3 return x[0][i][0] ^^^^^^^ -The corresponding locations marked here with ``^`` will show up highlighted in +The corresponding locations marked here with ``^`` will show up highlighted in the terminal and notebooks. Finally, a colon ``::`` and line number is appended after a filename in @@ -734,7 +883,7 @@ Previously, this was not the case for the Vi-mode prompts:: This is now fixed, and Vi prompt prefixes - ``[ins]`` and ``[nav]`` - are skipped just as the normal ``In`` would be. -IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``, +IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``, You should be able to change mode dynamically with ``%config TerminalInteractiveShell.editing_mode='vi'`` Empty History Ranges @@ -761,8 +910,8 @@ when followed with :kbd:`F2`), send it to `dpaste.org `_ using Windows timing implementation: Switch to process_time ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Timing on Windows, for example with ``%%time``, was changed from being based on ``time.perf_counter`` -(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead +Timing on Windows, for example with ``%%time``, was changed from being based on ``time.perf_counter`` +(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead (which only counts CPU time). This brings it closer to the behavior on Linux. See :ghpull:`12984`. Miscellaneous @@ -787,7 +936,7 @@ Re-added support for XDG config directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ XDG support through the years comes and goes. There is a tension between having -an identical location for configuration in all platforms versus having simple instructions. +an identical location for configuration in all platforms versus having simple instructions. After initial failures a couple of years ago, IPython was modified to automatically migrate XDG config files back into ``~/.ipython``. That migration code has now been removed. IPython now checks the XDG locations, so if you _manually_ move your config @@ -815,7 +964,7 @@ Removing support for older Python versions We are removing support for Python up through 3.7, allowing internal code to use the more -efficient ``pathlib`` and to make better use of type annotations. +efficient ``pathlib`` and to make better use of type annotations. .. image:: ../_images/8.0/pathlib_pathlib_everywhere.jpg :alt: "Meme image of Toy Story with Woody and Buzz, with the text 'pathlib, pathlib everywhere'" diff --git a/pytest.ini b/pytest.ini index 81511e9ce51..5cc977692b8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,18 +14,10 @@ addopts = --durations=10 --ignore=IPython/sphinxext --ignore=IPython/terminal/pt_inputhooks --ignore=IPython/__main__.py - --ignore=IPython/config.py - --ignore=IPython/frontend.py - --ignore=IPython/html.py - --ignore=IPython/nbconvert.py - --ignore=IPython/nbformat.py - --ignore=IPython/parallel.py - --ignore=IPython/qt.py --ignore=IPython/external/qt_for_kernel.py --ignore=IPython/html/widgets/widget_link.py --ignore=IPython/html/widgets/widget_output.py --ignore=IPython/terminal/console.py - --ignore=IPython/terminal/ptshell.py --ignore=IPython/utils/_process_cli.py --ignore=IPython/utils/_process_posix.py --ignore=IPython/utils/_process_win32.py diff --git a/setup.cfg b/setup.cfg index b3a26586cf3..3b796958a8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = matplotlib-inline pexpect>4.3; sys_platform != "win32" pickleshare - prompt_toolkit>3.0.1,<3.1.0 + prompt_toolkit>=3.0.30,<3.1.0 pygments>=2.4.0 stack_data traitlets>=5 @@ -79,7 +79,7 @@ test_extra = curio matplotlib!=3.2.0 nbformat - numpy>=1.20 + numpy>=1.21 pandas trio all = @@ -100,20 +100,12 @@ exclude = setupext [options.package_data] +IPython = py.typed IPython.core = profile/README* IPython.core.tests = *.png, *.jpg, daft_extension/*.py IPython.lib.tests = *.wav IPython.testing.plugin = *.txt -[options.entry_points] -console_scripts = - ipython = IPython:start_ipython - ipython3 = IPython:start_ipython -pygments.lexers = - ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer - ipython = IPython.lib.lexers:IPythonLexer - ipython3 = IPython.lib.lexers:IPython3Lexer - [velin] ignore_patterns = IPython/core/tests diff --git a/setup.py b/setup.py index bfdf5fb88bf..3f7cd6da664 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ # Our own imports sys.path.insert(0, ".") -from setupbase import target_update +from setupbase import target_update, find_entry_points from setupbase import ( setup_args, @@ -140,6 +140,15 @@ 'unsymlink': unsymlink, } +setup_args["entry_points"] = { + "console_scripts": find_entry_points(), + "pygments.lexers": [ + "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer", + "ipython = IPython.lib.lexers:IPythonLexer", + "ipython3 = IPython.lib.lexers:IPython3Lexer", + ], +} + #--------------------------------------------------------------------------- # Do the actual setup now #--------------------------------------------------------------------------- diff --git a/setupbase.py b/setupbase.py index 748b4dd6a8a..a867c73ecd0 100644 --- a/setupbase.py +++ b/setupbase.py @@ -211,20 +211,15 @@ def find_entry_points(): use, our own build_scripts_entrypt class below parses these and builds command line scripts. - Each of our entry points gets a plain name, e.g. ipython, a name - suffixed with the Python major version number, e.g. ipython3, and - a name suffixed with the Python major.minor version number, eg. ipython3.8. + Each of our entry points gets a plain name, e.g. ipython, and a name + suffixed with the Python major version number, e.g. ipython3. """ ep = [ 'ipython%s = IPython:start_ipython', ] major_suffix = str(sys.version_info[0]) - minor_suffix = ".".join([str(sys.version_info[0]), str(sys.version_info[1])]) - return ( - [e % "" for e in ep] - + [e % major_suffix for e in ep] - + [e % minor_suffix for e in ep] - ) + return [e % "" for e in ep] + [e % major_suffix for e in ep] + class install_lib_symlink(Command): user_options = [ diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 697ed859e84..ebf8098195c 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -2,14 +2,6 @@ # when releasing with bash, simple source it to get asked questions. # misc check before starting - -python -c 'import keyring' -python -c 'import twine' -python -c 'import sphinx' -python -c 'import sphinx_rtd_theme' -python -c 'import pytest' - - BLACK=$(tput setaf 1) RED=$(tput setaf 1) GREEN=$(tput setaf 2) @@ -21,6 +13,22 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Checking all tools are installed..." + +python -c 'import keyring' +python -c 'import twine' +python -c 'import sphinx' +python -c 'import sphinx_rtd_theme' +python -c 'import pytest' +python -c 'import build' +# those are necessary fo building the docs +echo "Checking imports for docs" +python -c 'import numpy' +python -c 'import matplotlib' + + + + echo "Will use $BLUE'$EDITOR'$NOR to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input