From 1c5ecd6fc3c010595a324ef2fa2a2e23f9566503 Mon Sep 17 00:00:00 2001 From: Petri Salminen Date: Wed, 28 Sep 2022 19:47:12 +0300 Subject: [PATCH 001/122] Allow spaces in file paths when using sphinx directive Closes: #11606 --- IPython/sphinxext/ipython_directive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 9e3c7b2276a..e55ba126c8a 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -981,7 +981,7 @@ def setup(self): self.shell.warning_is_error = warning_is_error # setup bookmark for saving figures directory - self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir, + self.shell.process_input_line('bookmark ipy_savedir "%s"'%savefig_dir, store_history=False) self.shell.clear_cout() From 8d2d355e76da23a696d429d9eb2e0a14ba778578 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 30 Oct 2022 10:32:00 +0100 Subject: [PATCH 002/122] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index d8eeae46901..d891c34cc62 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -16,11 +16,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 6 +_version_minor = 7 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -_version_extra = "" # Uncomment this for full releases +# _version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From a146297a8ec7e17bd8633c77bce0c19ebd56160a Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Wed, 9 Nov 2022 06:25:20 -0700 Subject: [PATCH 003/122] Lint formatting --- IPython/sphinxext/ipython_directive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index e55ba126c8a..c428e7917fd 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -981,8 +981,9 @@ def setup(self): self.shell.warning_is_error = warning_is_error # setup bookmark for saving figures directory - self.shell.process_input_line('bookmark ipy_savedir "%s"'%savefig_dir, - store_history=False) + self.shell.process_input_line( + 'bookmark ipy_savedir "%s"' % savefig_dir, store_history=False + ) self.shell.clear_cout() return rgxin, rgxout, promptin, promptout From 724c9d70d5ae1d23d6121ba2e88fa987af1fd8c5 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Wed, 9 Nov 2022 00:36:57 -0700 Subject: [PATCH 004/122] Fix mypy error by being more explicit and verbose --- IPython/core/magics/basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index af69b02676d..7dfa84ce2d5 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -297,7 +297,10 @@ def page(self, parameter_s=''): oname = args and args or '_' info = self.shell._ofind(oname) if info['found']: - txt = (raw and str or pformat)( info['obj'] ) + if raw: + txt = str(info["obj"]) + else: + txt = pformat(info["obj"]) page.page(txt) else: print('Object `%s` not found' % oname) From 03052c48c0a6b868d81f5f3678b218a079959f66 Mon Sep 17 00:00:00 2001 From: Bill Vineyard <71736216+wvineyard@users.noreply.github.com> Date: Sat, 29 Oct 2022 11:21:17 -0500 Subject: [PATCH 005/122] removed duplicate .vscode in gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index f4736530e10..3b6963b6317 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ __pycache__ .cache .coverage *.swp -.vscode .pytest_cache .python-version venv*/ From 1c3678bf1debbc6abd19fe94f4c574b329f3c749 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 8 Nov 2022 17:04:51 -0700 Subject: [PATCH 006/122] Make the formatting of a code block name extendable Currently the user display of a code block requires a tight coupling between the caching compiler and the ultratb file, i.e., ultratb needs to know internal private variables of the caching compiler. This change makes the user-visible display of the code block name the responsibility of the caching compiler. A nice result is that the caching compiler can be overridden to have custom terminology in different systems for code blocks executed. --- IPython/core/compilerop.py | 15 +++++++++++++++ IPython/core/ultratb.py | 26 ++++++++++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index 228f705666f..7799a4fc99e 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -116,6 +116,21 @@ def get_code_name(self, raw_code, transformed_code, number): """ return code_name(transformed_code, number) + def format_code_name(self, name): + """Return a user-friendly label and name for a code block. + + Parameters + ---------- + name : str + The name for the code block returned from get_code_name + + Returns + ------- + A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used. + """ + if name in self._filename_map: + return "Cell", "In[%s]" % self._filename_map[name] + def cache(self, transformed_code, number=0, raw_code=None): """Make a name for a block of code, and cache the code. diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index e83e2b4a0c1..18eff270829 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -173,7 +173,7 @@ def _format_traceback_lines(lines, Colors, has_colors: bool, lvals): def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): """ - Format filename lines with `In [n]` if it's the nth code cell or `File *.py` if it's a module. + Format filename lines with custom formatting from caching compiler or `File *.py` by default Parameters ---------- @@ -184,23 +184,29 @@ def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): ColorScheme's normal coloring to be used. """ ipinst = get_ipython() - - if ipinst is not None and file in ipinst.compile._filename_map: - file = "[%s]" % ipinst.compile._filename_map[file] + if ( + ipinst is not None + and (data := ipinst.compile.format_code_name(file)) is not None + ): + label, name = data if lineno is None: - tpl_link = f"Cell {ColorFilename}In {{file}}{ColorNormal}" + tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}" else: - tpl_link = f"Cell {ColorFilename}In {{file}}, line {{lineno}}{ColorNormal}" + tpl_link = ( + f"{{label}} {ColorFilename}{{name}}, line {{lineno}}{ColorNormal}" + ) else: - file = util_path.compress_user( + label = "File" + name = util_path.compress_user( py3compat.cast_unicode(file, util_path.fs_encoding) ) if lineno is None: - tpl_link = f"File {ColorFilename}{{file}}{ColorNormal}" + tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}" else: - tpl_link = f"File {ColorFilename}{{file}}:{{lineno}}{ColorNormal}" + # can we make this the more friendly ", line {{lineno}}", or do we need to preserve the formatting with the colon? + tpl_link = f"{{label}} {ColorFilename}{{name}}:{{lineno}}{ColorNormal}" - return tpl_link.format(file=file, lineno=lineno) + return tpl_link.format(label=label, name=name, lineno=lineno) #--------------------------------------------------------------------------- # Module classes From cd04814365c74d947b8ffc52bd21ea36a5f4fc8c Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Fri, 11 Nov 2022 20:18:03 +0000 Subject: [PATCH 007/122] Fix `merge_completions=False` error with Jedi Jedi uses filter to return an iterator hence we cannot just check `len()` to see if there are any completions. --- IPython/core/completer.py | 17 +++++++- IPython/core/tests/test_completer.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index fc3aea7b611..25b780be3da 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -671,6 +671,19 @@ def __call__(self, context: CompletionContext) -> MatcherResult: Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] +def has_any_completions(result: MatcherResult) -> bool: + """Check if any result includes any completions.""" + if hasattr(result["completions"], "__len__"): + return len(result["completions"]) != 0 + try: + old_iterator = result["completions"] + first = next(old_iterator) + result["completions"] = itertools.chain([first], old_iterator) + return True + except StopIteration: + return False + + def completion_matcher( *, priority: float = None, identifier: str = None, api_version: int = 1 ): @@ -1952,7 +1965,7 @@ def _jedi_matches( else: return [] - def python_matches(self, text:str)->List[str]: + def python_matches(self, text: str) -> Iterable[str]: """Match attributes or global python names""" if "." in text: try: @@ -2807,7 +2820,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, should_suppress = ( (suppression_config is True) or (suppression_recommended and (suppression_config is not False)) - ) and len(result["completions"]) + ) and has_any_completions(result) if should_suppress: suppression_exceptions = result.get("do_not_suppress", set()) diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index fd72cf7d57a..98ec814a769 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -1396,6 +1396,65 @@ def configure(suppression_config): configure({"b_matcher": True}) _("do not suppress", ["completion_b"]) + configure(True) + _("do not suppress", ["completion_a"]) + + def test_matcher_suppression_with_iterator(self): + @completion_matcher(identifier="matcher_returning_iterator") + def matcher_returning_iterator(text): + return iter(["completion_iter"]) + + @completion_matcher(identifier="matcher_returning_list") + def matcher_returning_list(text): + return ["completion_list"] + + with custom_matchers([matcher_returning_iterator, matcher_returning_list]): + ip = get_ipython() + c = ip.Completer + + def _(text, expected): + c.use_jedi = False + s, matches = c.complete(text) + self.assertEqual(expected, matches) + + def configure(suppression_config): + cfg = Config() + cfg.IPCompleter.suppress_competing_matchers = suppression_config + c.update_config(cfg) + + configure(False) + _("---", ["completion_iter", "completion_list"]) + + configure(True) + _("---", ["completion_iter"]) + + configure(None) + _("--", ["completion_iter", "completion_list"]) + + def test_matcher_suppression_with_jedi(self): + ip = get_ipython() + c = ip.Completer + c.use_jedi = True + + def configure(suppression_config): + cfg = Config() + cfg.IPCompleter.suppress_competing_matchers = suppression_config + c.update_config(cfg) + + def _(): + with provisionalcompleter(): + matches = [completion.text for completion in c.completions("dict.", 5)] + self.assertIn("keys", matches) + + configure(False) + _() + + configure(True) + _() + + configure(None) + _() + def test_matcher_disabling(self): @completion_matcher(identifier="a_matcher") def a_matcher(text): From c5994eee0ff57d26cde7f7e14de0c92ae9520f25 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 12 Nov 2022 22:16:01 +0000 Subject: [PATCH 008/122] Explain expected format for `disable_matchers`/identifiers --- IPython/core/completer.py | 13 ++++++++++--- IPython/core/magics/config.py | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index fc3aea7b611..645f8b806e5 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -684,7 +684,10 @@ def completion_matcher( identifier : Optional[str] identifier of the matcher allowing users to modify the behaviour via traitlets, and also used to for debugging (will be passed as ``origin`` with the completions). - Defaults to matcher function ``__qualname__``. + + Defaults to matcher function's ``__qualname__`` (for example, + ``IPCompleter.file_matcher`` for the built-in matched defined + as a ``file_matcher`` method of the ``IPCompleter`` class). api_version: Optional[int] version of the Matcher API used by this matcher. Currently supported values are 1 and 2. @@ -1447,14 +1450,18 @@ def _greedy_changed(self, change): If False, only the completion results from the first non-empty completer will be returned. - + As of version 8.6.0, setting the value to ``False`` is an alias for: ``IPCompleter.suppress_competing_matchers = True.``. """, ).tag(config=True) disable_matchers = ListTrait( - Unicode(), help="""List of matchers to disable.""" + Unicode(), + help="""List of matchers to disable. + + The list should contain matcher identifiers (see :any:`completion_matcher`). + """, ).tag(config=True) omit__names = Enum( diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index f442ba15259..87fe3eed3a5 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -82,6 +82,8 @@ def config(self, s): Current: False IPCompleter.disable_matchers=... List of matchers to disable. + The list should contain matcher identifiers (see + :any:`completion_matcher`). Current: [] IPCompleter.greedy= Activate greedy completion From 2c55f54336afc3bae882220e483e3dbacb9248f1 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 15 Nov 2022 11:31:53 +0100 Subject: [PATCH 009/122] MAINT:add py.typed --- IPython/py.typed | 0 MANIFEST.in | 1 + 2 files changed, 1 insertion(+) create mode 100644 IPython/py.typed diff --git a/IPython/py.typed b/IPython/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/MANIFEST.in b/MANIFEST.in index c70c57d346f..970adeef334 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include LICENSE include setupbase.py include MANIFEST.in include pytest.ini +include py.typed include mypy.ini include .mailmap include .flake8 From cfb05c7ae1e0576d89ea75ccd6d0ba5ff863d8ba Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Fri, 18 Nov 2022 15:31:42 +0200 Subject: [PATCH 010/122] Pin minimum prompt-toolkit to 3.0.11 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b3a26586cf3..18af28885e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = matplotlib-inline pexpect>4.3; sys_platform != "win32" pickleshare - prompt_toolkit>3.0.1,<3.1.0 + prompt_toolkit>=3.0.11,<3.1.0 pygments>=2.4.0 stack_data traitlets>=5 From bd008c4c8ac27f5f6779841c294074bb89f2ad4f Mon Sep 17 00:00:00 2001 From: nfgf Date: Fri, 25 Nov 2022 00:44:21 -0500 Subject: [PATCH 011/122] 1st proposal --- IPython/core/interactiveshell.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 21e428b54d4..689ddfe30d7 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2362,6 +2362,14 @@ def run_line_magic(self, magic_name: str, line, _stack_depth=1): kwargs['local_ns'] = self.get_local_scope(stack_depth) with self.builtin_trap: result = fn(*args, **kwargs) + + # The code below prevents output from being displayed + # when using magic %time. + # Output from '%time foo();', for instance, would never + # be displayed. + if magic_name == 'time' and len(magic_arg_s) > 0 and magic_arg_s[-1] == ';': + return None + return result def get_local_scope(self, stack_depth): From d0be67275d918e187de37943854f9db8e2ecec40 Mon Sep 17 00:00:00 2001 From: Nir Schulman Date: Fri, 25 Nov 2022 14:56:58 +0200 Subject: [PATCH 012/122] Reused the previously unused find_entry_points in setup.py --- setup.cfg | 3 --- setup.py | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index b3a26586cf3..5c7554a28fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -106,9 +106,6 @@ IPython.lib.tests = *.wav IPython.testing.plugin = *.txt [options.entry_points] -console_scripts = - ipython = IPython:start_ipython - ipython3 = IPython:start_ipython pygments.lexers = ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer ipython = IPython.lib.lexers:IPythonLexer diff --git a/setup.py b/setup.py index bfdf5fb88bf..dca0cd3b3aa 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ # Our own imports sys.path.insert(0, ".") -from setupbase import target_update +from setupbase import target_update, find_entry_points from setupbase import ( setup_args, @@ -139,6 +139,9 @@ 'install_scripts_sym': install_scripts_for_symlink, 'unsymlink': unsymlink, } +setup_args["entry_points"] = { + "console_scripts": find_entry_points() +} #--------------------------------------------------------------------------- # Do the actual setup now From cfa39c55f44e0f117856cf64ea1ff895392d8f82 Mon Sep 17 00:00:00 2001 From: Nir Schulman Date: Fri, 25 Nov 2022 17:35:07 +0200 Subject: [PATCH 013/122] Fixed formatting --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index dca0cd3b3aa..4939ca53836 100644 --- a/setup.py +++ b/setup.py @@ -139,9 +139,7 @@ 'install_scripts_sym': install_scripts_for_symlink, 'unsymlink': unsymlink, } -setup_args["entry_points"] = { - "console_scripts": find_entry_points() -} +setup_args["entry_points"] = {"console_scripts": find_entry_points()} #--------------------------------------------------------------------------- # Do the actual setup now From f4081a6ff29cce329ec1da25599ba06c4fafe67e Mon Sep 17 00:00:00 2001 From: nfgf Date: Fri, 25 Nov 2022 10:52:05 -0500 Subject: [PATCH 014/122] Formatting fixes after running dark. --- IPython/core/interactiveshell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 689ddfe30d7..c8aacdc22cf 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2367,7 +2367,7 @@ def run_line_magic(self, magic_name: str, line, _stack_depth=1): # when using magic %time. # Output from '%time foo();', for instance, would never # be displayed. - if magic_name == 'time' and len(magic_arg_s) > 0 and magic_arg_s[-1] == ';': + if magic_name == "time" and len(magic_arg_s) > 0 and magic_arg_s[-1] == ";": return None return result From b6e1073c7f9f8e5cc20b4349b607f6324f7b7d06 Mon Sep 17 00:00:00 2001 From: nfgf Date: Sat, 26 Nov 2022 14:54:01 -0500 Subject: [PATCH 015/122] Adding test. --- IPython/core/tests/test_magic.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 509dd66dd28..ce0fbbea684 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -416,6 +416,21 @@ def test_time(): with tt.AssertPrints("hihi", suppress=False): ip.run_cell("f('hi')") +# ';' at the end of %time prevents instruction value to be printed. +# This tests fix for #13837 +def test_time_no_outputwith_semicolon(): + ip = get_ipython() + + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456") + + with tt.AssertNotPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456;") + def test_time_last_not_expression(): ip.run_cell("%%time\n" "var_1 = 1\n" From 31f22916c1fd38d6570df4c64539d2a2f1e976b5 Mon Sep 17 00:00:00 2001 From: nfgf Date: Sat, 26 Nov 2022 15:07:55 -0500 Subject: [PATCH 016/122] Change after running darker. --- IPython/core/tests/test_magic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index ce0fbbea684..9c44320740a 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -416,6 +416,7 @@ def test_time(): with tt.AssertPrints("hihi", suppress=False): ip.run_cell("f('hi')") + # ';' at the end of %time prevents instruction value to be printed. # This tests fix for #13837 def test_time_no_outputwith_semicolon(): @@ -431,6 +432,7 @@ def test_time_no_outputwith_semicolon(): with tt.AssertPrints("CPU times: ", suppress=False): ip.run_cell("%time 123000+456;") + def test_time_last_not_expression(): ip.run_cell("%%time\n" "var_1 = 1\n" From a07a31dc85da8ee493f1718728ae39bf50bc8ea9 Mon Sep 17 00:00:00 2001 From: nfgf Date: Sat, 26 Nov 2022 15:21:41 -0500 Subject: [PATCH 017/122] Again --- IPython/core/tests/test_magic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 9c44320740a..cb9890c4682 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -418,7 +418,7 @@ def test_time(): # ';' at the end of %time prevents instruction value to be printed. -# This tests fix for #13837 +# This tests fix for #13837. def test_time_no_outputwith_semicolon(): ip = get_ipython() From 12d1fb6d179b4d1a40bc6b8ce0adfd3ac536fd6e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 28 Nov 2022 10:06:35 +0100 Subject: [PATCH 018/122] What's new 8.7 --- docs/source/whatsnew/version8.rst | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index eee7af0aa48..d3c33704bad 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -2,6 +2,32 @@ 8.x Series ============ + +.. _version 8.7.0: + +IPython 8.7.0 +------------- + + +Small release of IPython with a couple of bug fixes and new features for this +month. Next month is end of year, it is unclear if there will be a release close +the new year's eve, or if the next release will be at end of January. + +Here are a few of the relevant fixes, +as usual you can find the full list of PRs on GitHub under `the 8.7 milestone +`__. + + + - :ghpull:`13834` bump the minimum prompt toolkit to 3.0.11. + - IPython shipped with the ``py.typed`` marker now, and we are progressively + adding more types. :ghpull:`13831` + - :ghpull:`13817` add configuration of code blacks formatting. + + +Thanks to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. + + .. _version 8.6.0: IPython 8.6.0 @@ -40,7 +66,7 @@ As we follow NEP 29, we removed support for numpy 1.19 :ghpull:`13760`. The ``open()`` function present in the user namespace by default will now refuse to open the file descriptors 0,1,2 (stdin, out, err), to avoid crashing IPython. -This mostly occurs in teaching context when incorrect values get passed around. +This mostly occurs in teaching context when incorrect values get passed around. The ``?``, ``??``, and corresponding ``pinfo``, ``pinfo2`` magics can now find From ff770b25d03140e6f9355625601d609d4a5464e8 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 28 Nov 2022 14:50:12 +0100 Subject: [PATCH 019/122] release 8.7.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index d891c34cc62..ff1acafac66 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,7 +20,7 @@ _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -# _version_extra = "" # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 3f0bf05f072a91b2a3042d23ce250e5e906183fd Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 28 Nov 2022 14:51:08 +0100 Subject: [PATCH 020/122] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index ff1acafac66..e2ce2eac2b4 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -16,11 +16,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 7 +_version_minor = 8 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -_version_extra = "" # Uncomment this for full releases +# _version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 81300ca6de7e50ea521073dbf49cc5eb4a276d66 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 28 Nov 2022 14:57:08 +0100 Subject: [PATCH 021/122] Make sure build is installed --- tools/release_helper.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 697ed859e84..d221f551e66 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -8,6 +8,7 @@ python -c 'import twine' python -c 'import sphinx' python -c 'import sphinx_rtd_theme' python -c 'import pytest' +python -c 'import build' BLACK=$(tput setaf 1) From bd3807ac61ffb208e146bde78726e676e4092512 Mon Sep 17 00:00:00 2001 From: nfgf Date: Tue, 29 Nov 2022 17:59:33 -0500 Subject: [PATCH 022/122] Implementing decorator --- IPython/core/displayhook.py | 8 +++++++- IPython/core/interactiveshell.py | 12 ++++++------ IPython/core/magic.py | 11 ++++++++++- IPython/core/magics/execution.py | 2 ++ IPython/core/tests/test_magic.py | 17 ++++++++++++++++- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 578e783ab8e..25aa2f03274 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -91,7 +91,13 @@ def quiet(self): # some uses of ipshellembed may fail here return False - sio = _io.StringIO(cell) + return self.semicolon_at_end_of_expression(cell) + + @staticmethod + def semicolon_at_end_of_expression(expression): + """Parse Python expression and detects whether last token is ';'""" + + sio = _io.StringIO(expression) tokens = list(tokenize.generate_tokens(sio.readline)) for token in reversed(tokens): diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index c8aacdc22cf..69b441816a4 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2363,12 +2363,12 @@ def run_line_magic(self, magic_name: str, line, _stack_depth=1): with self.builtin_trap: result = fn(*args, **kwargs) - # The code below prevents output from being displayed - # when using magic %time. - # Output from '%time foo();', for instance, would never - # be displayed. - if magic_name == "time" and len(magic_arg_s) > 0 and magic_arg_s[-1] == ";": - return None + # The code below prevents the output from being displayed + # when using magics with decodator @output_can_be_disabled + # when the last Python token in the expression is a ';'. + if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_DISABLED, False): + if DisplayHook.semicolon_at_end_of_expression(magic_arg_s): + return None return result diff --git a/IPython/core/magic.py b/IPython/core/magic.py index cedba619378..0eadc179762 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -258,7 +258,7 @@ def mark(func, *a, **kw): MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand' - +MAGIC_OUTPUT_CAN_BE_DISABLED = '_ipython_magic_output_can_be_disabled' def no_var_expand(magic_func): """Mark a magic function as not needing variable expansion @@ -275,6 +275,15 @@ def no_var_expand(magic_func): setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True) return magic_func +def output_can_be_disabled(magic_func): + """Mark a magic function so its output may be disabled. + + The output is disabled if the Python expression used as a parameter of + the magic ends in a semicolon, not counting a Python comment that can + follows it. + """ + setattr(magic_func, MAGIC_OUTPUT_CAN_BE_DISABLED, True) + return magic_func # Create the actual decorators for public use diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index da7f780b9cb..5d7942f6472 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -37,6 +37,7 @@ magics_class, needs_local_scope, no_var_expand, + output_can_be_disabled, on_off, ) from IPython.testing.skipdoctest import skip_doctest @@ -1194,6 +1195,7 @@ def timeit(self, line='', cell=None, local_ns=None): @no_var_expand @needs_local_scope @line_cell_magic + @output_can_be_disabled def time(self,line='', cell=None, local_ns=None): """Time execution of a Python statement or expression. diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index cb9890c4682..55408d4af1e 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -419,7 +419,7 @@ def test_time(): # ';' at the end of %time prevents instruction value to be printed. # This tests fix for #13837. -def test_time_no_outputwith_semicolon(): +def test_time_no_output_with_semicolon(): ip = get_ipython() with tt.AssertPrints(" 123456"): @@ -432,6 +432,21 @@ def test_time_no_outputwith_semicolon(): with tt.AssertPrints("CPU times: ", suppress=False): ip.run_cell("%time 123000+456;") + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456 # Comment") + + with tt.AssertNotPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456; # Comment") + + with tt.AssertPrints(" 123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%time 123000+456 # ;Comment") + def test_time_last_not_expression(): ip.run_cell("%%time\n" From a2de719356f944389f23a9e90747a9853d5a6e64 Mon Sep 17 00:00:00 2001 From: nfgf Date: Tue, 29 Nov 2022 18:07:19 -0500 Subject: [PATCH 023/122] Formatting to make darker happy. --- IPython/core/magic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 0eadc179762..2381bf96dac 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -275,6 +275,7 @@ def no_var_expand(magic_func): setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True) return magic_func + def output_can_be_disabled(magic_func): """Mark a magic function so its output may be disabled. From 33f18f7c11f178a5610ce620151a1abf1abc53fb Mon Sep 17 00:00:00 2001 From: nfgf Date: Tue, 29 Nov 2022 18:16:15 -0500 Subject: [PATCH 024/122] Again --- IPython/core/magic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 2381bf96dac..a5a55e6fbcf 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -260,6 +260,7 @@ def mark(func, *a, **kw): MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand' MAGIC_OUTPUT_CAN_BE_DISABLED = '_ipython_magic_output_can_be_disabled' + def no_var_expand(magic_func): """Mark a magic function as not needing variable expansion From 10a27e1d026040a5f0415b052aa5e0614bbece46 Mon Sep 17 00:00:00 2001 From: nfgf Date: Tue, 29 Nov 2022 18:26:59 -0500 Subject: [PATCH 025/122] And again --- IPython/core/displayhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 25aa2f03274..aba4f904d8d 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -92,7 +92,7 @@ def quiet(self): return False return self.semicolon_at_end_of_expression(cell) - + @staticmethod def semicolon_at_end_of_expression(expression): """Parse Python expression and detects whether last token is ';'""" From a9f3943371cd8fd9e028f4e04d3bee0d43fc5dbe Mon Sep 17 00:00:00 2001 From: nfgf Date: Tue, 29 Nov 2022 18:35:18 -0500 Subject: [PATCH 026/122] sigh --- IPython/core/magic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index a5a55e6fbcf..82728cdebd1 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -257,8 +257,8 @@ def mark(func, *a, **kw): return magic_deco -MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand' -MAGIC_OUTPUT_CAN_BE_DISABLED = '_ipython_magic_output_can_be_disabled' +MAGIC_NO_VAR_EXPAND_ATTR = "_ipython_magic_no_var_expand" +MAGIC_OUTPUT_CAN_BE_DISABLED = "_ipython_magic_output_can_be_disabled" def no_var_expand(magic_func): From 4b0aed94df9573566e87941eec4e23bd58727097 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Wed, 30 Nov 2022 02:23:01 +0000 Subject: [PATCH 027/122] Implement guarded evaluation, replace greedy, implement: - completion of integer keys - completion in pandas for loc indexer `.loc[:, ` --- IPython/core/completer.py | 243 +++++++---- IPython/core/guarded_eval.py | 541 ++++++++++++++++++++++++ IPython/core/tests/test_completer.py | 56 ++- IPython/core/tests/test_guarded_eval.py | 286 +++++++++++++ 4 files changed, 1027 insertions(+), 99 deletions(-) create mode 100644 IPython/core/guarded_eval.py create mode 100644 IPython/core/tests/test_guarded_eval.py diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 2dff9efbbfe..a497f12f01c 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -190,6 +190,7 @@ import unicodedata import uuid import warnings +from ast import literal_eval from contextlib import contextmanager from dataclasses import dataclass from functools import cached_property, partial @@ -212,6 +213,7 @@ Literal, ) +from IPython.core.guarded_eval import guarded_eval, EvaluationContext from IPython.core.error import TryNext from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol @@ -296,6 +298,9 @@ def cast(obj, type_): # Completion type reported when no type can be inferred. _UNKNOWN_TYPE = "" +# sentinel value to signal lack of a match +not_found = object() + class ProvisionalCompleterWarning(FutureWarning): """ Exception raise by an experimental feature in this module. @@ -902,12 +907,33 @@ def split_line(self, line, cursor_pos=None): class Completer(Configurable): - greedy = Bool(False, - help="""Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. + greedy = Bool( + False, + help="""Activate greedy completion. + + .. deprecated:: 8.8 + Use :any:`evaluation` instead. + + As of IPython 8.8 proxy for ``evaluation = 'unsafe'`` when set to ``True``, + and for ``'forbidden'`` when set to ``False``. + """, + ).tag(config=True) - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. + evaluation = Enum( + ('forbidden', 'minimal', 'limitted', 'unsafe', 'dangerous'), + default_value='limitted', + help="""Code evaluation under completion. + + Successive options allow to enable more eager evaluation for more accurate completion suggestions, + including for nested dictionaries, nested lists, or even results of function calls. Setting `unsafe` + or higher can lead to evaluation of arbitrary user code on TAB with potentially dangerous side effects. + + Allowed values are: + - `forbidden`: no evaluation at all + - `minimal`: evaluation of literals and access to built-in namespaces; no item/attribute evaluation nor access to locals/globals + - `limitted` (default): access to all namespaces, evaluation of hard-coded methods (``keys()``, ``__getattr__``, ``__getitems__``, etc) on allow-listed objects (e.g. ``dict``, ``list``, ``tuple``, ``pandas.Series``) + - `unsafe`: evaluation of all methods and function calls but not of syntax with side-effects like `del x`, + - `dangerous`: completely arbitrary evaluation """, ).tag(config=True) @@ -1029,28 +1055,16 @@ def attr_matches(self, text): with a __getattr__ hook is evaluated. """ + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return [] + expr, attr = m2.group(1,2) - # Another option, seems to work great. Catches things like ''. - m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) + obj = self._evaluate_expr(expr) - if m: - expr, attr = m.group(1, 3) - elif self.greedy: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return [] - expr, attr = m2.group(1,2) - else: + if obj is not_found: return [] - try: - obj = eval(expr, self.namespace) - except: - try: - obj = eval(expr, self.global_namespace) - except: - return [] - if self.limit_to__all__ and hasattr(obj, '__all__'): words = get__all__entries(obj) else: @@ -1068,9 +1082,33 @@ def attr_matches(self, text): pass # Build match list to return n = len(attr) - return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] + return ["%s.%s" % (expr, w) for w in words if w[:n] == attr ] + def _evaluate_expr(self, expr): + obj = not_found + done = False + while not done and expr: + try: + obj = guarded_eval( + expr, + EvaluationContext( + globals_=self.global_namespace, + locals_=self.namespace, + evaluation=self.evaluation + ) + ) + done = True + except Exception as e: + if self.debug: + print('Evaluation exception', e) + # trim the expression to remove any invalid prefix + # e.g. user starts `(d[`, so we get `expr = '(d'`, + # where parenthesis is not closed. + # TODO: make this faster by reusing parts of the computation? + expr = expr[1:] + return obj + def get__all__entries(obj): """returns the strings in the __all__ attribute""" try: @@ -1081,8 +1119,8 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str, - extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]: +def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], prefix: str, delims: str, + extra_prefix: Optional[Tuple[Union[str, bytes], ...]]=None) -> Tuple[str, int, List[str]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1106,25 +1144,28 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre """ prefix_tuple = extra_prefix if extra_prefix else () + Nprefix = len(prefix_tuple) + text_serializable_types = (str, bytes, int, float, slice) def filter_prefix_tuple(key): # Reject too short keys if len(key) <= Nprefix: return False - # Reject keys with non str/bytes in it + # Reject keys which cannot be serialised to text for k in key: - if not isinstance(k, (str, bytes)): + if not isinstance(k, text_serializable_types): return False # Reject keys that do not match the prefix for k, pt in zip(key, prefix_tuple): - if k != pt: + if k != pt and not isinstance(pt, slice): return False # All checks passed! return True - filtered_keys:List[Union[str,bytes]] = [] + filtered_keys: List[Union[str, bytes, int, float, slice]] = [] + def _add_to_filtered_keys(key): - if isinstance(key, (str, bytes)): + if isinstance(key, text_serializable_types): filtered_keys.append(key) for k in keys: @@ -1140,7 +1181,7 @@ def _add_to_filtered_keys(key): assert quote_match is not None # silence mypy quote = quote_match.group() try: - prefix_str = eval(prefix + quote, {}) + prefix_str = literal_eval(prefix + quote) except Exception: return '', 0, [] @@ -1150,17 +1191,18 @@ def _add_to_filtered_keys(key): token_start = token_match.start() token_prefix = token_match.group() - matched:List[str] = [] + matched: List[str] = [] for key in filtered_keys: + str_key = key if isinstance(key, (str, bytes)) else str(key) try: - if not key.startswith(prefix_str): + if not str_key.startswith(prefix_str): continue except (AttributeError, TypeError, UnicodeError): # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue # reformat remainder of key to begin with prefix - rem = key[len(prefix_str):] + rem = str_key[len(prefix_str):] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') rem_repr = rem_repr[1 + rem_repr.index("'"):-2] @@ -1237,11 +1279,14 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: return line, col -def _safe_isinstance(obj, module, class_name): +def _safe_isinstance(obj, module, class_name, *attrs): """Checks if obj is an instance of module.class_name if loaded """ - return (module in sys.modules and - isinstance(obj, getattr(import_module(module), class_name))) + if module in sys.modules: + m = sys.modules[module] + for attr in [class_name, *attrs]: + m = getattr(m, attr) + return isinstance(obj, m) @context_matcher() @@ -1394,6 +1439,37 @@ def _make_signature(completion)-> str: _CompleteResult = Dict[str, MatcherResult] +DICT_MATCHER_REGEX = re.compile(r"""(?x) +( # match dict-referring - or any get item object - expression + .+ +) +\[ # open bracket +\s* # and optional whitespace +# Capture any number of serializable objects (e.g. "a", "b", 'c') +# and slices +((?:[uUbB]? # string prefix (r not handled) + (?: + '(?:[^']|(? List[Any]: return method() # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): + if (isinstance(obj, dict) or + _safe_isinstance(obj, 'pandas', 'DataFrame')): try: return list(obj.keys()) except Exception: return [] + elif _safe_isinstance(obj, 'pandas', 'core', 'indexing', '_LocIndexer'): + try: + return list(obj.obj.keys()) + except Exception: + return [] elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ _safe_isinstance(obj, 'numpy', 'void'): return obj.dtype.names or [] @@ -2175,65 +2256,43 @@ def dict_key_matches(self, text: str) -> List[str]: You can use :meth:`dict_key_matcher` instead. """ - if self.__dict_key_regexps is not None: - regexps = self.__dict_key_regexps - else: - dict_key_re_fmt = r'''(?x) - ( # match dict-referring expression wrt greedy setting - %s - ) - \[ # open bracket - \s* # and optional whitespace - # Capture any number of str-like objects (e.g. "a", "b", 'c') - ((?:[uUbB]? # string prefix (r not handled) - (?: - '(?:[^']|(? List[str]: # - the start of the key text # - the start of the completion text_start = len(self.text_until_cursor) - len(text) - if prefix: + if key_prefix: key_start = match.start(3) completion_start = key_start + token_offset else: diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py new file mode 100644 index 00000000000..f477c6bc2c1 --- /dev/null +++ b/IPython/core/guarded_eval.py @@ -0,0 +1,541 @@ +from typing import Callable, Protocol, Set, Tuple, NamedTuple, Literal, Union +import collections +import sys +import ast +import types +from functools import cached_property +from dataclasses import dataclass, field + + +class HasGetItem(Protocol): + def __getitem__(self, key) -> None: ... + + +class InstancesHaveGetItem(Protocol): + def __call__(self) -> HasGetItem: ... + + +class HasGetAttr(Protocol): + def __getattr__(self, key) -> None: ... + + +class DoesNotHaveGetAttr(Protocol): + pass + +# By default `__getattr__` is not explicitly implemented on most objects +MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] + + +def unbind_method(func: Callable) -> Union[Callable, None]: + """Get unbound method for given bound method. + + Returns None if cannot get unbound method.""" + owner = getattr(func, '__self__', None) + owner_class = type(owner) + name = getattr(func, '__name__', None) + instance_dict_overrides = getattr(owner, '__dict__', None) + if ( + owner is not None + and + name + and + ( + not instance_dict_overrides + or + ( + instance_dict_overrides + and name not in instance_dict_overrides + ) + ) + ): + return getattr(owner_class, name) + + +@dataclass +class EvaluationPolicy: + allow_locals_access: bool = False + allow_globals_access: bool = False + allow_item_access: bool = False + allow_attr_access: bool = False + allow_builtins_access: bool = False + allow_any_calls: bool = False + allowed_calls: Set[Callable] = field(default_factory=set) + + def can_get_item(self, value, item): + return self.allow_item_access + + def can_get_attr(self, value, attr): + return self.allow_attr_access + + def can_call(self, func): + if self.allow_any_calls: + return True + + if func in self.allowed_calls: + return True + + owner_method = unbind_method(func) + if owner_method and owner_method in self.allowed_calls: + return True + +def has_original_dunder_external(value, module_name, access_path, method_name,): + try: + if module_name not in sys.modules: + return False + member_type = sys.modules[module_name] + for attr in access_path: + member_type = getattr(member_type, attr) + value_type = type(value) + if type(value) == member_type: + return True + if isinstance(value, member_type): + method = getattr(value_type, method_name, None) + member_method = getattr(member_type, method_name, None) + if member_method == method: + return True + except (AttributeError, KeyError): + return False + + +def has_original_dunder( + value, + allowed_types, + allowed_methods, + allowed_external, + method_name +): + # note: Python ignores `__getattr__`/`__getitem__` on instances, + # we only need to check at class level + value_type = type(value) + + # strict type check passes → no need to check method + if value_type in allowed_types: + return True + + method = getattr(value_type, method_name, None) + + if not method: + return None + + if method in allowed_methods: + return True + + for module_name, *access_path in allowed_external: + if has_original_dunder_external(value, module_name, access_path, method_name): + return True + + return False + + +@dataclass +class SelectivePolicy(EvaluationPolicy): + allowed_getitem: Set[HasGetItem] = field(default_factory=set) + allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) + allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) + allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) + + def can_get_attr(self, value, attr): + has_original_attribute = has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattribute_methods, + allowed_external=self.allowed_getattr_external, + method_name='__getattribute__' + ) + has_original_attr = has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattr_methods, + allowed_external=self.allowed_getattr_external, + method_name='__getattr__' + ) + # Many objects do not have `__getattr__`, this is fine + if has_original_attr is None and has_original_attribute: + return True + + # Accept objects without modifications to `__getattr__` and `__getattribute__` + return has_original_attr and has_original_attribute + + def get_attr(self, value, attr): + if self.can_get_attr(value, attr): + return getattr(value, attr) + + + def can_get_item(self, value, item): + """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" + return has_original_dunder( + value, + allowed_types=self.allowed_getitem, + allowed_methods=self._getitem_methods, + allowed_external=self.allowed_getitem_external, + method_name='__getitem__' + ) + + @cached_property + def _getitem_methods(self) -> Set[Callable]: + return self._safe_get_methods( + self.allowed_getitem, + '__getitem__' + ) + + @cached_property + def _getattr_methods(self) -> Set[Callable]: + return self._safe_get_methods( + self.allowed_getattr, + '__getattr__' + ) + + @cached_property + def _getattribute_methods(self) -> Set[Callable]: + return self._safe_get_methods( + self.allowed_getattr, + '__getattribute__' + ) + + def _safe_get_methods(self, classes, name) -> Set[Callable]: + return { + method + for class_ in classes + for method in [getattr(class_, name, None)] + if method + } + + +class DummyNamedTuple(NamedTuple): + pass + + +class EvaluationContext(NamedTuple): + locals_: dict + globals_: dict + evaluation: Literal['forbidden', 'minimal', 'limitted', 'unsafe', 'dangerous'] = 'forbidden' + in_subscript: bool = False + + +class IdentitySubscript: + def __getitem__(self, key): + return key + +IDENTITY_SUBSCRIPT = IdentitySubscript() +SUBSCRIPT_MARKER = '__SUBSCRIPT_SENTINEL__' + +class GuardRejection(ValueError): + pass + + +def guarded_eval( + code: str, + context: EvaluationContext +): + locals_ = context.locals_ + + if context.evaluation == 'forbidden': + raise GuardRejection('Forbidden mode') + + # note: not using `ast.literal_eval` as it does not implement + # getitem at all, for example it fails on simple `[0][1]` + + if context.in_subscript: + # syntatic sugar for ellipsis (:) is only available in susbcripts + # so we need to trick the ast parser into thinking that we have + # a subscript, but we need to be able to later recognise that we did + # it so we can ignore the actual __getitem__ operation + if not code: + return tuple() + locals_ = locals_.copy() + locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT + code = SUBSCRIPT_MARKER + '[' + code + ']' + context = EvaluationContext(**{ + **context._asdict(), + **{'locals_': locals_} + }) + + if context.evaluation == 'dangerous': + return eval(code, context.globals_, context.locals_) + + expression = ast.parse(code, mode='eval') + + return eval_node(expression, context) + +def eval_node(node: Union[ast.AST, None], context: EvaluationContext): + """ + Evaluate AST node in provided context. + + Applies evaluation restrictions defined in the context. + + Currently does not support evaluation of functions with arguments. + + Does not evaluate actions which always have side effects: + - class definitions (`class sth: ...`) + - function definitions (`def sth: ...`) + - variable assignments (`x = 1`) + - augumented assignments (`x += 1`) + - deletions (`del x`) + + Does not evaluate operations which do not return values: + - assertions (`assert x`) + - pass (`pass`) + - imports (`import x`) + - control flow + - conditionals (`if x:`) except for terenary IfExp (`a if x else b`) + - loops (`for` and `while`) + - exception handling + """ + policy = EVALUATION_POLICIES[context.evaluation] + if node is None: + return None + if isinstance(node, ast.Expression): + return eval_node(node.body, context) + if isinstance(node, ast.BinOp): + # TODO: add guards + left = eval_node(node.left, context) + right = eval_node(node.right, context) + if isinstance(node.op, ast.Add): + return left + right + if isinstance(node.op, ast.Sub): + return left - right + if isinstance(node.op, ast.Mult): + return left * right + if isinstance(node.op, ast.Div): + return left / right + if isinstance(node.op, ast.FloorDiv): + return left // right + if isinstance(node.op, ast.Mod): + return left % right + if isinstance(node.op, ast.Pow): + return left ** right + if isinstance(node.op, ast.LShift): + return left << right + if isinstance(node.op, ast.RShift): + return left >> right + if isinstance(node.op, ast.BitOr): + return left | right + if isinstance(node.op, ast.BitXor): + return left ^ right + if isinstance(node.op, ast.BitAnd): + return left & right + if isinstance(node.op, ast.MatMult): + return left @ right + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Index): + return eval_node(node.value, context) + if isinstance(node, ast.Tuple): + return tuple( + eval_node(e, context) + for e in node.elts + ) + if isinstance(node, ast.List): + return [ + eval_node(e, context) + for e in node.elts + ] + if isinstance(node, ast.Set): + return { + eval_node(e, context) + for e in node.elts + } + if isinstance(node, ast.Dict): + return dict(zip( + [eval_node(k, context) for k in node.keys], + [eval_node(v, context) for v in node.values] + )) + if isinstance(node, ast.Slice): + return slice( + eval_node(node.lower, context), + eval_node(node.upper, context), + eval_node(node.step, context) + ) + if isinstance(node, ast.ExtSlice): + return tuple([ + eval_node(dim, context) + for dim in node.dims + ]) + if isinstance(node, ast.UnaryOp): + # TODO: add guards + value = eval_node(node.operand, context) + if isinstance(node.op, ast.USub): + return -value + if isinstance(node.op, ast.UAdd): + return +value + if isinstance(node.op, ast.Invert): + return ~value + if isinstance(node.op, ast.Not): + return not value + raise ValueError('Unhandled unary operation:', node.op) + if isinstance(node, ast.Subscript): + value = eval_node(node.value, context) + slice_ = eval_node(node.slice, context) + if policy.can_get_item(value, slice_): + return value[slice_] + raise GuardRejection( + 'Subscript access (`__getitem__`) for', + type(value), # not joined to avoid calling `repr` + f' not allowed in {context.evaluation} mode' + ) + if isinstance(node, ast.Name): + if policy.allow_locals_access and node.id in context.locals_: + return context.locals_[node.id] + if policy.allow_globals_access and node.id in context.globals_: + return context.globals_[node.id] + if policy.allow_builtins_access and node.id in __builtins__: + return __builtins__[node.id] + if not policy.allow_globals_access and not policy.allow_locals_access: + raise GuardRejection( + f'Namespace access not allowed in {context.evaluation} mode' + ) + else: + raise NameError(f'{node.id} not found in locals nor globals') + if isinstance(node, ast.Attribute): + value = eval_node(node.value, context) + if policy.can_get_attr(value, node.attr): + return getattr(value, node.attr) + raise GuardRejection( + 'Attribute access (`__getattr__`) for', + type(value), # not joined to avoid calling `repr` + f'not allowed in {context.evaluation} mode' + ) + if isinstance(node, ast.IfExp): + test = eval_node(node.test, context) + if test: + return eval_node(node.body, context) + else: + return eval_node(node.orelse, context) + if isinstance(node, ast.Call): + func = eval_node(node.func, context) + print(node.keywords) + if policy.can_call(func) and not node.keywords: + args = [ + eval_node(arg, context) + for arg in node.args + ] + return func(*args) + raise GuardRejection( + 'Call for', + func, # not joined to avoid calling `repr` + f'not allowed in {context.evaluation} mode' + ) + raise ValueError('Unhandled node', node) + + +SUPPORTED_EXTERNAL_GETITEM = { + ('pandas', 'core', 'indexing', '_iLocIndexer'), + ('pandas', 'core', 'indexing', '_LocIndexer'), + ('pandas', 'DataFrame'), + ('pandas', 'Series'), + ('numpy', 'ndarray'), + ('numpy', 'void') +} + +BUILTIN_GETITEM = { + dict, + str, + bytes, + list, + tuple, + collections.defaultdict, + collections.deque, + collections.OrderedDict, + collections.ChainMap, + collections.UserDict, + collections.UserList, + collections.UserString, + DummyNamedTuple, + IdentitySubscript +} + + +def _list_methods(cls, source=None): + """For use on immutable objects or with methods returning a copy""" + return [ + getattr(cls, k) + for k in (source if source else dir(cls)) + ] + + +dict_non_mutating_methods = ('copy', 'keys', 'values', 'items') +list_non_mutating_methods = ('copy', 'index', 'count') +set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) + + +dict_keys = type({}.keys()) +method_descriptor = type(list.copy) + +ALLOWED_CALLS = { + bytes, + *_list_methods(bytes), + dict, + *_list_methods(dict, dict_non_mutating_methods), + dict_keys.isdisjoint, + list, + *_list_methods(list, list_non_mutating_methods), + set, + *_list_methods(set, set_non_mutating_methods), + frozenset, + *_list_methods(frozenset), + range, + str, + *_list_methods(str), + tuple, + *_list_methods(tuple), + collections.deque, + *_list_methods(collections.deque, list_non_mutating_methods), + collections.defaultdict, + *_list_methods(collections.defaultdict, dict_non_mutating_methods), + collections.OrderedDict, + *_list_methods(collections.OrderedDict, dict_non_mutating_methods), + collections.UserDict, + *_list_methods(collections.UserDict, dict_non_mutating_methods), + collections.UserList, + *_list_methods(collections.UserList, list_non_mutating_methods), + collections.UserString, + *_list_methods(collections.UserString, dir(str)), + collections.Counter, + *_list_methods(collections.Counter, dict_non_mutating_methods), + collections.Counter.elements, + collections.Counter.most_common +} + +EVALUATION_POLICIES = { + 'minimal': EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=False, + allow_globals_access=False, + allow_item_access=False, + allow_attr_access=False, + allowed_calls=set(), + allow_any_calls=False + ), + 'limitted': SelectivePolicy( + # TODO: + # - should reject binary and unary operations if custom methods would be dispatched + allowed_getitem=BUILTIN_GETITEM, + allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, + allowed_getattr={ + *BUILTIN_GETITEM, + set, + frozenset, + object, + type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + dict_keys, + method_descriptor + }, + allowed_getattr_external={ + # pandas Series/Frame implements custom `__getattr__` + ('pandas', 'DataFrame'), + ('pandas', 'Series') + }, + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allowed_calls=ALLOWED_CALLS + ), + 'unsafe': EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allow_attr_access=True, + allow_item_access=True, + allow_any_calls=True + ) +} \ No newline at end of file diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 98ec814a769..7a99a2655ab 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -112,6 +112,17 @@ def greedy_completion(): ip.Completer.greedy = greedy_original +@contextmanager +def evaluation_level(evaluation: str): + ip = get_ipython() + evaluation_original = ip.Completer.evaluation + try: + ip.Completer.evaluation = evaluation + yield + finally: + ip.Completer.evaluation = evaluation_original + + @contextmanager def custom_matchers(matchers): ip = get_ipython() @@ -522,10 +533,10 @@ class Z: def test_greedy_completions(self): """ - Test the capability of the Greedy completer. + Test the capability of the Greedy completer. Most of the test here does not really show off the greedy completer, for proof - each of the text below now pass with Jedi. The greedy completer is capable of more. + each of the text below now pass with Jedi. The greedy completer is capable of more. See the :any:`test_dict_key_completion_contexts` @@ -852,15 +863,13 @@ def test_match_dict_keys(self): assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"]) assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"]) - match_dict_keys - def test_match_dict_keys_tuple(self): """ Test that match_dict_keys called with extra prefix works on a couple of use case, does return what expected, and does not crash. """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" - + keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')] # Completion on first key == "foo" @@ -883,6 +892,11 @@ def test_match_dict_keys_tuple(self): assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, 'test')] + assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["1111", "2222"]) + assert match_dict_keys(keys, "'", delims=delims, extra_prefix=(3333,)) == ("'", 1, ["bar", "test"]) + assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("3333",)) == ("'", 1, []) + def test_dict_key_completion_string(self): """Test dictionary key completion for string keys""" ip = get_ipython() @@ -1050,6 +1064,7 @@ class C: ip.user_ns["C"] = C ip.user_ns["get"] = lambda: d + ip.user_ns["nested"] = {'x': d} def assert_no_completion(**kwargs): _, matches = complete(**kwargs) @@ -1075,6 +1090,13 @@ def assert_completion(**kwargs): assert_completion(line_buffer="(d[") assert_completion(line_buffer="C.data[") + # nested dict completion + assert_completion(line_buffer="nested['x'][") + + with evaluation_level('minimal'): + with pytest.raises(AssertionError): + assert_completion(line_buffer="nested['x'][") + # greedy flag def assert_completion(**kwargs): _, matches = complete(**kwargs) @@ -1162,12 +1184,21 @@ def test_struct_array_key_completion(self): _, matches = complete(line_buffer="d['") self.assertIn("my_head", matches) self.assertIn("my_data", matches) - # complete on a nested level - with greedy_completion(): + def completes_on_nested(): ip.user_ns["d"] = numpy.zeros(2, dtype=dt) _, matches = complete(line_buffer="d[1]['my_head']['") self.assertTrue(any(["my_dt" in m for m in matches])) self.assertTrue(any(["my_df" in m for m in matches])) + # complete on a nested level + with greedy_completion(): + completes_on_nested() + + with evaluation_level('limitted'): + completes_on_nested() + + with evaluation_level('minimal'): + with pytest.raises(AssertionError): + completes_on_nested() @dec.skip_without("pandas") def test_dataframe_key_completion(self): @@ -1180,6 +1211,17 @@ def test_dataframe_key_completion(self): _, matches = complete(line_buffer="d['") self.assertIn("hello", matches) self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[:, '") + self.assertIn("hello", matches) + self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[1:, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1:-1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[::, '") + self.assertIn("hello", matches) def test_dict_key_completion_invalids(self): """Smoke test cases dict key completion can't handle""" diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py new file mode 100644 index 00000000000..5c89a68f637 --- /dev/null +++ b/IPython/core/tests/test_guarded_eval.py @@ -0,0 +1,286 @@ +from typing import NamedTuple +from IPython.core.guarded_eval import EvaluationContext, GuardRejection, guarded_eval, unbind_method +from IPython.testing import decorators as dec +import pytest + + +def limitted(**kwargs): + return EvaluationContext( + locals_=kwargs, + globals_={}, + evaluation='limitted' + ) + + +def unsafe(**kwargs): + return EvaluationContext( + locals_=kwargs, + globals_={}, + evaluation='unsafe' + ) + +@dec.skip_without('pandas') +def test_pandas_series_iloc(): + import pandas as pd + series = pd.Series([1], index=['a']) + context = limitted(data=series) + assert guarded_eval('data.iloc[0]', context) == 1 + + +@dec.skip_without('pandas') +def test_pandas_series(): + import pandas as pd + context = limitted(data=pd.Series([1], index=['a'])) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(KeyError): + guarded_eval('data["c"]', context) + + +@dec.skip_without('pandas') +def test_pandas_bad_series(): + import pandas as pd + class BadItemSeries(pd.Series): + def __getitem__(self, key): + return 'CUSTOM_ITEM' + + class BadAttrSeries(pd.Series): + def __getattr__(self, key): + return 'CUSTOM_ATTR' + + bad_series = BadItemSeries([1], index=['a']) + context = limitted(data=bad_series) + + with pytest.raises(GuardRejection): + guarded_eval('data["a"]', context) + with pytest.raises(GuardRejection): + guarded_eval('data["c"]', context) + + # note: here result is a bit unexpected because + # pandas `__getattr__` calls `__getitem__`; + # FIXME - special case to handle it? + assert guarded_eval('data.a', context) == 'CUSTOM_ITEM' + + context = unsafe(data=bad_series) + assert guarded_eval('data["a"]', context) == 'CUSTOM_ITEM' + + bad_attr_series = BadAttrSeries([1], index=['a']) + context = limitted(data=bad_attr_series) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(GuardRejection): + guarded_eval('data.a', context) + + +@dec.skip_without('pandas') +def test_pandas_dataframe_loc(): + import pandas as pd + from pandas.testing import assert_series_equal + data = pd.DataFrame([{'a': 1}]) + context = limitted(data=data) + assert_series_equal( + guarded_eval('data.loc[:, "a"]', context), + data['a'] + ) + + +def test_named_tuple(): + + class GoodNamedTuple(NamedTuple): + a: str + pass + + class BadNamedTuple(NamedTuple): + a: str + def __getitem__(self, key): + return None + + good = GoodNamedTuple(a='x') + bad = BadNamedTuple(a='x') + + context = limitted(data=good) + assert guarded_eval('data[0]', context) == 'x' + + context = limitted(data=bad) + with pytest.raises(GuardRejection): + guarded_eval('data[0]', context) + + +def test_dict(): + context = limitted( + data={'a': 1, 'b': {'x': 2}, ('x', 'y'): 3} + ) + assert guarded_eval('data["a"]', context) == 1 + assert guarded_eval('data["b"]', context) == {'x': 2} + assert guarded_eval('data["b"]["x"]', context) == 2 + assert guarded_eval('data["x", "y"]', context) == 3 + + assert guarded_eval('data.keys', context) + + +def test_set(): + context = limitted(data={'a', 'b'}) + assert guarded_eval('data.difference', context) + + +def test_list(): + context = limitted(data=[1, 2, 3]) + assert guarded_eval('data[1]', context) == 2 + assert guarded_eval('data.copy', context) + + +def test_dict_literal(): + context = limitted() + assert guarded_eval('{}', context) == {} + assert guarded_eval('{"a": 1}', context) == {"a": 1} + + +def test_list_literal(): + context = limitted() + assert guarded_eval('[]', context) == [] + assert guarded_eval('[1, "a"]', context) == [1, "a"] + + +def test_set_literal(): + context = limitted() + assert guarded_eval('set()', context) == set() + assert guarded_eval('{"a"}', context) == {"a"} + + +def test_if_expression(): + context = limitted() + assert guarded_eval('2 if True else 3', context) == 2 + assert guarded_eval('4 if False else 5', context) == 5 + + +def test_object(): + obj = object() + context = limitted(obj=obj) + assert guarded_eval('obj.__dir__', context) == obj.__dir__ + + +@pytest.mark.parametrize( + "code,expected", + [ + [ + 'int.numerator', + int.numerator + ], + [ + 'float.is_integer', + float.is_integer + ], + [ + 'complex.real', + complex.real + ] + ] +) +def test_number_attributes(code, expected): + assert guarded_eval(code, limitted()) == expected + + +def test_method_descriptor(): + context = limitted() + assert guarded_eval('list.copy.__name__', context) == 'copy' + + +@pytest.mark.parametrize( + "data,good,bad,expected", + [ + [ + [1, 2, 3], + 'data.index(2)', + 'data.append(4)', + 1 + ], + [ + {'a': 1}, + 'data.keys().isdisjoint({})', + 'data.update()', + True + ] + ] +) +def test_calls(data, good, bad, expected): + context = limitted(data=data) + assert guarded_eval(good, context) == expected + + with pytest.raises(GuardRejection): + guarded_eval(bad, context) + + +@pytest.mark.parametrize( + "code,expected", + [ + [ + '(1\n+\n1)', + 2 + ], + [ + 'list(range(10))[-1:]', + [9] + ], + [ + 'list(range(20))[3:-2:3]', + [3, 6, 9, 12, 15] + ] + ] +) +def test_literals(code, expected): + context = limitted() + assert guarded_eval(code, context) == expected + + +def test_subscript(): + context = EvaluationContext( + locals_={}, + globals_={}, + evaluation='limitted', + in_subscript=True + ) + empty_slice = slice(None, None, None) + assert guarded_eval('', context) == tuple() + assert guarded_eval(':', context) == empty_slice + assert guarded_eval('1:2:3', context) == slice(1, 2, 3) + assert guarded_eval(':, "a"', context) == (empty_slice, "a") + + +def test_unbind_method(): + class X(list): + def index(self, k): + return 'CUSTOM' + x = X() + assert unbind_method(x.index) is X.index + assert unbind_method([].index) is list.index + + +def test_assumption_instance_attr_do_not_matter(): + """This is semi-specified in Python documentation. + + However, since the specification says 'not guaranted + to work' rather than 'is forbidden to work', future + versions could invalidate this assumptions. This test + is meant to catch such a change if it ever comes true. + """ + class T: + def __getitem__(self, k): + return 'a' + def __getattr__(self, k): + return 'a' + t = T() + t.__getitem__ = lambda f: 'b' + t.__getattr__ = lambda f: 'b' + assert t[1] == 'a' + assert t[1] == 'a' + + +def test_assumption_named_tuples_share_getitem(): + """Check assumption on named tuples sharing __getitem__""" + from typing import NamedTuple + + class A(NamedTuple): + pass + + class B(NamedTuple): + pass + + assert A.__getitem__ == B.__getitem__ From 4c580c1497d56a21acf380031b2b60a22219913a Mon Sep 17 00:00:00 2001 From: nfgf Date: Wed, 30 Nov 2022 19:27:49 -0500 Subject: [PATCH 028/122] Terminology: output is silenced, not disabled. --- IPython/core/interactiveshell.py | 4 ++-- IPython/core/magic.py | 12 ++++++------ IPython/core/magics/execution.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 69b441816a4..12503e9d916 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2364,9 +2364,9 @@ def run_line_magic(self, magic_name: str, line, _stack_depth=1): result = fn(*args, **kwargs) # The code below prevents the output from being displayed - # when using magics with decodator @output_can_be_disabled + # when using magics with decodator @output_can_be_silenced # when the last Python token in the expression is a ';'. - if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_DISABLED, False): + if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): if DisplayHook.semicolon_at_end_of_expression(magic_arg_s): return None diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 82728cdebd1..95653dc7893 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -258,7 +258,7 @@ def mark(func, *a, **kw): MAGIC_NO_VAR_EXPAND_ATTR = "_ipython_magic_no_var_expand" -MAGIC_OUTPUT_CAN_BE_DISABLED = "_ipython_magic_output_can_be_disabled" +MAGIC_OUTPUT_CAN_BE_SILENCED = "_ipython_magic_output_can_be_silenced" def no_var_expand(magic_func): @@ -277,14 +277,14 @@ def no_var_expand(magic_func): return magic_func -def output_can_be_disabled(magic_func): - """Mark a magic function so its output may be disabled. +def output_can_be_silenced(magic_func): + """Mark a magic function so its output may be silenced. - The output is disabled if the Python expression used as a parameter of + The output is silenced if the Python expression used as a parameter of the magic ends in a semicolon, not counting a Python comment that can - follows it. + follow it. """ - setattr(magic_func, MAGIC_OUTPUT_CAN_BE_DISABLED, True) + setattr(magic_func, MAGIC_OUTPUT_CAN_BE_SILENCED, True) return magic_func # Create the actual decorators for public use diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 5d7942f6472..7b558d5bc6a 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -37,7 +37,7 @@ magics_class, needs_local_scope, no_var_expand, - output_can_be_disabled, + output_can_be_silenced, on_off, ) from IPython.testing.skipdoctest import skip_doctest @@ -1195,7 +1195,7 @@ def timeit(self, line='', cell=None, local_ns=None): @no_var_expand @needs_local_scope @line_cell_magic - @output_can_be_disabled + @output_can_be_silenced def time(self,line='', cell=None, local_ns=None): """Time execution of a Python statement or expression. From 61b0fb8f67cdc870ae116b9efbdc6fab76119cc1 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Thu, 1 Dec 2022 07:07:14 -0600 Subject: [PATCH 029/122] move all entry_point definitions to setup.py --- setup.cfg | 6 ------ setup.py | 10 +++++++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 226506f08f0..769bfda14a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,12 +105,6 @@ IPython.core.tests = *.png, *.jpg, daft_extension/*.py IPython.lib.tests = *.wav IPython.testing.plugin = *.txt -[options.entry_points] -pygments.lexers = - ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer - ipython = IPython.lib.lexers:IPythonLexer - ipython3 = IPython.lib.lexers:IPython3Lexer - [velin] ignore_patterns = IPython/core/tests diff --git a/setup.py b/setup.py index 4939ca53836..454c297524f 100644 --- a/setup.py +++ b/setup.py @@ -139,7 +139,15 @@ 'install_scripts_sym': install_scripts_for_symlink, 'unsymlink': unsymlink, } -setup_args["entry_points"] = {"console_scripts": find_entry_points()} + +setup_args["entry_points"] = { + "console_scripts": find_entry_points(), + "pygments.lexers": [ + "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer", + "ipython = IPython.lib.lexers:IPythonLexer", + "ipython3 = IPython.lib.lexers:IPython3Lexer", + ] +} #--------------------------------------------------------------------------- # Do the actual setup now From 5a611b0080ff68ad344125c27dc52b8ea041abc5 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 2 Dec 2022 21:47:31 +0000 Subject: [PATCH 030/122] docs: remove mention of `_ipython_display_` being ignored in the REPL --- docs/source/config/integrating.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/config/integrating.rst b/docs/source/config/integrating.rst index 07429ef1792..23cc1e58875 100644 --- a/docs/source/config/integrating.rst +++ b/docs/source/config/integrating.rst @@ -128,7 +128,6 @@ More powerful methods Displays the object as a side effect; the return value is ignored. If this is defined, all other display methods are ignored. - This method is ignored in the REPL. Metadata From c986c9eddefc3363d98a27c9cd6aa8d58fe8053b Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:37:34 +0000 Subject: [PATCH 031/122] Re-implement key closing behaviour with a setting, improve number handling and static typing --- IPython/core/completer.py | 307 +++++++++++++++++++++------ IPython/core/guarded_eval.py | 35 +-- IPython/core/tests/test_completer.py | 191 ++++++++++++++--- 3 files changed, 430 insertions(+), 103 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index a497f12f01c..e53e83b38e9 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -178,6 +178,7 @@ from __future__ import annotations import builtins as builtin_mod +import enum import glob import inspect import itertools @@ -186,15 +187,16 @@ import re import string import sys +import tokenize import time import unicodedata import uuid import warnings from ast import literal_eval +from collections import defaultdict from contextlib import contextmanager from dataclasses import dataclass from functools import cached_property, partial -from importlib import import_module from types import SimpleNamespace from typing import ( Iterable, @@ -205,8 +207,6 @@ Any, Sequence, Dict, - NamedTuple, - Pattern, Optional, TYPE_CHECKING, Set, @@ -233,7 +233,6 @@ Unicode, Dict as DictTrait, Union as UnionTrait, - default, observe, ) from traitlets.config.configurable import Configurable @@ -559,7 +558,7 @@ class SimpleCompletion: __slots__ = ["text", "type"] - def __init__(self, text: str, *, type: str = None): + def __init__(self, text: str, *, type: Optional[str] = None): self.text = text self.type = type @@ -647,16 +646,18 @@ def line_with_cursor(self) -> str: class _MatcherAPIv1Base(Protocol): - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol): #: API version matcher_api_version: Optional[Literal[1]] - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... #: Protocol describing Matcher API v1. @@ -671,6 +672,7 @@ class MatcherAPIv2(Protocol): def __call__(self, context: CompletionContext) -> MatcherResult: """Call signature.""" + ... Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] @@ -912,10 +914,11 @@ class Completer(Configurable): help="""Activate greedy completion. .. deprecated:: 8.8 - Use :any:`evaluation` instead. + Use :any:`evaluation` and :any:`auto_close_dict_keys` instead. - As of IPython 8.8 proxy for ``evaluation = 'unsafe'`` when set to ``True``, - and for ``'forbidden'`` when set to ``False``. + Whent enabled in IPython 8.8+ activates following settings for compatibility: + - ``evaluation = 'unsafe'`` + - ``auto_close_dict_keys = True`` """, ).tag(config=True) @@ -957,6 +960,11 @@ class Completer(Configurable): "Includes completion of latex commands, unicode names, and expanding " "unicode characters back to latex commands.").tag(config=True) + auto_close_dict_keys = Bool( + False, + help="""Enable auto-closing dictionary keys.""" + ).tag(config=True) + def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -1119,8 +1127,80 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], prefix: str, delims: str, - extra_prefix: Optional[Tuple[Union[str, bytes], ...]]=None) -> Tuple[str, int, List[str]]: +class DictKeyState(enum.Flag): + """Represent state of the key match in context of other possible matches. + + - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. + - given `d2 = {('a', 'b'): 1}`: `d2['a', '` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`. + - given `d3 = {('a', 'b'): 1}`: `d3['` will yield `{'a': IN_TUPLE}` as `'a'` can be added. + - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}` + """ + BASELINE = 0 + END_OF_ITEM = enum.auto() + END_OF_TUPLE = enum.auto() + IN_TUPLE = enum.auto() + + +def _parse_tokens(c): + tokens = [] + token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) + while True: + try: + tokens.append(next(token_generator)) + except tokenize.TokenError: + return tokens + except StopIteration: + return tokens + + +def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: + """Match any valid Python numeric literal in a prefix of dictionary keys. + + References: + - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals + - https://docs.python.org/3/library/tokenize.html + """ + if prefix[-1].isspace(): + # if user typed a space we do not have anything to complete + # even if there was a valid number token before + return None + tokens = _parse_tokens(prefix) + rev_tokens = reversed(tokens) + skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE} + number = None + for token in rev_tokens: + if token.type in skip_over: + continue + if number is None: + if token.type == tokenize.NUMBER: + number = token.string + continue + else: + # we did not match a number + return None + if token.type == tokenize.OP: + if token.string == ',': + break + if token.string in {'+', '-'}: + number = token.string + number + else: + return None + return number + + +_INT_FORMATS = { + '0b': bin, + '0o': oct, + '0x': hex, +} + + +def match_dict_keys( + keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], + prefix: str, + delims: str, + extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None +) -> Tuple[str, int, Dict[str, DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1140,16 +1220,21 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]] A tuple of three elements: ``quote``, ``token_start``, ``matched``, with ``quote`` being the quote that need to be used to close current string. ``token_start`` the position where the replacement should start occurring, - ``matches`` a list of replacement/completion - + ``matches`` a dictionary of replacement/completion keys on keys and values + indicating whether the state. """ prefix_tuple = extra_prefix if extra_prefix else () - Nprefix = len(prefix_tuple) + prefix_tuple_size = sum([ + # for pandas, do not count slices as taking space + not isinstance(k, slice) + for k in prefix_tuple + ]) text_serializable_types = (str, bytes, int, float, slice) + def filter_prefix_tuple(key): # Reject too short keys - if len(key) <= Nprefix: + if len(key) <= prefix_tuple_size: return False # Reject keys which cannot be serialised to text for k in key: @@ -1162,28 +1247,58 @@ def filter_prefix_tuple(key): # All checks passed! return True - filtered_keys: List[Union[str, bytes, int, float, slice]] = [] - - def _add_to_filtered_keys(key): - if isinstance(key, text_serializable_types): - filtered_keys.append(key) + filtered_key_is_final: Dict[Union[str, bytes, int, float], DictKeyState] = defaultdict(lambda: DictKeyState.BASELINE) for k in keys: + # If at least one of the matches is not final, mark as undetermined. + # This can happen with `d = {111: 'b', (111, 222): 'a'}` where + # `111` appears final on first match but is not final on the second. + if isinstance(k, tuple): if filter_prefix_tuple(k): - _add_to_filtered_keys(k[Nprefix]) + key_fragment = k[prefix_tuple_size] + filtered_key_is_final[key_fragment] |= ( + DictKeyState.END_OF_TUPLE + if len(k) == prefix_tuple_size + 1 else + DictKeyState.IN_TUPLE + ) + elif prefix_tuple_size > 0: + # we are completing a tuple but this key is not a tuple, + # so we should ignore it + pass else: - _add_to_filtered_keys(k) + if isinstance(k, text_serializable_types): + filtered_key_is_final[k] |= DictKeyState.END_OF_ITEM + + filtered_keys = filtered_key_is_final.keys() if not prefix: - return '', 0, [repr(k) for k in filtered_keys] - quote_match = re.search('["\']', prefix) - assert quote_match is not None # silence mypy - quote = quote_match.group() - try: - prefix_str = literal_eval(prefix + quote) - except Exception: - return '', 0, [] + return '', 0, {repr(k): v for k, v in filtered_key_is_final.items()} + + quote_match = re.search('(?:"|\')', prefix) + is_user_prefix_numeric = False + + if quote_match: + quote = quote_match.group() + valid_prefix = prefix + quote + try: + prefix_str = literal_eval(valid_prefix) + except Exception: + return '', 0, {} + else: + # If it does not look like a string, let's assume + # we are dealing with a number or variable. + number_match = _match_number_in_dict_key_prefix(prefix) + + # We do not want the key matcher to suggest variable names so we yield: + if number_match is None: + # The alternative would be to assume that user forgort the quote + # and if the substring matches, suggest adding it at the start. + return '', 0, {} + + prefix_str = number_match + is_user_prefix_numeric = True + quote = '' pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) @@ -1191,13 +1306,29 @@ def _add_to_filtered_keys(key): token_start = token_match.start() token_prefix = token_match.group() - matched: List[str] = [] + matched: Dict[str, DictKeyState] = {} + for key in filtered_keys: - str_key = key if isinstance(key, (str, bytes)) else str(key) + if isinstance(key, (int, float)): + # User typed a number but this key is not a number. + if not is_user_prefix_numeric: + continue + str_key = str(key) + if isinstance(key, int): + int_base = prefix_str[:2].lower() + # if user typed integer using binary/oct/hex notation: + if int_base in _INT_FORMATS: + int_format = _INT_FORMATS[int_base] + str_key = int_format(key) + else: + # User typed a string but this key is a number. + if is_user_prefix_numeric: + continue + str_key = key try: if not str_key.startswith(prefix_str): continue - except (AttributeError, TypeError, UnicodeError): + except (AttributeError, TypeError, UnicodeError) as e: # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue @@ -1213,7 +1344,9 @@ def _add_to_filtered_keys(key): rem_repr = rem_repr.replace('"', '\\"') # then reinsert prefix from start of token - matched.append('%s%s' % (token_prefix, rem_repr)) + match = '%s%s' % (token_prefix, rem_repr) + + matched[match] = filtered_key_is_final[key] return quote, token_start, matched @@ -1447,24 +1580,39 @@ def _make_signature(completion)-> str: \s* # and optional whitespace # Capture any number of serializable objects (e.g. "a", "b", 'c') # and slices -((?:[uUbB]? # string prefix (r not handled) - (?: - '(?:[^']|(? str: def _convert_matcher_v1_result_to_v2( matches: Sequence[str], type: str, - fragment: str = None, + fragment: Optional[str] = None, suppress_if_matches: bool = False, ) -> SimpleMatcherResult: """Utility to help with transition""" @@ -1494,9 +1642,11 @@ def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" if change['new']: self.evaluation = 'unsafe' + self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: self.evaluation = 'limitted' + self.auto_close_dict_keys = False self.splitter.delims = DELIMS dict_keys_only = Bool( @@ -2294,7 +2444,7 @@ def dict_key_matches(self, text: str) -> List[str]: extra_prefix=tuple_prefix ) if not matches: - return matches + return [] # get the cursor position of # - the text being completed @@ -2313,26 +2463,55 @@ def dict_key_matches(self, text: str) -> List[str]: else: leading = text[text_start:completion_start] - # the index of the `[` character - bracket_idx = match.end(1) - # append closing quote and bracket as appropriate # this is *not* appropriate if the opening quote or bracket is outside - # the text given to this method - suf = '' - continuation = self.line_buffer[len(self.text_until_cursor):] - if key_start > text_start and closing_quote: - # quotes were opened inside text, maybe close them - if continuation.startswith(closing_quote): - continuation = continuation[len(closing_quote):] - else: - suf += closing_quote - if bracket_idx > text_start: - # brackets were opened inside text, maybe close them - if not continuation.startswith(']'): - suf += ']' + # the text given to this method, e.g. `d["""a\nt + can_close_quote = False + can_close_bracket = False + + continuation = self.line_buffer[len(self.text_until_cursor):].strip() + + if continuation.startswith(closing_quote): + # do not close if already closed, e.g. `d['a'` + continuation = continuation[len(closing_quote):] + else: + can_close_quote = True + + continuation = continuation.strip() + + # e.g. `pandas.DataFrame` has different tuple indexer behaviour, + # handling it is out of scope, so let's avoid appending suffixes. + has_known_tuple_handling = isinstance(obj, dict) - return [leading + k + suf for k in matches] + can_close_bracket = not continuation.startswith(']') and self.auto_close_dict_keys + can_close_tuple_item = not continuation.startswith(',') and has_known_tuple_handling and self.auto_close_dict_keys + can_close_quote = can_close_quote and self.auto_close_dict_keys + + # fast path if closing qoute should be appended but not suffix is allowed + if not can_close_quote and not can_close_bracket and closing_quote: + return [leading + k for k in matches] + + results = [] + + end_of_tuple_or_item = DictKeyState.END_OF_TUPLE | DictKeyState.END_OF_ITEM + + for k, state_flag in matches.items(): + result = leading + k + if can_close_quote and closing_quote: + result += closing_quote + + if state_flag == end_of_tuple_or_item: + # We do not know which suffix to add, + # e.g. both tuple item and string + # match this item. + pass + + if state_flag in end_of_tuple_or_item and can_close_bracket: + result += ']' + if state_flag == DictKeyState.IN_TUPLE and can_close_tuple_item: + result += ', ' + results.append(result) + return results @context_matcher() def unicode_name_matcher(self, context: CompletionContext): diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index f477c6bc2c1..d420ca80980 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -1,11 +1,19 @@ -from typing import Callable, Protocol, Set, Tuple, NamedTuple, Literal, Union +from typing import Callable, Set, Tuple, NamedTuple, Literal, Union, TYPE_CHECKING import collections import sys import ast -import types from functools import cached_property from dataclasses import dataclass, field +from IPython.utils.docs import GENERATING_DOCUMENTATION + + +if TYPE_CHECKING or GENERATING_DOCUMENTATION: + from typing_extensions import Protocol +else: + # do not require on runtime + Protocol = object # requires Python >=3.8 + class HasGetItem(Protocol): def __getitem__(self, key) -> None: ... @@ -266,20 +274,23 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): Currently does not support evaluation of functions with arguments. Does not evaluate actions which always have side effects: - - class definitions (`class sth: ...`) - - function definitions (`def sth: ...`) - - variable assignments (`x = 1`) - - augumented assignments (`x += 1`) - - deletions (`del x`) + - class definitions (``class sth: ...``) + - function definitions (``def sth: ...``) + - variable assignments (``x = 1``) + - augumented assignments (``x += 1``) + - deletions (``del x``) Does not evaluate operations which do not return values: - - assertions (`assert x`) - - pass (`pass`) - - imports (`import x`) + - assertions (``assert x``) + - pass (``pass``) + - imports (``import x``) - control flow - - conditionals (`if x:`) except for terenary IfExp (`a if x else b`) - - loops (`for` and `while`) + - conditionals (``if x:``) except for terenary IfExp (``a if x else b``) + - loops (``for`` and `while``) - exception handling + + The purpose of this function is to guard against unwanted side-effects; + it does not give guarantees on protection from malicious code execution. """ policy = EVALUATION_POLICIES[context.evaluation] if node is None: diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 7a99a2655ab..4d8eecec1f8 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -24,6 +24,7 @@ provisionalcompleter, match_dict_keys, _deduplicate_completions, + _match_number_in_dict_key_prefix, completion_matcher, SimpleCompletion, CompletionContext, @@ -181,7 +182,6 @@ def check_line_split(splitter, test_specs): out = splitter.split_line(line, cursor_pos) assert out == split - def test_line_split(): """Basic line splitter test with default specs.""" sp = completer.CompletionSplitter() @@ -852,16 +852,37 @@ def test_match_dict_keys(self): """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" + def match(*args, **kwargs): + quote, offset, matches = match_dict_keys(*args, **kwargs) + return quote, offset, list(matches) + keys = ["foo", b"far"] - assert match_dict_keys(keys, "b'", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, "b'f", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, 'b"', delims=delims) == ('"', 2, ["far"]) - assert match_dict_keys(keys, 'b"f', delims=delims) == ('"', 2, ["far"]) + assert match(keys, "b'", delims=delims) == ("'", 2, ["far"]) + assert match(keys, "b'f", delims=delims) == ("'", 2, ["far"]) + assert match(keys, 'b"', delims=delims) == ('"', 2, ["far"]) + assert match(keys, 'b"f', delims=delims) == ('"', 2, ["far"]) - assert match_dict_keys(keys, "'", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, "'f", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"]) - assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"]) + assert match(keys, "'", delims=delims) == ("'", 1, ["foo"]) + assert match(keys, "'f", delims=delims) == ("'", 1, ["foo"]) + assert match(keys, '"', delims=delims) == ('"', 1, ["foo"]) + assert match(keys, '"f', delims=delims) == ('"', 1, ["foo"]) + + # Completion on first item of tuple + keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, 'test')] + assert match(keys, "'f", delims=delims) == ("'", 1, ["foo"]) + assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) + + # Completion on numbers + keys = [ + 0xdeadbeef, # 3735928559 + 1111, 1234, "1999", + 0b10101, # 21 + 22 + ] + assert match(keys, "0xdead", delims=delims) == ("", 0, ["0xdeadbeef"]) + assert match(keys, "1", delims=delims) == ("", 0, ["1111", "1234"]) + assert match(keys, "2", delims=delims) == ("", 0, ["21", "22"]) + assert match(keys, "0b101", delims=delims) == ("", 0, ['0b10101', '0b10110']) def test_match_dict_keys_tuple(self): """ @@ -872,30 +893,85 @@ def test_match_dict_keys_tuple(self): keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')] + def match(*args, **kwargs): + quote, offset, matches = match_dict_keys(*args, **kwargs) + return quote, offset, list(matches) + # Completion on first key == "foo" - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"]) - assert match_dict_keys(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"]) - assert match_dict_keys(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) - assert match_dict_keys(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) + assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"]) + assert match(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"]) + assert match(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"]) + assert match(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"]) + assert match(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) + assert match(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) # No Completion - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) + assert match(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) + assert match(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')] - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2", "foo2"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + assert match(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2"]) + assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"]) + assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) + assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + + keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)] + assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["2222"]) + assert match(keys, "", delims=delims, extra_prefix=("foo",)) == ("", 0, ["1111", "'2222'"]) + assert match(keys, "'", delims=delims, extra_prefix=(3333,)) == ("'", 1, ["bar"]) + assert match(keys, "", delims=delims, extra_prefix=(3333,)) == ("", 0, ["'bar'", "4444"]) + assert match(keys, "'", delims=delims, extra_prefix=("3333",)) == ("'", 1, []) + assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) + + def test_dict_key_completion_closures(self): + ip = get_ipython() + complete = ip.Completer.complete + ip.Completer.auto_close_dict_keys = True - keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, 'test')] - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["1111", "2222"]) - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=(3333,)) == ("'", 1, ["bar", "test"]) - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("3333",)) == ("'", 1, []) + ip.user_ns["d"] = { + # tuple only + ('aa', 11): None, + # tuple and non-tuple + ('bb', 22): None, + 'bb': None, + # non-tuple only + 'cc': None, + # numeric tuple only + (77, 'x'): None, + # numeric tuple and non-tuple + (88, 'y'): None, + 88: None, + # numeric non-tuple only + 99: None, + } + + _, matches = complete(line_buffer="d[") + # should append `, ` if matches a tuple only + self.assertIn("'aa', ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("'bb'", matches) + # should append `]` if matches and item only + self.assertIn("'cc']", matches) + + # should append `, ` if matches a tuple only + self.assertIn("77, ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("88", matches) + # should append `]` if matches and item only + self.assertIn("99]", matches) + + _, matches = complete(line_buffer="d['aa', ") + # should restrict matches to those matching tuple prefix + self.assertIn("11]", matches) + self.assertNotIn("'bb'", matches) + self.assertNotIn("'bb', ", matches) + self.assertNotIn("'bb']", matches) + self.assertNotIn("'cc'", matches) + self.assertNotIn("'cc', ", matches) + self.assertNotIn("'cc']", matches) + ip.Completer.auto_close_dict_keys = False def test_dict_key_completion_string(self): """Test dictionary key completion for string keys""" @@ -1052,6 +1128,35 @@ def test_dict_key_completion_string(self): self.assertNotIn("foo", matches) self.assertNotIn("bar", matches) + def test_dict_key_completion_numbers(self): + ip = get_ipython() + complete = ip.Completer.complete + + ip.user_ns["d"] = { + 0xdeadbeef: None, # 3735928559 + 1111: None, + 1234: None, + "1999": None, + 0b10101: None, # 21 + 22: None + } + _, matches = complete(line_buffer="d[1") + self.assertIn("1111", matches) + self.assertIn("1234", matches) + self.assertNotIn("1999", matches) + self.assertNotIn("'1999'", matches) + + _, matches = complete(line_buffer="d[0xdead") + self.assertIn("0xdeadbeef", matches) + + _, matches = complete(line_buffer="d[2") + self.assertIn("21", matches) + self.assertIn("22", matches) + + _, matches = complete(line_buffer="d[0b101") + self.assertIn("0b10101", matches) + self.assertIn("0b10110", matches) + def test_dict_key_completion_contexts(self): """Test expression contexts in which dict key completion occurs""" ip = get_ipython() @@ -1545,3 +1650,35 @@ def _(expected): _(["completion_b"]) a_matcher.matcher_priority = 3 _(["completion_a"]) + + +@pytest.mark.parametrize( + 'input, expected', + [ + ['1.234', '1.234'], + # should match signed numbers + ['+1', '+1'], + ['-1', '-1'], + ['-1.0', '-1.0'], + ['-1.', '-1.'], + ['+1.', '+1.'], + ['.1', '.1'], + # should not match non-numbers + ['1..', None], + ['..', None], + ['.1.', None], + # should match after comma + [',1', '1'], + [', 1', '1'], + [', .1', '.1'], + [', +.1', '+.1'], + # should not match after trailing spaces + ['.1 ', None], + # some complex cases + ['0b_0011_1111_0100_1110', '0b_0011_1111_0100_1110'], + ['0xdeadbeef', '0xdeadbeef'], + ['0b_1110_0101', '0b_1110_0101'] + ] +) +def test_match_numeric_literal_for_dict_key(input, expected): + assert _match_number_in_dict_key_prefix(input) == expected From 21ae8802bd0d3b62a7e4e5fdc64648069616997d Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 00:41:40 +0000 Subject: [PATCH 032/122] You Want It Darker --- IPython/core/completer.py | 112 +++++++------ IPython/core/guarded_eval.py | 207 +++++++++++------------- IPython/core/tests/test_completer.py | 172 +++++++++++++------- IPython/core/tests/test_guarded_eval.py | 178 +++++++++----------- 4 files changed, 338 insertions(+), 331 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index e53e83b38e9..3ea3dd99258 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -923,8 +923,8 @@ class Completer(Configurable): ).tag(config=True) evaluation = Enum( - ('forbidden', 'minimal', 'limitted', 'unsafe', 'dangerous'), - default_value='limitted', + ("forbidden", "minimal", "limitted", "unsafe", "dangerous"), + default_value="limitted", help="""Code evaluation under completion. Successive options allow to enable more eager evaluation for more accurate completion suggestions, @@ -961,8 +961,7 @@ class Completer(Configurable): "unicode characters back to latex commands.").tag(config=True) auto_close_dict_keys = Bool( - False, - help="""Enable auto-closing dictionary keys.""" + False, help="""Enable auto-closing dictionary keys.""" ).tag(config=True) def __init__(self, namespace=None, global_namespace=None, **kwargs): @@ -1066,7 +1065,7 @@ def attr_matches(self, text): m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) if not m2: return [] - expr, attr = m2.group(1,2) + expr, attr = m2.group(1, 2) obj = self._evaluate_expr(expr) @@ -1090,8 +1089,7 @@ def attr_matches(self, text): pass # Build match list to return n = len(attr) - return ["%s.%s" % (expr, w) for w in words if w[:n] == attr ] - + return ["%s.%s" % (expr, w) for w in words if w[:n] == attr] def _evaluate_expr(self, expr): obj = not_found @@ -1103,13 +1101,13 @@ def _evaluate_expr(self, expr): EvaluationContext( globals_=self.global_namespace, locals_=self.namespace, - evaluation=self.evaluation - ) + evaluation=self.evaluation, + ), ) done = True except Exception as e: if self.debug: - print('Evaluation exception', e) + print("Evaluation exception", e) # trim the expression to remove any invalid prefix # e.g. user starts `(d[`, so we get `expr = '(d'`, # where parenthesis is not closed. @@ -1135,6 +1133,7 @@ class DictKeyState(enum.Flag): - given `d3 = {('a', 'b'): 1}`: `d3['` will yield `{'a': IN_TUPLE}` as `'a'` can be added. - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}` """ + BASELINE = 0 END_OF_ITEM = enum.auto() END_OF_TUPLE = enum.auto() @@ -1179,9 +1178,9 @@ def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: # we did not match a number return None if token.type == tokenize.OP: - if token.string == ',': + if token.string == ",": break - if token.string in {'+', '-'}: + if token.string in {"+", "-"}: number = token.string + number else: return None @@ -1189,9 +1188,9 @@ def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: _INT_FORMATS = { - '0b': bin, - '0o': oct, - '0x': hex, + "0b": bin, + "0o": oct, + "0x": hex, } @@ -1199,7 +1198,7 @@ def match_dict_keys( keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], prefix: str, delims: str, - extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None + extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, ) -> Tuple[str, int, Dict[str, DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys @@ -1225,11 +1224,13 @@ def match_dict_keys( """ prefix_tuple = extra_prefix if extra_prefix else () - prefix_tuple_size = sum([ - # for pandas, do not count slices as taking space - not isinstance(k, slice) - for k in prefix_tuple - ]) + prefix_tuple_size = sum( + [ + # for pandas, do not count slices as taking space + not isinstance(k, slice) + for k in prefix_tuple + ] + ) text_serializable_types = (str, bytes, int, float, slice) def filter_prefix_tuple(key): @@ -1247,7 +1248,9 @@ def filter_prefix_tuple(key): # All checks passed! return True - filtered_key_is_final: Dict[Union[str, bytes, int, float], DictKeyState] = defaultdict(lambda: DictKeyState.BASELINE) + filtered_key_is_final: Dict[ + Union[str, bytes, int, float], DictKeyState + ] = defaultdict(lambda: DictKeyState.BASELINE) for k in keys: # If at least one of the matches is not final, mark as undetermined. @@ -1259,8 +1262,8 @@ def filter_prefix_tuple(key): key_fragment = k[prefix_tuple_size] filtered_key_is_final[key_fragment] |= ( DictKeyState.END_OF_TUPLE - if len(k) == prefix_tuple_size + 1 else - DictKeyState.IN_TUPLE + if len(k) == prefix_tuple_size + 1 + else DictKeyState.IN_TUPLE ) elif prefix_tuple_size > 0: # we are completing a tuple but this key is not a tuple, @@ -1273,9 +1276,9 @@ def filter_prefix_tuple(key): filtered_keys = filtered_key_is_final.keys() if not prefix: - return '', 0, {repr(k): v for k, v in filtered_key_is_final.items()} + return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()} - quote_match = re.search('(?:"|\')', prefix) + quote_match = re.search("(?:\"|')", prefix) is_user_prefix_numeric = False if quote_match: @@ -1284,7 +1287,7 @@ def filter_prefix_tuple(key): try: prefix_str = literal_eval(valid_prefix) except Exception: - return '', 0, {} + return "", 0, {} else: # If it does not look like a string, let's assume # we are dealing with a number or variable. @@ -1294,11 +1297,11 @@ def filter_prefix_tuple(key): if number_match is None: # The alternative would be to assume that user forgort the quote # and if the substring matches, suggest adding it at the start. - return '', 0, {} + return "", 0, {} prefix_str = number_match is_user_prefix_numeric = True - quote = '' + quote = "" pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) @@ -1333,7 +1336,7 @@ def filter_prefix_tuple(key): continue # reformat remainder of key to begin with prefix - rem = str_key[len(prefix_str):] + rem = str_key[len(prefix_str) :] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') rem_repr = rem_repr[1 + rem_repr.index("'"):-2] @@ -1344,7 +1347,7 @@ def filter_prefix_tuple(key): rem_repr = rem_repr.replace('"', '\\"') # then reinsert prefix from start of token - match = '%s%s' % (token_prefix, rem_repr) + match = "%s%s" % (token_prefix, rem_repr) matched[match] = filtered_key_is_final[key] return quote, token_start, matched @@ -1572,7 +1575,8 @@ def _make_signature(completion)-> str: _CompleteResult = Dict[str, MatcherResult] -DICT_MATCHER_REGEX = re.compile(r"""(?x) +DICT_MATCHER_REGEX = re.compile( + r"""(?x) ( # match dict-referring - or any get item object - expression .+ ) @@ -1616,7 +1620,9 @@ def _make_signature(completion)-> str: ) )? $ -""") +""" +) + def _convert_matcher_v1_result_to_v2( matches: Sequence[str], @@ -1640,12 +1646,12 @@ class IPCompleter(Completer): @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" - if change['new']: - self.evaluation = 'unsafe' + if change["new"]: + self.evaluation = "unsafe" self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: - self.evaluation = 'limitted' + self.evaluation = "limitted" self.auto_close_dict_keys = False self.splitter.delims = DELIMS @@ -2375,13 +2381,12 @@ def _get_keys(obj: Any) -> List[Any]: return method() # Special case some common in-memory dict-like types - if (isinstance(obj, dict) or - _safe_isinstance(obj, 'pandas', 'DataFrame')): + if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"): try: return list(obj.keys()) except Exception: return [] - elif _safe_isinstance(obj, 'pandas', 'core', 'indexing', '_LocIndexer'): + elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"): try: return list(obj.obj.keys()) except Exception: @@ -2408,7 +2413,7 @@ def dict_key_matches(self, text: str) -> List[str]: # Short-circuit on closed dictionary (regular expression would # not match anyway, but would take quite a while). - if self.text_until_cursor.strip().endswith(']'): + if self.text_until_cursor.strip().endswith("]"): return [] match = DICT_MATCHER_REGEX.search(self.text_until_cursor) @@ -2433,15 +2438,12 @@ def dict_key_matches(self, text: str) -> List[str]: globals_=self.global_namespace, locals_=self.namespace, evaluation=self.evaluation, - in_subscript=True - ) + in_subscript=True, + ), ) closing_quote, token_offset, matches = match_dict_keys( - keys, - key_prefix, - self.splitter.delims, - extra_prefix=tuple_prefix + keys, key_prefix, self.splitter.delims, extra_prefix=tuple_prefix ) if not matches: return [] @@ -2469,11 +2471,11 @@ def dict_key_matches(self, text: str) -> List[str]: can_close_quote = False can_close_bracket = False - continuation = self.line_buffer[len(self.text_until_cursor):].strip() + continuation = self.line_buffer[len(self.text_until_cursor) :].strip() if continuation.startswith(closing_quote): # do not close if already closed, e.g. `d['a'` - continuation = continuation[len(closing_quote):] + continuation = continuation[len(closing_quote) :] else: can_close_quote = True @@ -2483,8 +2485,14 @@ def dict_key_matches(self, text: str) -> List[str]: # handling it is out of scope, so let's avoid appending suffixes. has_known_tuple_handling = isinstance(obj, dict) - can_close_bracket = not continuation.startswith(']') and self.auto_close_dict_keys - can_close_tuple_item = not continuation.startswith(',') and has_known_tuple_handling and self.auto_close_dict_keys + can_close_bracket = ( + not continuation.startswith("]") and self.auto_close_dict_keys + ) + can_close_tuple_item = ( + not continuation.startswith(",") + and has_known_tuple_handling + and self.auto_close_dict_keys + ) can_close_quote = can_close_quote and self.auto_close_dict_keys # fast path if closing qoute should be appended but not suffix is allowed @@ -2507,9 +2515,9 @@ def dict_key_matches(self, text: str) -> List[str]: pass if state_flag in end_of_tuple_or_item and can_close_bracket: - result += ']' + result += "]" if state_flag == DictKeyState.IN_TUPLE and can_close_tuple_item: - result += ', ' + result += ", " results.append(result) return results diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index d420ca80980..2c278a238ae 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -16,20 +16,24 @@ class HasGetItem(Protocol): - def __getitem__(self, key) -> None: ... + def __getitem__(self, key) -> None: + ... class InstancesHaveGetItem(Protocol): - def __call__(self) -> HasGetItem: ... + def __call__(self) -> HasGetItem: + ... class HasGetAttr(Protocol): - def __getattr__(self, key) -> None: ... + def __getattr__(self, key) -> None: + ... class DoesNotHaveGetAttr(Protocol): pass + # By default `__getattr__` is not explicitly implemented on most objects MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] @@ -38,22 +42,16 @@ def unbind_method(func: Callable) -> Union[Callable, None]: """Get unbound method for given bound method. Returns None if cannot get unbound method.""" - owner = getattr(func, '__self__', None) + owner = getattr(func, "__self__", None) owner_class = type(owner) - name = getattr(func, '__name__', None) - instance_dict_overrides = getattr(owner, '__dict__', None) + name = getattr(func, "__name__", None) + instance_dict_overrides = getattr(owner, "__dict__", None) if ( owner is not None - and - name - and - ( + and name + and ( not instance_dict_overrides - or - ( - instance_dict_overrides - and name not in instance_dict_overrides - ) + or (instance_dict_overrides and name not in instance_dict_overrides) ) ): return getattr(owner_class, name) @@ -86,7 +84,13 @@ def can_call(self, func): if owner_method and owner_method in self.allowed_calls: return True -def has_original_dunder_external(value, module_name, access_path, method_name,): + +def has_original_dunder_external( + value, + module_name, + access_path, + method_name, +): try: if module_name not in sys.modules: return False @@ -106,11 +110,7 @@ def has_original_dunder_external(value, module_name, access_path, method_name,): def has_original_dunder( - value, - allowed_types, - allowed_methods, - allowed_external, - method_name + value, allowed_types, allowed_methods, allowed_external, method_name ): # note: Python ignores `__getattr__`/`__getitem__` on instances, # we only need to check at class level @@ -148,14 +148,14 @@ def can_get_attr(self, value, attr): allowed_types=self.allowed_getattr, allowed_methods=self._getattribute_methods, allowed_external=self.allowed_getattr_external, - method_name='__getattribute__' + method_name="__getattribute__", ) has_original_attr = has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattr_methods, allowed_external=self.allowed_getattr_external, - method_name='__getattr__' + method_name="__getattr__", ) # Many objects do not have `__getattr__`, this is fine if has_original_attr is None and has_original_attribute: @@ -168,7 +168,6 @@ def get_attr(self, value, attr): if self.can_get_attr(value, attr): return getattr(value, attr) - def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" return has_original_dunder( @@ -176,29 +175,20 @@ def can_get_item(self, value, item): allowed_types=self.allowed_getitem, allowed_methods=self._getitem_methods, allowed_external=self.allowed_getitem_external, - method_name='__getitem__' + method_name="__getitem__", ) @cached_property def _getitem_methods(self) -> Set[Callable]: - return self._safe_get_methods( - self.allowed_getitem, - '__getitem__' - ) + return self._safe_get_methods(self.allowed_getitem, "__getitem__") @cached_property def _getattr_methods(self) -> Set[Callable]: - return self._safe_get_methods( - self.allowed_getattr, - '__getattr__' - ) + return self._safe_get_methods(self.allowed_getattr, "__getattr__") @cached_property def _getattribute_methods(self) -> Set[Callable]: - return self._safe_get_methods( - self.allowed_getattr, - '__getattribute__' - ) + return self._safe_get_methods(self.allowed_getattr, "__getattribute__") def _safe_get_methods(self, classes, name) -> Set[Callable]: return { @@ -216,7 +206,9 @@ class DummyNamedTuple(NamedTuple): class EvaluationContext(NamedTuple): locals_: dict globals_: dict - evaluation: Literal['forbidden', 'minimal', 'limitted', 'unsafe', 'dangerous'] = 'forbidden' + evaluation: Literal[ + "forbidden", "minimal", "limitted", "unsafe", "dangerous" + ] = "forbidden" in_subscript: bool = False @@ -224,21 +216,20 @@ class IdentitySubscript: def __getitem__(self, key): return key + IDENTITY_SUBSCRIPT = IdentitySubscript() -SUBSCRIPT_MARKER = '__SUBSCRIPT_SENTINEL__' +SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" + class GuardRejection(ValueError): pass -def guarded_eval( - code: str, - context: EvaluationContext -): +def guarded_eval(code: str, context: EvaluationContext): locals_ = context.locals_ - if context.evaluation == 'forbidden': - raise GuardRejection('Forbidden mode') + if context.evaluation == "forbidden": + raise GuardRejection("Forbidden mode") # note: not using `ast.literal_eval` as it does not implement # getitem at all, for example it fails on simple `[0][1]` @@ -252,19 +243,17 @@ def guarded_eval( return tuple() locals_ = locals_.copy() locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT - code = SUBSCRIPT_MARKER + '[' + code + ']' - context = EvaluationContext(**{ - **context._asdict(), - **{'locals_': locals_} - }) + code = SUBSCRIPT_MARKER + "[" + code + "]" + context = EvaluationContext(**{**context._asdict(), **{"locals_": locals_}}) - if context.evaluation == 'dangerous': + if context.evaluation == "dangerous": return eval(code, context.globals_, context.locals_) - expression = ast.parse(code, mode='eval') + expression = ast.parse(code, mode="eval") return eval_node(expression, context) + def eval_node(node: Union[ast.AST, None], context: EvaluationContext): """ Evaluate AST node in provided context. @@ -314,7 +303,7 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if isinstance(node.op, ast.Mod): return left % right if isinstance(node.op, ast.Pow): - return left ** right + return left**right if isinstance(node.op, ast.LShift): return left << right if isinstance(node.op, ast.RShift): @@ -332,36 +321,26 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if isinstance(node, ast.Index): return eval_node(node.value, context) if isinstance(node, ast.Tuple): - return tuple( - eval_node(e, context) - for e in node.elts - ) + return tuple(eval_node(e, context) for e in node.elts) if isinstance(node, ast.List): - return [ - eval_node(e, context) - for e in node.elts - ] + return [eval_node(e, context) for e in node.elts] if isinstance(node, ast.Set): - return { - eval_node(e, context) - for e in node.elts - } + return {eval_node(e, context) for e in node.elts} if isinstance(node, ast.Dict): - return dict(zip( - [eval_node(k, context) for k in node.keys], - [eval_node(v, context) for v in node.values] - )) + return dict( + zip( + [eval_node(k, context) for k in node.keys], + [eval_node(v, context) for v in node.values], + ) + ) if isinstance(node, ast.Slice): return slice( eval_node(node.lower, context), eval_node(node.upper, context), - eval_node(node.step, context) + eval_node(node.step, context), ) if isinstance(node, ast.ExtSlice): - return tuple([ - eval_node(dim, context) - for dim in node.dims - ]) + return tuple([eval_node(dim, context) for dim in node.dims]) if isinstance(node, ast.UnaryOp): # TODO: add guards value = eval_node(node.operand, context) @@ -373,16 +352,16 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): return ~value if isinstance(node.op, ast.Not): return not value - raise ValueError('Unhandled unary operation:', node.op) + raise ValueError("Unhandled unary operation:", node.op) if isinstance(node, ast.Subscript): value = eval_node(node.value, context) slice_ = eval_node(node.slice, context) if policy.can_get_item(value, slice_): return value[slice_] raise GuardRejection( - 'Subscript access (`__getitem__`) for', - type(value), # not joined to avoid calling `repr` - f' not allowed in {context.evaluation} mode' + "Subscript access (`__getitem__`) for", + type(value), # not joined to avoid calling `repr` + f" not allowed in {context.evaluation} mode", ) if isinstance(node, ast.Name): if policy.allow_locals_access and node.id in context.locals_: @@ -393,49 +372,46 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): return __builtins__[node.id] if not policy.allow_globals_access and not policy.allow_locals_access: raise GuardRejection( - f'Namespace access not allowed in {context.evaluation} mode' + f"Namespace access not allowed in {context.evaluation} mode" ) else: - raise NameError(f'{node.id} not found in locals nor globals') + raise NameError(f"{node.id} not found in locals nor globals") if isinstance(node, ast.Attribute): value = eval_node(node.value, context) if policy.can_get_attr(value, node.attr): return getattr(value, node.attr) raise GuardRejection( - 'Attribute access (`__getattr__`) for', - type(value), # not joined to avoid calling `repr` - f'not allowed in {context.evaluation} mode' + "Attribute access (`__getattr__`) for", + type(value), # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", ) if isinstance(node, ast.IfExp): test = eval_node(node.test, context) if test: - return eval_node(node.body, context) + return eval_node(node.body, context) else: return eval_node(node.orelse, context) if isinstance(node, ast.Call): func = eval_node(node.func, context) print(node.keywords) if policy.can_call(func) and not node.keywords: - args = [ - eval_node(arg, context) - for arg in node.args - ] + args = [eval_node(arg, context) for arg in node.args] return func(*args) raise GuardRejection( - 'Call for', - func, # not joined to avoid calling `repr` - f'not allowed in {context.evaluation} mode' + "Call for", + func, # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", ) - raise ValueError('Unhandled node', node) + raise ValueError("Unhandled node", node) SUPPORTED_EXTERNAL_GETITEM = { - ('pandas', 'core', 'indexing', '_iLocIndexer'), - ('pandas', 'core', 'indexing', '_LocIndexer'), - ('pandas', 'DataFrame'), - ('pandas', 'Series'), - ('numpy', 'ndarray'), - ('numpy', 'void') + ("pandas", "core", "indexing", "_iLocIndexer"), + ("pandas", "core", "indexing", "_LocIndexer"), + ("pandas", "DataFrame"), + ("pandas", "Series"), + ("numpy", "ndarray"), + ("numpy", "void"), } BUILTIN_GETITEM = { @@ -452,20 +428,17 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): collections.UserList, collections.UserString, DummyNamedTuple, - IdentitySubscript + IdentitySubscript, } def _list_methods(cls, source=None): """For use on immutable objects or with methods returning a copy""" - return [ - getattr(cls, k) - for k in (source if source else dir(cls)) - ] + return [getattr(cls, k) for k in (source if source else dir(cls))] -dict_non_mutating_methods = ('copy', 'keys', 'values', 'items') -list_non_mutating_methods = ('copy', 'index', 'count') +dict_non_mutating_methods = ("copy", "keys", "values", "items") +list_non_mutating_methods = ("copy", "index", "count") set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) @@ -504,20 +477,20 @@ def _list_methods(cls, source=None): collections.Counter, *_list_methods(collections.Counter, dict_non_mutating_methods), collections.Counter.elements, - collections.Counter.most_common + collections.Counter.most_common, } EVALUATION_POLICIES = { - 'minimal': EvaluationPolicy( + "minimal": EvaluationPolicy( allow_builtins_access=True, allow_locals_access=False, allow_globals_access=False, allow_item_access=False, allow_attr_access=False, allowed_calls=set(), - allow_any_calls=False + allow_any_calls=False, ), - 'limitted': SelectivePolicy( + "limitted": SelectivePolicy( # TODO: # - should reject binary and unary operations if custom methods would be dispatched allowed_getitem=BUILTIN_GETITEM, @@ -529,24 +502,24 @@ def _list_methods(cls, source=None): object, type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. dict_keys, - method_descriptor + method_descriptor, }, allowed_getattr_external={ # pandas Series/Frame implements custom `__getattr__` - ('pandas', 'DataFrame'), - ('pandas', 'Series') + ("pandas", "DataFrame"), + ("pandas", "Series"), }, allow_builtins_access=True, allow_locals_access=True, allow_globals_access=True, - allowed_calls=ALLOWED_CALLS + allowed_calls=ALLOWED_CALLS, ), - 'unsafe': EvaluationPolicy( + "unsafe": EvaluationPolicy( allow_builtins_access=True, allow_locals_access=True, allow_globals_access=True, allow_attr_access=True, allow_item_access=True, - allow_any_calls=True - ) -} \ No newline at end of file + allow_any_calls=True, + ), +} diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 4d8eecec1f8..4e385d546a0 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -868,21 +868,16 @@ def match(*args, **kwargs): assert match(keys, '"f', delims=delims) == ('"', 1, ["foo"]) # Completion on first item of tuple - keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, 'test')] + keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")] assert match(keys, "'f", delims=delims) == ("'", 1, ["foo"]) assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) # Completion on numbers - keys = [ - 0xdeadbeef, # 3735928559 - 1111, 1234, "1999", - 0b10101, # 21 - 22 - ] + keys = [0xDEADBEEF, 1111, 1234, "1999", 0b10101, 22] # 3735928559 # 21 assert match(keys, "0xdead", delims=delims) == ("", 0, ["0xdeadbeef"]) assert match(keys, "1", delims=delims) == ("", 0, ["1111", "1234"]) assert match(keys, "2", delims=delims) == ("", 0, ["21", "22"]) - assert match(keys, "0b101", delims=delims) == ("", 0, ['0b10101', '0b10110']) + assert match(keys, "0b101", delims=delims) == ("", 0, ["0b10101", "0b10110"]) def test_match_dict_keys_tuple(self): """ @@ -898,30 +893,90 @@ def match(*args, **kwargs): return quote, offset, list(matches) # Completion on first key == "foo" - assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"]) - assert match(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"]) - assert match(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"]) - assert match(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"]) - assert match(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) - assert match(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) + assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ( + "'", + 1, + ["bar", "oof"], + ) + assert match(keys, '"', delims=delims, extra_prefix=("foo",)) == ( + '"', + 1, + ["bar", "oof"], + ) + assert match(keys, "'o", delims=delims, extra_prefix=("foo",)) == ( + "'", + 1, + ["oof"], + ) + assert match(keys, '"o', delims=delims, extra_prefix=("foo",)) == ( + '"', + 1, + ["oof"], + ) + assert match(keys, "b'", delims=delims, extra_prefix=("foo",)) == ( + "'", + 2, + ["bar"], + ) + assert match(keys, 'b"', delims=delims, extra_prefix=("foo",)) == ( + '"', + 2, + ["bar"], + ) + assert match(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ( + "'", + 2, + ["bar"], + ) + assert match(keys, 'b"b', delims=delims, extra_prefix=("foo",)) == ( + '"', + 2, + ["bar"], + ) # No Completion assert match(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) assert match(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) - keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')] - assert match(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2"]) - assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"]) - assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) - assert match(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")] + assert match(keys, "'foo", delims=delims, extra_prefix=("foo1",)) == ( + "'", + 1, + ["foo2"], + ) + assert match(keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2")) == ( + "'", + 1, + ["foo3"], + ) + assert match( + keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2", "foo3") + ) == ("'", 1, ["foo4"]) + assert match( + keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2", "foo3", "foo4") + ) == ("'", 1, []) keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)] - assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["2222"]) - assert match(keys, "", delims=delims, extra_prefix=("foo",)) == ("", 0, ["1111", "'2222'"]) - assert match(keys, "'", delims=delims, extra_prefix=(3333,)) == ("'", 1, ["bar"]) - assert match(keys, "", delims=delims, extra_prefix=(3333,)) == ("", 0, ["'bar'", "4444"]) + assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ( + "'", + 1, + ["2222"], + ) + assert match(keys, "", delims=delims, extra_prefix=("foo",)) == ( + "", + 0, + ["1111", "'2222'"], + ) + assert match(keys, "'", delims=delims, extra_prefix=(3333,)) == ( + "'", + 1, + ["bar"], + ) + assert match(keys, "", delims=delims, extra_prefix=(3333,)) == ( + "", + 0, + ["'bar'", "4444"], + ) assert match(keys, "'", delims=delims, extra_prefix=("3333",)) == ("'", 1, []) assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) @@ -932,16 +987,16 @@ def test_dict_key_completion_closures(self): ip.user_ns["d"] = { # tuple only - ('aa', 11): None, + ("aa", 11): None, # tuple and non-tuple - ('bb', 22): None, - 'bb': None, + ("bb", 22): None, + "bb": None, # non-tuple only - 'cc': None, + "cc": None, # numeric tuple only - (77, 'x'): None, + (77, "x"): None, # numeric tuple and non-tuple - (88, 'y'): None, + (88, "y"): None, 88: None, # numeric non-tuple only 99: None, @@ -1133,12 +1188,12 @@ def test_dict_key_completion_numbers(self): complete = ip.Completer.complete ip.user_ns["d"] = { - 0xdeadbeef: None, # 3735928559 + 0xDEADBEEF: None, # 3735928559 1111: None, 1234: None, "1999": None, - 0b10101: None, # 21 - 22: None + 0b10101: None, # 21 + 22: None, } _, matches = complete(line_buffer="d[1") self.assertIn("1111", matches) @@ -1169,7 +1224,7 @@ class C: ip.user_ns["C"] = C ip.user_ns["get"] = lambda: d - ip.user_ns["nested"] = {'x': d} + ip.user_ns["nested"] = {"x": d} def assert_no_completion(**kwargs): _, matches = complete(**kwargs) @@ -1198,7 +1253,7 @@ def assert_completion(**kwargs): # nested dict completion assert_completion(line_buffer="nested['x'][") - with evaluation_level('minimal'): + with evaluation_level("minimal"): with pytest.raises(AssertionError): assert_completion(line_buffer="nested['x'][") @@ -1289,6 +1344,7 @@ def test_struct_array_key_completion(self): _, matches = complete(line_buffer="d['") self.assertIn("my_head", matches) self.assertIn("my_data", matches) + def completes_on_nested(): ip.user_ns["d"] = numpy.zeros(2, dtype=dt) _, matches = complete(line_buffer="d[1]['my_head']['") @@ -1298,10 +1354,10 @@ def completes_on_nested(): with greedy_completion(): completes_on_nested() - with evaluation_level('limitted'): + with evaluation_level("limitted"): completes_on_nested() - with evaluation_level('minimal'): + with evaluation_level("minimal"): with pytest.raises(AssertionError): completes_on_nested() @@ -1653,32 +1709,32 @@ def _(expected): @pytest.mark.parametrize( - 'input, expected', + "input, expected", [ - ['1.234', '1.234'], + ["1.234", "1.234"], # should match signed numbers - ['+1', '+1'], - ['-1', '-1'], - ['-1.0', '-1.0'], - ['-1.', '-1.'], - ['+1.', '+1.'], - ['.1', '.1'], + ["+1", "+1"], + ["-1", "-1"], + ["-1.0", "-1.0"], + ["-1.", "-1."], + ["+1.", "+1."], + [".1", ".1"], # should not match non-numbers - ['1..', None], - ['..', None], - ['.1.', None], + ["1..", None], + ["..", None], + [".1.", None], # should match after comma - [',1', '1'], - [', 1', '1'], - [', .1', '.1'], - [', +.1', '+.1'], + [",1", "1"], + [", 1", "1"], + [", .1", ".1"], + [", +.1", "+.1"], # should not match after trailing spaces - ['.1 ', None], + [".1 ", None], # some complex cases - ['0b_0011_1111_0100_1110', '0b_0011_1111_0100_1110'], - ['0xdeadbeef', '0xdeadbeef'], - ['0b_1110_0101', '0b_1110_0101'] - ] + ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"], + ["0xdeadbeef", "0xdeadbeef"], + ["0b_1110_0101", "0b_1110_0101"], + ], ) def test_match_numeric_literal_for_dict_key(input, expected): assert _match_number_in_dict_key_prefix(input) == expected diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 5c89a68f637..129112ff4db 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -1,53 +1,54 @@ from typing import NamedTuple -from IPython.core.guarded_eval import EvaluationContext, GuardRejection, guarded_eval, unbind_method +from IPython.core.guarded_eval import ( + EvaluationContext, + GuardRejection, + guarded_eval, + unbind_method, +) from IPython.testing import decorators as dec import pytest def limitted(**kwargs): - return EvaluationContext( - locals_=kwargs, - globals_={}, - evaluation='limitted' - ) + return EvaluationContext(locals_=kwargs, globals_={}, evaluation="limitted") def unsafe(**kwargs): - return EvaluationContext( - locals_=kwargs, - globals_={}, - evaluation='unsafe' - ) + return EvaluationContext(locals_=kwargs, globals_={}, evaluation="unsafe") + -@dec.skip_without('pandas') +@dec.skip_without("pandas") def test_pandas_series_iloc(): import pandas as pd - series = pd.Series([1], index=['a']) + + series = pd.Series([1], index=["a"]) context = limitted(data=series) - assert guarded_eval('data.iloc[0]', context) == 1 + assert guarded_eval("data.iloc[0]", context) == 1 -@dec.skip_without('pandas') +@dec.skip_without("pandas") def test_pandas_series(): import pandas as pd - context = limitted(data=pd.Series([1], index=['a'])) + + context = limitted(data=pd.Series([1], index=["a"])) assert guarded_eval('data["a"]', context) == 1 with pytest.raises(KeyError): guarded_eval('data["c"]', context) -@dec.skip_without('pandas') +@dec.skip_without("pandas") def test_pandas_bad_series(): import pandas as pd + class BadItemSeries(pd.Series): def __getitem__(self, key): - return 'CUSTOM_ITEM' + return "CUSTOM_ITEM" class BadAttrSeries(pd.Series): def __getattr__(self, key): - return 'CUSTOM_ATTR' + return "CUSTOM_ATTR" - bad_series = BadItemSeries([1], index=['a']) + bad_series = BadItemSeries([1], index=["a"]) context = limitted(data=bad_series) with pytest.raises(GuardRejection): @@ -58,121 +59,108 @@ def __getattr__(self, key): # note: here result is a bit unexpected because # pandas `__getattr__` calls `__getitem__`; # FIXME - special case to handle it? - assert guarded_eval('data.a', context) == 'CUSTOM_ITEM' + assert guarded_eval("data.a", context) == "CUSTOM_ITEM" context = unsafe(data=bad_series) - assert guarded_eval('data["a"]', context) == 'CUSTOM_ITEM' + assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM" - bad_attr_series = BadAttrSeries([1], index=['a']) + bad_attr_series = BadAttrSeries([1], index=["a"]) context = limitted(data=bad_attr_series) assert guarded_eval('data["a"]', context) == 1 with pytest.raises(GuardRejection): - guarded_eval('data.a', context) + guarded_eval("data.a", context) -@dec.skip_without('pandas') +@dec.skip_without("pandas") def test_pandas_dataframe_loc(): import pandas as pd from pandas.testing import assert_series_equal - data = pd.DataFrame([{'a': 1}]) + + data = pd.DataFrame([{"a": 1}]) context = limitted(data=data) - assert_series_equal( - guarded_eval('data.loc[:, "a"]', context), - data['a'] - ) + assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"]) def test_named_tuple(): - class GoodNamedTuple(NamedTuple): a: str pass class BadNamedTuple(NamedTuple): a: str + def __getitem__(self, key): return None - good = GoodNamedTuple(a='x') - bad = BadNamedTuple(a='x') + good = GoodNamedTuple(a="x") + bad = BadNamedTuple(a="x") context = limitted(data=good) - assert guarded_eval('data[0]', context) == 'x' + assert guarded_eval("data[0]", context) == "x" context = limitted(data=bad) with pytest.raises(GuardRejection): - guarded_eval('data[0]', context) + guarded_eval("data[0]", context) def test_dict(): - context = limitted( - data={'a': 1, 'b': {'x': 2}, ('x', 'y'): 3} - ) + context = limitted(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) assert guarded_eval('data["a"]', context) == 1 - assert guarded_eval('data["b"]', context) == {'x': 2} + assert guarded_eval('data["b"]', context) == {"x": 2} assert guarded_eval('data["b"]["x"]', context) == 2 assert guarded_eval('data["x", "y"]', context) == 3 - assert guarded_eval('data.keys', context) + assert guarded_eval("data.keys", context) def test_set(): - context = limitted(data={'a', 'b'}) - assert guarded_eval('data.difference', context) + context = limitted(data={"a", "b"}) + assert guarded_eval("data.difference", context) def test_list(): context = limitted(data=[1, 2, 3]) - assert guarded_eval('data[1]', context) == 2 - assert guarded_eval('data.copy', context) + assert guarded_eval("data[1]", context) == 2 + assert guarded_eval("data.copy", context) def test_dict_literal(): context = limitted() - assert guarded_eval('{}', context) == {} + assert guarded_eval("{}", context) == {} assert guarded_eval('{"a": 1}', context) == {"a": 1} def test_list_literal(): context = limitted() - assert guarded_eval('[]', context) == [] + assert guarded_eval("[]", context) == [] assert guarded_eval('[1, "a"]', context) == [1, "a"] def test_set_literal(): context = limitted() - assert guarded_eval('set()', context) == set() + assert guarded_eval("set()", context) == set() assert guarded_eval('{"a"}', context) == {"a"} def test_if_expression(): context = limitted() - assert guarded_eval('2 if True else 3', context) == 2 - assert guarded_eval('4 if False else 5', context) == 5 + assert guarded_eval("2 if True else 3", context) == 2 + assert guarded_eval("4 if False else 5", context) == 5 def test_object(): obj = object() context = limitted(obj=obj) - assert guarded_eval('obj.__dir__', context) == obj.__dir__ + assert guarded_eval("obj.__dir__", context) == obj.__dir__ @pytest.mark.parametrize( "code,expected", [ - [ - 'int.numerator', - int.numerator - ], - [ - 'float.is_integer', - float.is_integer - ], - [ - 'complex.real', - complex.real - ] - ] + ["int.numerator", int.numerator], + ["float.is_integer", float.is_integer], + ["complex.real", complex.real], + ], ) def test_number_attributes(code, expected): assert guarded_eval(code, limitted()) == expected @@ -180,25 +168,15 @@ def test_number_attributes(code, expected): def test_method_descriptor(): context = limitted() - assert guarded_eval('list.copy.__name__', context) == 'copy' + assert guarded_eval("list.copy.__name__", context) == "copy" @pytest.mark.parametrize( "data,good,bad,expected", [ - [ - [1, 2, 3], - 'data.index(2)', - 'data.append(4)', - 1 - ], - [ - {'a': 1}, - 'data.keys().isdisjoint({})', - 'data.update()', - True - ] - ] + [[1, 2, 3], "data.index(2)", "data.append(4)", 1], + [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True], + ], ) def test_calls(data, good, bad, expected): context = limitted(data=data) @@ -211,19 +189,10 @@ def test_calls(data, good, bad, expected): @pytest.mark.parametrize( "code,expected", [ - [ - '(1\n+\n1)', - 2 - ], - [ - 'list(range(10))[-1:]', - [9] - ], - [ - 'list(range(20))[3:-2:3]', - [3, 6, 9, 12, 15] - ] - ] + ["(1\n+\n1)", 2], + ["list(range(10))[-1:]", [9]], + ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]], + ], ) def test_literals(code, expected): context = limitted() @@ -232,22 +201,20 @@ def test_literals(code, expected): def test_subscript(): context = EvaluationContext( - locals_={}, - globals_={}, - evaluation='limitted', - in_subscript=True + locals_={}, globals_={}, evaluation="limitted", in_subscript=True ) empty_slice = slice(None, None, None) - assert guarded_eval('', context) == tuple() - assert guarded_eval(':', context) == empty_slice - assert guarded_eval('1:2:3', context) == slice(1, 2, 3) + assert guarded_eval("", context) == tuple() + assert guarded_eval(":", context) == empty_slice + assert guarded_eval("1:2:3", context) == slice(1, 2, 3) assert guarded_eval(':, "a"', context) == (empty_slice, "a") def test_unbind_method(): class X(list): def index(self, k): - return 'CUSTOM' + return "CUSTOM" + x = X() assert unbind_method(x.index) is X.index assert unbind_method([].index) is list.index @@ -261,16 +228,19 @@ def test_assumption_instance_attr_do_not_matter(): versions could invalidate this assumptions. This test is meant to catch such a change if it ever comes true. """ + class T: def __getitem__(self, k): - return 'a' + return "a" + def __getattr__(self, k): - return 'a' + return "a" + t = T() - t.__getitem__ = lambda f: 'b' - t.__getattr__ = lambda f: 'b' - assert t[1] == 'a' - assert t[1] == 'a' + t.__getitem__ = lambda f: "b" + t.__getattr__ = lambda f: "b" + assert t[1] == "a" + assert t[1] == "a" def test_assumption_named_tuples_share_getitem(): From b2af257cac51c788faeb2186fd9494f4f9fc8b1c Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 01:14:39 +0000 Subject: [PATCH 033/122] Fix typos --- IPython/core/completer.py | 10 +++--- IPython/core/guarded_eval.py | 11 +++---- IPython/core/tests/test_completer.py | 2 +- IPython/core/tests/test_guarded_eval.py | 44 ++++++++++++------------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 3ea3dd99258..46ab19423cb 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -916,15 +916,15 @@ class Completer(Configurable): .. deprecated:: 8.8 Use :any:`evaluation` and :any:`auto_close_dict_keys` instead. - Whent enabled in IPython 8.8+ activates following settings for compatibility: + When enabled in IPython 8.8+ activates following settings for compatibility: - ``evaluation = 'unsafe'`` - ``auto_close_dict_keys = True`` """, ).tag(config=True) evaluation = Enum( - ("forbidden", "minimal", "limitted", "unsafe", "dangerous"), - default_value="limitted", + ("forbidden", "minimal", "limited", "unsafe", "dangerous"), + default_value="limited", help="""Code evaluation under completion. Successive options allow to enable more eager evaluation for more accurate completion suggestions, @@ -934,7 +934,7 @@ class Completer(Configurable): Allowed values are: - `forbidden`: no evaluation at all - `minimal`: evaluation of literals and access to built-in namespaces; no item/attribute evaluation nor access to locals/globals - - `limitted` (default): access to all namespaces, evaluation of hard-coded methods (``keys()``, ``__getattr__``, ``__getitems__``, etc) on allow-listed objects (e.g. ``dict``, ``list``, ``tuple``, ``pandas.Series``) + - `limited` (default): access to all namespaces, evaluation of hard-coded methods (``keys()``, ``__getattr__``, ``__getitems__``, etc) on allow-listed objects (e.g. ``dict``, ``list``, ``tuple``, ``pandas.Series``) - `unsafe`: evaluation of all methods and function calls but not of syntax with side-effects like `del x`, - `dangerous`: completely arbitrary evaluation """, @@ -1651,7 +1651,7 @@ def _greedy_changed(self, change): self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: - self.evaluation = "limitted" + self.evaluation = "limited" self.auto_close_dict_keys = False self.splitter.delims = DELIMS diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index 2c278a238ae..0ed69dbb677 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -207,7 +207,7 @@ class EvaluationContext(NamedTuple): locals_: dict globals_: dict evaluation: Literal[ - "forbidden", "minimal", "limitted", "unsafe", "dangerous" + "forbidden", "minimal", "limited", "unsafe", "dangerous" ] = "forbidden" in_subscript: bool = False @@ -260,13 +260,13 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): Applies evaluation restrictions defined in the context. - Currently does not support evaluation of functions with arguments. + Currently does not support evaluation of functions with keyword arguments. Does not evaluate actions which always have side effects: - class definitions (``class sth: ...``) - function definitions (``def sth: ...``) - variable assignments (``x = 1``) - - augumented assignments (``x += 1``) + - augmented assignments (``x += 1``) - deletions (``del x``) Does not evaluate operations which do not return values: @@ -274,7 +274,7 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): - pass (``pass``) - imports (``import x``) - control flow - - conditionals (``if x:``) except for terenary IfExp (``a if x else b``) + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) - loops (``for`` and `while``) - exception handling @@ -393,7 +393,6 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): return eval_node(node.orelse, context) if isinstance(node, ast.Call): func = eval_node(node.func, context) - print(node.keywords) if policy.can_call(func) and not node.keywords: args = [eval_node(arg, context) for arg in node.args] return func(*args) @@ -490,7 +489,7 @@ def _list_methods(cls, source=None): allowed_calls=set(), allow_any_calls=False, ), - "limitted": SelectivePolicy( + "limited": SelectivePolicy( # TODO: # - should reject binary and unary operations if custom methods would be dispatched allowed_getitem=BUILTIN_GETITEM, diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 4e385d546a0..849d963905f 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -1354,7 +1354,7 @@ def completes_on_nested(): with greedy_completion(): completes_on_nested() - with evaluation_level("limitted"): + with evaluation_level("limited"): completes_on_nested() with evaluation_level("minimal"): diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 129112ff4db..2c9db81bb19 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -9,8 +9,8 @@ import pytest -def limitted(**kwargs): - return EvaluationContext(locals_=kwargs, globals_={}, evaluation="limitted") +def limited(**kwargs): + return EvaluationContext(locals_=kwargs, globals_={}, evaluation="limited") def unsafe(**kwargs): @@ -22,7 +22,7 @@ def test_pandas_series_iloc(): import pandas as pd series = pd.Series([1], index=["a"]) - context = limitted(data=series) + context = limited(data=series) assert guarded_eval("data.iloc[0]", context) == 1 @@ -30,7 +30,7 @@ def test_pandas_series_iloc(): def test_pandas_series(): import pandas as pd - context = limitted(data=pd.Series([1], index=["a"])) + context = limited(data=pd.Series([1], index=["a"])) assert guarded_eval('data["a"]', context) == 1 with pytest.raises(KeyError): guarded_eval('data["c"]', context) @@ -49,7 +49,7 @@ def __getattr__(self, key): return "CUSTOM_ATTR" bad_series = BadItemSeries([1], index=["a"]) - context = limitted(data=bad_series) + context = limited(data=bad_series) with pytest.raises(GuardRejection): guarded_eval('data["a"]', context) @@ -65,7 +65,7 @@ def __getattr__(self, key): assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM" bad_attr_series = BadAttrSeries([1], index=["a"]) - context = limitted(data=bad_attr_series) + context = limited(data=bad_attr_series) assert guarded_eval('data["a"]', context) == 1 with pytest.raises(GuardRejection): guarded_eval("data.a", context) @@ -77,7 +77,7 @@ def test_pandas_dataframe_loc(): from pandas.testing import assert_series_equal data = pd.DataFrame([{"a": 1}]) - context = limitted(data=data) + context = limited(data=data) assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"]) @@ -95,16 +95,16 @@ def __getitem__(self, key): good = GoodNamedTuple(a="x") bad = BadNamedTuple(a="x") - context = limitted(data=good) + context = limited(data=good) assert guarded_eval("data[0]", context) == "x" - context = limitted(data=bad) + context = limited(data=bad) with pytest.raises(GuardRejection): guarded_eval("data[0]", context) def test_dict(): - context = limitted(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) + context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) assert guarded_eval('data["a"]', context) == 1 assert guarded_eval('data["b"]', context) == {"x": 2} assert guarded_eval('data["b"]["x"]', context) == 2 @@ -114,43 +114,43 @@ def test_dict(): def test_set(): - context = limitted(data={"a", "b"}) + context = limited(data={"a", "b"}) assert guarded_eval("data.difference", context) def test_list(): - context = limitted(data=[1, 2, 3]) + context = limited(data=[1, 2, 3]) assert guarded_eval("data[1]", context) == 2 assert guarded_eval("data.copy", context) def test_dict_literal(): - context = limitted() + context = limited() assert guarded_eval("{}", context) == {} assert guarded_eval('{"a": 1}', context) == {"a": 1} def test_list_literal(): - context = limitted() + context = limited() assert guarded_eval("[]", context) == [] assert guarded_eval('[1, "a"]', context) == [1, "a"] def test_set_literal(): - context = limitted() + context = limited() assert guarded_eval("set()", context) == set() assert guarded_eval('{"a"}', context) == {"a"} def test_if_expression(): - context = limitted() + context = limited() assert guarded_eval("2 if True else 3", context) == 2 assert guarded_eval("4 if False else 5", context) == 5 def test_object(): obj = object() - context = limitted(obj=obj) + context = limited(obj=obj) assert guarded_eval("obj.__dir__", context) == obj.__dir__ @@ -163,11 +163,11 @@ def test_object(): ], ) def test_number_attributes(code, expected): - assert guarded_eval(code, limitted()) == expected + assert guarded_eval(code, limited()) == expected def test_method_descriptor(): - context = limitted() + context = limited() assert guarded_eval("list.copy.__name__", context) == "copy" @@ -179,7 +179,7 @@ def test_method_descriptor(): ], ) def test_calls(data, good, bad, expected): - context = limitted(data=data) + context = limited(data=data) assert guarded_eval(good, context) == expected with pytest.raises(GuardRejection): @@ -195,13 +195,13 @@ def test_calls(data, good, bad, expected): ], ) def test_literals(code, expected): - context = limitted() + context = limited() assert guarded_eval(code, context) == expected def test_subscript(): context = EvaluationContext( - locals_={}, globals_={}, evaluation="limitted", in_subscript=True + locals_={}, globals_={}, evaluation="limited", in_subscript=True ) empty_slice = slice(None, None, None) assert guarded_eval("", context) == tuple() From 467747d1e6716a0b5b796d2f9dcabe7474833d29 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 14:55:42 +0000 Subject: [PATCH 034/122] Check types with mypy --- .github/workflows/mypy.yml | 1 + IPython/core/guarded_eval.py | 49 ++++++++++++++++--------- IPython/core/tests/test_completer.py | 8 ++-- IPython/core/tests/test_guarded_eval.py | 5 +++ 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 8d1927d6b36..3ff6f86349e 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -31,6 +31,7 @@ jobs: run: | mypy -p IPython.terminal mypy -p IPython.core.magics + mypy -p IPython.core.guarded_eval - name: Lint with pyflakes run: | flake8 IPython/core/magics/script.py diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index 0ed69dbb677..a510d381485 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -1,4 +1,15 @@ -from typing import Callable, Set, Tuple, NamedTuple, Literal, Union, TYPE_CHECKING +from typing import ( + Any, + Callable, + Set, + Tuple, + NamedTuple, + Type, + Literal, + Union, + TYPE_CHECKING, +) +import builtins import collections import sys import ast @@ -21,7 +32,7 @@ def __getitem__(self, key) -> None: class InstancesHaveGetItem(Protocol): - def __call__(self) -> HasGetItem: + def __call__(self, *args, **kwargs) -> HasGetItem: ... @@ -55,6 +66,7 @@ def unbind_method(func: Callable) -> Union[Callable, None]: ) ): return getattr(owner_class, name) + return None @dataclass @@ -137,7 +149,7 @@ def has_original_dunder( @dataclass class SelectivePolicy(EvaluationPolicy): - allowed_getitem: Set[HasGetItem] = field(default_factory=set) + allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) @@ -368,8 +380,9 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): return context.locals_[node.id] if policy.allow_globals_access and node.id in context.globals_: return context.globals_[node.id] - if policy.allow_builtins_access and node.id in __builtins__: - return __builtins__[node.id] + if policy.allow_builtins_access and hasattr(builtins, node.id): + # note: do not use __builtins__, it is implementation detail of Python + return getattr(builtins, node.id) if not policy.allow_globals_access and not policy.allow_locals_access: raise GuardRejection( f"Namespace access not allowed in {context.evaluation} mode" @@ -413,7 +426,7 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): ("numpy", "void"), } -BUILTIN_GETITEM = { +BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { dict, str, bytes, @@ -441,8 +454,8 @@ def _list_methods(cls, source=None): set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) -dict_keys = type({}.keys()) -method_descriptor = type(list.copy) +dict_keys: Type[collections.abc.KeysView] = type({}.keys()) +method_descriptor: Any = type(list.copy) ALLOWED_CALLS = { bytes, @@ -479,6 +492,16 @@ def _list_methods(cls, source=None): collections.Counter.most_common, } +BUILTIN_GETATTR: Set[MayHaveGetattr] = { + *BUILTIN_GETITEM, + set, + frozenset, + object, + type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + dict_keys, + method_descriptor, +} + EVALUATION_POLICIES = { "minimal": EvaluationPolicy( allow_builtins_access=True, @@ -494,15 +517,7 @@ def _list_methods(cls, source=None): # - should reject binary and unary operations if custom methods would be dispatched allowed_getitem=BUILTIN_GETITEM, allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, - allowed_getattr={ - *BUILTIN_GETITEM, - set, - frozenset, - object, - type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. - dict_keys, - method_descriptor, - }, + allowed_getattr=BUILTIN_GETATTR, allowed_getattr_external={ # pandas Series/Frame implements custom `__getattr__` ("pandas", "DataFrame"), diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 849d963905f..bd2fa3cefdb 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -114,7 +114,7 @@ def greedy_completion(): @contextmanager -def evaluation_level(evaluation: str): +def evaluation_policy(evaluation: str): ip = get_ipython() evaluation_original = ip.Completer.evaluation try: @@ -1253,7 +1253,7 @@ def assert_completion(**kwargs): # nested dict completion assert_completion(line_buffer="nested['x'][") - with evaluation_level("minimal"): + with evaluation_policy("minimal"): with pytest.raises(AssertionError): assert_completion(line_buffer="nested['x'][") @@ -1354,10 +1354,10 @@ def completes_on_nested(): with greedy_completion(): completes_on_nested() - with evaluation_level("limited"): + with evaluation_policy("limited"): completes_on_nested() - with evaluation_level("minimal"): + with evaluation_policy("minimal"): with pytest.raises(AssertionError): completes_on_nested() diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 2c9db81bb19..b908f2af255 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -199,6 +199,11 @@ def test_literals(code, expected): assert guarded_eval(code, context) == expected +def test_access_builtins(): + context = limited() + assert guarded_eval("round", context) == round + + def test_subscript(): context = EvaluationContext( locals_={}, globals_={}, evaluation="limited", in_subscript=True From 79c46895a728bb4d3ccb8d3fa03a1a0ddd327bec Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 15:04:48 +0000 Subject: [PATCH 035/122] Compactify assertions spaghettified by black --- IPython/core/tests/test_completer.py | 143 +++++++++------------------ 1 file changed, 48 insertions(+), 95 deletions(-) diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index bd2fa3cefdb..423979a297f 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -853,31 +853,38 @@ def test_match_dict_keys(self): delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" def match(*args, **kwargs): - quote, offset, matches = match_dict_keys(*args, **kwargs) + quote, offset, matches = match_dict_keys(*args, delims=delims, **kwargs) return quote, offset, list(matches) keys = ["foo", b"far"] - assert match(keys, "b'", delims=delims) == ("'", 2, ["far"]) - assert match(keys, "b'f", delims=delims) == ("'", 2, ["far"]) - assert match(keys, 'b"', delims=delims) == ('"', 2, ["far"]) - assert match(keys, 'b"f', delims=delims) == ('"', 2, ["far"]) + assert match(keys, "b'") == ("'", 2, ["far"]) + assert match(keys, "b'f") == ("'", 2, ["far"]) + assert match(keys, 'b"') == ('"', 2, ["far"]) + assert match(keys, 'b"f') == ('"', 2, ["far"]) - assert match(keys, "'", delims=delims) == ("'", 1, ["foo"]) - assert match(keys, "'f", delims=delims) == ("'", 1, ["foo"]) - assert match(keys, '"', delims=delims) == ('"', 1, ["foo"]) - assert match(keys, '"f', delims=delims) == ('"', 1, ["foo"]) + assert match(keys, "'") == ("'", 1, ["foo"]) + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, '"') == ('"', 1, ["foo"]) + assert match(keys, '"f') == ('"', 1, ["foo"]) # Completion on first item of tuple keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")] - assert match(keys, "'f", delims=delims) == ("'", 1, ["foo"]) - assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, "33") == ("", 0, ["3333"]) # Completion on numbers - keys = [0xDEADBEEF, 1111, 1234, "1999", 0b10101, 22] # 3735928559 # 21 - assert match(keys, "0xdead", delims=delims) == ("", 0, ["0xdeadbeef"]) - assert match(keys, "1", delims=delims) == ("", 0, ["1111", "1234"]) - assert match(keys, "2", delims=delims) == ("", 0, ["21", "22"]) - assert match(keys, "0b101", delims=delims) == ("", 0, ["0b10101", "0b10110"]) + keys = [ + 0xDEADBEEF, + 1111, + 1234, + "1999", + 0b10101, + 22, + ] # 0xDEADBEEF = 3735928559; 0b10101 = 21 + assert match(keys, "0xdead") == ("", 0, ["0xdeadbeef"]) + assert match(keys, "1") == ("", 0, ["1111", "1234"]) + assert match(keys, "2") == ("", 0, ["21", "22"]) + assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"]) def test_match_dict_keys_tuple(self): """ @@ -888,97 +895,43 @@ def test_match_dict_keys_tuple(self): keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')] - def match(*args, **kwargs): - quote, offset, matches = match_dict_keys(*args, **kwargs) + def match(*args, extra=None, **kwargs): + quote, offset, matches = match_dict_keys( + *args, delims=delims, extra_prefix=extra, **kwargs + ) return quote, offset, list(matches) # Completion on first key == "foo" - assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ( - "'", - 1, - ["bar", "oof"], - ) - assert match(keys, '"', delims=delims, extra_prefix=("foo",)) == ( - '"', - 1, - ["bar", "oof"], - ) - assert match(keys, "'o", delims=delims, extra_prefix=("foo",)) == ( - "'", - 1, - ["oof"], - ) - assert match(keys, '"o', delims=delims, extra_prefix=("foo",)) == ( - '"', - 1, - ["oof"], - ) - assert match(keys, "b'", delims=delims, extra_prefix=("foo",)) == ( - "'", - 2, - ["bar"], - ) - assert match(keys, 'b"', delims=delims, extra_prefix=("foo",)) == ( - '"', - 2, - ["bar"], - ) - assert match(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ( - "'", - 2, - ["bar"], - ) - assert match(keys, 'b"b', delims=delims, extra_prefix=("foo",)) == ( - '"', - 2, - ["bar"], - ) + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["bar", "oof"]) + assert match(keys, '"', extra=("foo",)) == ('"', 1, ["bar", "oof"]) + assert match(keys, "'o", extra=("foo",)) == ("'", 1, ["oof"]) + assert match(keys, '"o', extra=("foo",)) == ('"', 1, ["oof"]) + assert match(keys, "b'", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"', extra=("foo",)) == ('"', 2, ["bar"]) + assert match(keys, "b'b", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"b', extra=("foo",)) == ('"', 2, ["bar"]) # No Completion - assert match(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) - assert match(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) + assert match(keys, "'", extra=("no_foo",)) == ("'", 1, []) + assert match(keys, "'", extra=("fo",)) == ("'", 1, []) keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")] - assert match(keys, "'foo", delims=delims, extra_prefix=("foo1",)) == ( + assert match(keys, "'foo", extra=("foo1",)) == ("'", 1, ["foo2"]) + assert match(keys, "'foo", extra=("foo1", "foo2")) == ("'", 1, ["foo3"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3")) == ("'", 1, ["foo4"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3", "foo4")) == ( "'", 1, - ["foo2"], + [], ) - assert match(keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2")) == ( - "'", - 1, - ["foo3"], - ) - assert match( - keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2", "foo3") - ) == ("'", 1, ["foo4"]) - assert match( - keys, "'foo", delims=delims, extra_prefix=("foo1", "foo2", "foo3", "foo4") - ) == ("'", 1, []) keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)] - assert match(keys, "'", delims=delims, extra_prefix=("foo",)) == ( - "'", - 1, - ["2222"], - ) - assert match(keys, "", delims=delims, extra_prefix=("foo",)) == ( - "", - 0, - ["1111", "'2222'"], - ) - assert match(keys, "'", delims=delims, extra_prefix=(3333,)) == ( - "'", - 1, - ["bar"], - ) - assert match(keys, "", delims=delims, extra_prefix=(3333,)) == ( - "", - 0, - ["'bar'", "4444"], - ) - assert match(keys, "'", delims=delims, extra_prefix=("3333",)) == ("'", 1, []) - assert match(keys, "33", delims=delims) == ("", 0, ["3333"]) + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["2222"]) + assert match(keys, "", extra=("foo",)) == ("", 0, ["1111", "'2222'"]) + assert match(keys, "'", extra=(3333,)) == ("'", 1, ["bar"]) + assert match(keys, "", extra=(3333,)) == ("", 0, ["'bar'", "4444"]) + assert match(keys, "'", extra=("3333",)) == ("'", 1, []) + assert match(keys, "33") == ("", 0, ["3333"]) def test_dict_key_completion_closures(self): ip = get_ipython() From 80b4bcf36ababf554d26b1f789460b911c5a97c3 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 16:56:09 +0000 Subject: [PATCH 036/122] Enable mypy typing checks for completer --- .github/workflows/mypy.yml | 1 + IPython/core/completer.py | 185 ++++++++++++++++++++++++++----------- 2 files changed, 133 insertions(+), 53 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 3ff6f86349e..e05678f724d 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -32,6 +32,7 @@ jobs: mypy -p IPython.terminal mypy -p IPython.core.magics mypy -p IPython.core.guarded_eval + mypy -p IPython.core.completer - name: Lint with pyflakes run: | flake8 IPython/core/magics/script.py diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 46ab19423cb..2f3b4f01457 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -210,6 +210,8 @@ Optional, TYPE_CHECKING, Set, + Sized, + TypeVar, Literal, ) @@ -255,10 +257,11 @@ if TYPE_CHECKING or GENERATING_DOCUMENTATION: from typing import cast - from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias + from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard else: + from typing import Generic - def cast(obj, type_): + def cast(type_, obj): """Workaround for `TypeError: MatcherAPIv2() takes no arguments`""" return obj @@ -267,6 +270,7 @@ def cast(obj, type_): TypedDict = Dict # by extension of `NotRequired` requires 3.11 too Protocol = object # requires Python >=3.8 TypeAlias = Any # requires Python >=3.10 + TypeGuard = Generic # requires Python >=3.10 if GENERATING_DOCUMENTATION: from typing import TypedDict @@ -470,8 +474,9 @@ def __init__(self, name): self.complete = name self.type = 'crashed' self.name_with_symbols = name - self.signature = '' - self._origin = 'fake' + self.signature = "" + self._origin = "fake" + self.text = "crashed" def __repr__(self): return '' @@ -507,11 +512,23 @@ class Completion: __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] - def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: - warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) + def __init__( + self, + start: int, + end: int, + text: str, + *, + type: Optional[str] = None, + _origin="", + signature="", + ) -> None: + warnings.warn( + "``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, + stacklevel=2, + ) self.start = start self.end = end @@ -524,7 +541,7 @@ def __repr__(self): return '' % \ (self.start, self.end, self.text, self.type or '?', self.signature or '?') - def __eq__(self, other)->Bool: + def __eq__(self, other) -> bool: """ Equality and hash do not hash the type (as some completer may not be able to infer the type), but are use to (partially) de-duplicate @@ -592,14 +609,18 @@ class SimpleMatcherResult(_MatcherResultBase, TypedDict): # in order to get __orig_bases__ for documentation #: List of candidate completions - completions: Sequence[SimpleCompletion] + completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion] class _JediMatcherResult(_MatcherResultBase): """Matching result returned by Jedi (will be processed differently)""" #: list of candidate completions - completions: Iterable[_JediCompletionLike] + completions: Iterator[_JediCompletionLike] + + +AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion] +AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion) @dataclass @@ -650,6 +671,9 @@ def __call__(self, text: str) -> List[str]: """Call signature.""" ... + #: Used to construct the default matcher identifier + __qualname__: str + class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol): #: API version @@ -674,25 +698,59 @@ def __call__(self, context: CompletionContext) -> MatcherResult: """Call signature.""" ... + #: Used to construct the default matcher identifier + __qualname__: str + Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] +def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]: + api_version = _get_matcher_api_version(matcher) + return api_version == 1 + + +def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]: + api_version = _get_matcher_api_version(matcher) + return api_version == 2 + + +def _is_sizable(value: Any) -> TypeGuard[Sized]: + """Determines whether objects is sizable""" + return hasattr(value, "__len__") + + +def _is_iterator(value: Any) -> TypeGuard[Iterator]: + """Determines whether objects is sizable""" + return hasattr(value, "__next__") + + def has_any_completions(result: MatcherResult) -> bool: """Check if any result includes any completions.""" - if hasattr(result["completions"], "__len__"): - return len(result["completions"]) != 0 - try: - old_iterator = result["completions"] - first = next(old_iterator) - result["completions"] = itertools.chain([first], old_iterator) - return True - except StopIteration: - return False + completions = result["completions"] + if _is_sizable(completions): + return len(completions) != 0 + if _is_iterator(completions): + try: + old_iterator = completions + first = next(old_iterator) + result["completions"] = cast( + Iterator[SimpleCompletion], + itertools.chain([first], old_iterator), + ) + return True + except StopIteration: + return False + raise ValueError( + "Completions returned by matcher need to be an Iterator or a Sizable" + ) def completion_matcher( - *, priority: float = None, identifier: str = None, api_version: int = 1 + *, + priority: Optional[float] = None, + identifier: Optional[str] = None, + api_version: int = 1, ): """Adds attributes describing the matcher. @@ -715,14 +773,14 @@ def completion_matcher( """ def wrapper(func: Matcher): - func.matcher_priority = priority or 0 - func.matcher_identifier = identifier or func.__qualname__ - func.matcher_api_version = api_version + func.matcher_priority = priority or 0 # type: ignore + func.matcher_identifier = identifier or func.__qualname__ # type: ignore + func.matcher_api_version = api_version # type: ignore if TYPE_CHECKING: if api_version == 1: - func = cast(func, MatcherAPIv1) + func = cast(MatcherAPIv1, func) elif api_version == 2: - func = cast(func, MatcherAPIv2) + func = cast(MatcherAPIv2, func) return func return wrapper @@ -1311,6 +1369,8 @@ def filter_prefix_tuple(key): matched: Dict[str, DictKeyState] = {} + str_key: Union[str, bytes] + for key in filtered_keys: if isinstance(key, (int, float)): # User typed a number but this key is not a number. @@ -1637,7 +1697,7 @@ def _convert_matcher_v1_result_to_v2( } if fragment is not None: result["matched_fragment"] = fragment - return result + return cast(SimpleMatcherResult, result) class IPCompleter(Completer): @@ -1839,7 +1899,7 @@ def __init__( if not self.backslash_combining_completions: for matcher in self._backslash_combining_matchers: - self.disable_matchers.append(matcher.matcher_identifier) + self.disable_matchers.append(_get_matcher_id(matcher)) if not self.merge_completions: self.suppress_competing_matchers = True @@ -2129,7 +2189,7 @@ def _jedi_matcher(self, context: CompletionContext) -> _JediMatcherResult: def _jedi_matches( self, cursor_column: int, cursor_line: int, text: str - ) -> Iterable[_JediCompletionLike]: + ) -> Iterator[_JediCompletionLike]: """ Return a list of :any:`jedi.api.Completion`s object from a ``text`` and cursor position. @@ -2195,15 +2255,23 @@ def _jedi_matches( print("Error detecting if completing a non-finished string :", e, '|') if not try_jedi: - return [] + return iter([]) try: return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1)) except Exception as e: if self.debug: - return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + return iter( + [ + _FakeJediCompletion( + 'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' + % (e) + ) + ] + ) else: - return [] + return iter([]) + @completion_matcher(api_version=1) def python_matches(self, text: str) -> Iterable[str]: """Match attributes or global python names""" if "." in text: @@ -2762,17 +2830,23 @@ def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Com jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + def is_non_jedi_result( + result: MatcherResult, identifier: str + ) -> TypeGuard[SimpleMatcherResult]: + return identifier != jedi_matcher_id + results = self._complete( full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column ) + non_jedi_results: Dict[str, SimpleMatcherResult] = { identifier: result for identifier, result in results.items() - if identifier != jedi_matcher_id + if is_non_jedi_result(result, identifier) } jedi_matches = ( - cast(results[jedi_matcher_id], _JediMatcherResult)["completions"] + cast(_JediMatcherResult, results[jedi_matcher_id])["completions"] if jedi_matcher_id in results else () ) @@ -2827,8 +2901,8 @@ def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Com signature="", ) - ordered = [] - sortable = [] + ordered: List[Completion] = [] + sortable: List[Completion] = [] for origin, result in non_jedi_results.items(): matched_text = result["matched_fragment"] @@ -2918,8 +2992,8 @@ def _arrange_and_extract( abort_if_offset_changes: bool, ): - sortable = [] - ordered = [] + sortable: List[AnyMatcherCompletion] = [] + ordered: List[AnyMatcherCompletion] = [] most_recent_fragment = None for identifier, result in results.items(): if identifier in skip_matchers: @@ -3018,11 +3092,11 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, ) # Start with a clean slate of completions - results = {} + results: Dict[str, MatcherResult] = {} jedi_matcher_id = _get_matcher_id(self._jedi_matcher) - suppressed_matchers = set() + suppressed_matchers: Set[str] = set() matchers = { _get_matcher_id(matcher): matcher @@ -3032,7 +3106,6 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, } for matcher_id, matcher in matchers.items(): - api_version = _get_matcher_api_version(matcher) matcher_id = _get_matcher_id(matcher) if matcher_id in self.disable_matchers: @@ -3044,14 +3117,16 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, if matcher_id in suppressed_matchers: continue + result: MatcherResult try: - if api_version == 1: + if _is_matcher_v1(matcher): result = _convert_matcher_v1_result_to_v2( matcher(text), type=_UNKNOWN_TYPE ) - elif api_version == 2: - result = cast(matcher, MatcherAPIv2)(context) + elif _is_matcher_v2(matcher): + result = matcher(context) else: + api_version = _get_matcher_api_version(matcher) raise ValueError(f"Unsupported API version {api_version}") except: # Show the ugly traceback if the matcher causes an @@ -3063,7 +3138,9 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, result["matched_fragment"] = result.get("matched_fragment", context.token) if not suppressed_matchers: - suppression_recommended = result.get("suppress", False) + suppression_recommended: Union[bool, Set[str]] = result.get( + "suppress", False + ) suppression_config = ( self.suppress_competing_matchers.get(matcher_id, None) @@ -3076,10 +3153,12 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, ) and has_any_completions(result) if should_suppress: - suppression_exceptions = result.get("do_not_suppress", set()) - try: + suppression_exceptions: Set[str] = result.get( + "do_not_suppress", set() + ) + if isinstance(suppression_recommended, Iterable): to_suppress = set(suppression_recommended) - except TypeError: + else: to_suppress = set(matchers) suppressed_matchers = to_suppress - suppression_exceptions @@ -3106,9 +3185,9 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, @staticmethod def _deduplicate( - matches: Sequence[SimpleCompletion], - ) -> Iterable[SimpleCompletion]: - filtered_matches = {} + matches: Sequence[AnyCompletion], + ) -> Iterable[AnyCompletion]: + filtered_matches: Dict[str, AnyCompletion] = {} for match in matches: text = match.text if ( @@ -3120,7 +3199,7 @@ def _deduplicate( return filtered_matches.values() @staticmethod - def _sort(matches: Sequence[SimpleCompletion]): + def _sort(matches: Sequence[AnyCompletion]): return sorted(matches, key=lambda x: completions_sorting_key(x.text)) @context_matcher() From bbf990daf76dce132ac2f66e782d463f54a21f7f Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 20:27:08 +0000 Subject: [PATCH 037/122] Polish documentation, hide private functions --- IPython/core/completer.py | 80 ++++++++++++-------- IPython/core/guarded_eval.py | 98 +++++++++++++++++-------- IPython/core/magics/config.py | 88 ++-------------------- IPython/core/tests/test_guarded_eval.py | 12 +-- 4 files changed, 129 insertions(+), 149 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 2f3b4f01457..7dd585bce36 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -50,7 +50,7 @@ It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press ```` to expand it to its latex form. +and press :kbd:`Tab` to expand it to its latex form. .. code:: @@ -59,7 +59,7 @@ Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. +:any:`Completer.backslash_combining_completions` option to ``False``. Experimental @@ -95,7 +95,7 @@ ... myvar[1].bi Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` +executing almost any code unlike the deprecated :any:`IPCompleter.greedy` option. Be sure to update :any:`jedi` to the latest stable version or to try the @@ -972,29 +972,38 @@ class Completer(Configurable): help="""Activate greedy completion. .. deprecated:: 8.8 - Use :any:`evaluation` and :any:`auto_close_dict_keys` instead. + Use :any:`Completer.evaluation` and :any:`Completer.auto_close_dict_keys` instead. - When enabled in IPython 8.8+ activates following settings for compatibility: - - ``evaluation = 'unsafe'`` - - ``auto_close_dict_keys = True`` + When enabled in IPython 8.8 or newer, changes configuration as follows: + + - ``Completer.evaluation = 'unsafe'`` + - ``Completer.auto_close_dict_keys = True`` """, ).tag(config=True) evaluation = Enum( ("forbidden", "minimal", "limited", "unsafe", "dangerous"), default_value="limited", - help="""Code evaluation under completion. + help="""Policy for code evaluation under completion. - Successive options allow to enable more eager evaluation for more accurate completion suggestions, - including for nested dictionaries, nested lists, or even results of function calls. Setting `unsafe` - or higher can lead to evaluation of arbitrary user code on TAB with potentially dangerous side effects. + Successive options allow to enable more eager evaluation for better + completion suggestions, including for nested dictionaries, nested lists, + or even results of function calls. + Setting ``unsafe`` or higher can lead to evaluation of arbitrary user + code on :kbd:`Tab` with potentially unwanted or dangerous side effects. Allowed values are: - - `forbidden`: no evaluation at all - - `minimal`: evaluation of literals and access to built-in namespaces; no item/attribute evaluation nor access to locals/globals - - `limited` (default): access to all namespaces, evaluation of hard-coded methods (``keys()``, ``__getattr__``, ``__getitems__``, etc) on allow-listed objects (e.g. ``dict``, ``list``, ``tuple``, ``pandas.Series``) - - `unsafe`: evaluation of all methods and function calls but not of syntax with side-effects like `del x`, - - `dangerous`: completely arbitrary evaluation + + - ``forbidden``: no evaluation of code is permitted, + - ``minimal``: evaluation of literals and access to built-in namespace; + no item/attribute evaluation nor access to locals/globals, + - ``limited``: access to all namespaces, evaluation of hard-coded methods + (for example: :any:`dict.keys`, :any:`object.__getattr__`, + :any:`object.__getitem__`) on allow-listed objects (for example: + :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``), + - ``unsafe``: evaluation of all methods and function calls but not of + syntax with side-effects like `del x`, + - ``dangerous``: completely arbitrary evaluation. """, ).tag(config=True) @@ -1019,7 +1028,15 @@ class Completer(Configurable): "unicode characters back to latex commands.").tag(config=True) auto_close_dict_keys = Bool( - False, help="""Enable auto-closing dictionary keys.""" + False, + help=""" + Enable auto-closing dictionary keys. + + When enabled string keys will be suffixed with a final quote + (matching the opening quote), tuple keys will also receive a + separating comma if needed, and keys which are final will + receive a closing bracket (``]``). + """, ).tag(config=True) def __init__(self, namespace=None, global_namespace=None, **kwargs): @@ -1157,8 +1174,8 @@ def _evaluate_expr(self, expr): obj = guarded_eval( expr, EvaluationContext( - globals_=self.global_namespace, - locals_=self.namespace, + globals=self.global_namespace, + locals=self.namespace, evaluation=self.evaluation, ), ) @@ -1183,7 +1200,7 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -class DictKeyState(enum.Flag): +class _DictKeyState(enum.Flag): """Represent state of the key match in context of other possible matches. - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. @@ -1199,6 +1216,7 @@ class DictKeyState(enum.Flag): def _parse_tokens(c): + """Parse tokens even if there is an error.""" tokens = [] token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) while True: @@ -1257,7 +1275,7 @@ def match_dict_keys( prefix: str, delims: str, extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, -) -> Tuple[str, int, Dict[str, DictKeyState]]: +) -> Tuple[str, int, Dict[str, _DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1307,8 +1325,8 @@ def filter_prefix_tuple(key): return True filtered_key_is_final: Dict[ - Union[str, bytes, int, float], DictKeyState - ] = defaultdict(lambda: DictKeyState.BASELINE) + Union[str, bytes, int, float], _DictKeyState + ] = defaultdict(lambda: _DictKeyState.BASELINE) for k in keys: # If at least one of the matches is not final, mark as undetermined. @@ -1319,9 +1337,9 @@ def filter_prefix_tuple(key): if filter_prefix_tuple(k): key_fragment = k[prefix_tuple_size] filtered_key_is_final[key_fragment] |= ( - DictKeyState.END_OF_TUPLE + _DictKeyState.END_OF_TUPLE if len(k) == prefix_tuple_size + 1 - else DictKeyState.IN_TUPLE + else _DictKeyState.IN_TUPLE ) elif prefix_tuple_size > 0: # we are completing a tuple but this key is not a tuple, @@ -1329,7 +1347,7 @@ def filter_prefix_tuple(key): pass else: if isinstance(k, text_serializable_types): - filtered_key_is_final[k] |= DictKeyState.END_OF_ITEM + filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM filtered_keys = filtered_key_is_final.keys() @@ -1367,7 +1385,7 @@ def filter_prefix_tuple(key): token_start = token_match.start() token_prefix = token_match.group() - matched: Dict[str, DictKeyState] = {} + matched: Dict[str, _DictKeyState] = {} str_key: Union[str, bytes] @@ -2503,8 +2521,8 @@ def dict_key_matches(self, text: str) -> List[str]: tuple_prefix = guarded_eval( prior_tuple_keys, EvaluationContext( - globals_=self.global_namespace, - locals_=self.namespace, + globals=self.global_namespace, + locals=self.namespace, evaluation=self.evaluation, in_subscript=True, ), @@ -2569,7 +2587,7 @@ def dict_key_matches(self, text: str) -> List[str]: results = [] - end_of_tuple_or_item = DictKeyState.END_OF_TUPLE | DictKeyState.END_OF_ITEM + end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM for k, state_flag in matches.items(): result = leading + k @@ -2584,7 +2602,7 @@ def dict_key_matches(self, text: str) -> List[str]: if state_flag in end_of_tuple_or_item and can_close_bracket: result += "]" - if state_flag == DictKeyState.IN_TUPLE and can_close_tuple_item: + if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item: result += ", " results.append(result) return results diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index a510d381485..637d329a17e 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -17,6 +17,7 @@ from dataclasses import dataclass, field from IPython.utils.docs import GENERATING_DOCUMENTATION +from IPython.utils.decorators import undoc if TYPE_CHECKING or GENERATING_DOCUMENTATION: @@ -26,21 +27,25 @@ Protocol = object # requires Python >=3.8 +@undoc class HasGetItem(Protocol): def __getitem__(self, key) -> None: ... +@undoc class InstancesHaveGetItem(Protocol): def __call__(self, *args, **kwargs) -> HasGetItem: ... +@undoc class HasGetAttr(Protocol): def __getattr__(self, key) -> None: ... +@undoc class DoesNotHaveGetAttr(Protocol): pass @@ -49,7 +54,7 @@ class DoesNotHaveGetAttr(Protocol): MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] -def unbind_method(func: Callable) -> Union[Callable, None]: +def _unbind_method(func: Callable) -> Union[Callable, None]: """Get unbound method for given bound method. Returns None if cannot get unbound method.""" @@ -69,8 +74,11 @@ def unbind_method(func: Callable) -> Union[Callable, None]: return None +@undoc @dataclass class EvaluationPolicy: + """Definition of evaluation policy.""" + allow_locals_access: bool = False allow_globals_access: bool = False allow_item_access: bool = False @@ -92,12 +100,12 @@ def can_call(self, func): if func in self.allowed_calls: return True - owner_method = unbind_method(func) + owner_method = _unbind_method(func) if owner_method and owner_method in self.allowed_calls: return True -def has_original_dunder_external( +def _has_original_dunder_external( value, module_name, access_path, @@ -121,7 +129,7 @@ def has_original_dunder_external( return False -def has_original_dunder( +def _has_original_dunder( value, allowed_types, allowed_methods, allowed_external, method_name ): # note: Python ignores `__getattr__`/`__getitem__` on instances, @@ -141,12 +149,13 @@ def has_original_dunder( return True for module_name, *access_path in allowed_external: - if has_original_dunder_external(value, module_name, access_path, method_name): + if _has_original_dunder_external(value, module_name, access_path, method_name): return True return False +@undoc @dataclass class SelectivePolicy(EvaluationPolicy): allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) @@ -155,14 +164,14 @@ class SelectivePolicy(EvaluationPolicy): allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) def can_get_attr(self, value, attr): - has_original_attribute = has_original_dunder( + has_original_attribute = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattribute_methods, allowed_external=self.allowed_getattr_external, method_name="__getattribute__", ) - has_original_attr = has_original_dunder( + has_original_attr = _has_original_dunder( value, allowed_types=self.allowed_getattr, allowed_methods=self._getattr_methods, @@ -182,7 +191,7 @@ def get_attr(self, value, attr): def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" - return has_original_dunder( + return _has_original_dunder( value, allowed_types=self.allowed_getitem, allowed_methods=self._getitem_methods, @@ -211,34 +220,50 @@ def _safe_get_methods(self, classes, name) -> Set[Callable]: } -class DummyNamedTuple(NamedTuple): +class _DummyNamedTuple(NamedTuple): pass class EvaluationContext(NamedTuple): - locals_: dict - globals_: dict + #: Local namespace + locals: dict + #: Global namespace + globals: dict + #: Evaluation policy identifier evaluation: Literal[ "forbidden", "minimal", "limited", "unsafe", "dangerous" ] = "forbidden" + #: Whether the evalution of code takes place inside of a subscript. + #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. in_subscript: bool = False -class IdentitySubscript: +class _IdentitySubscript: + """Returns the key itself when item is requested via subscript.""" + def __getitem__(self, key): return key -IDENTITY_SUBSCRIPT = IdentitySubscript() +IDENTITY_SUBSCRIPT = _IdentitySubscript() SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" -class GuardRejection(ValueError): +class GuardRejection(Exception): + """Exception raised when guard rejects evaluation attempt.""" + pass def guarded_eval(code: str, context: EvaluationContext): - locals_ = context.locals_ + """Evaluate provided code in the evaluation context. + + If evaluation policy given by context is set to ``forbidden`` + no evaluation will be performed; if it is set to ``dangerous`` + standard :func:`eval` will be used; finally, for any other, + policy :func:`eval_node` will be called on parsed AST. + """ + locals_ = context.locals if context.evaluation == "forbidden": raise GuardRejection("Forbidden mode") @@ -256,10 +281,10 @@ def guarded_eval(code: str, context: EvaluationContext): locals_ = locals_.copy() locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT code = SUBSCRIPT_MARKER + "[" + code + "]" - context = EvaluationContext(**{**context._asdict(), **{"locals_": locals_}}) + context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}}) if context.evaluation == "dangerous": - return eval(code, context.globals_, context.locals_) + return eval(code, context.globals, context.locals) expression = ast.parse(code, mode="eval") @@ -267,14 +292,12 @@ def guarded_eval(code: str, context: EvaluationContext): def eval_node(node: Union[ast.AST, None], context: EvaluationContext): - """ - Evaluate AST node in provided context. + """Evaluate AST node in provided context. - Applies evaluation restrictions defined in the context. + Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. - Currently does not support evaluation of functions with keyword arguments. + Does not evaluate actions that always have side effects: - Does not evaluate actions which always have side effects: - class definitions (``class sth: ...``) - function definitions (``def sth: ...``) - variable assignments (``x = 1``) @@ -282,13 +305,15 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): - deletions (``del x``) Does not evaluate operations which do not return values: + - assertions (``assert x``) - pass (``pass``) - imports (``import x``) - - control flow - - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) - - loops (``for`` and `while``) - - exception handling + - control flow: + + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) + - loops (``for`` and `while``) + - exception handling The purpose of this function is to guard against unwanted side-effects; it does not give guarantees on protection from malicious code execution. @@ -376,10 +401,10 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): f" not allowed in {context.evaluation} mode", ) if isinstance(node, ast.Name): - if policy.allow_locals_access and node.id in context.locals_: - return context.locals_[node.id] - if policy.allow_globals_access and node.id in context.globals_: - return context.globals_[node.id] + if policy.allow_locals_access and node.id in context.locals: + return context.locals[node.id] + if policy.allow_globals_access and node.id in context.globals: + return context.globals[node.id] if policy.allow_builtins_access and hasattr(builtins, node.id): # note: do not use __builtins__, it is implementation detail of Python return getattr(builtins, node.id) @@ -439,8 +464,8 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): collections.UserDict, collections.UserList, collections.UserString, - DummyNamedTuple, - IdentitySubscript, + _DummyNamedTuple, + _IdentitySubscript, } @@ -537,3 +562,12 @@ def _list_methods(cls, source=None): allow_any_calls=True, ), } + + +__all__ = [ + "guarded_eval", + "eval_node", + "GuardRejection", + "EvaluationContext", + "_unbind_method", +] diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index 87fe3eed3a5..9e1cb38c254 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -68,94 +68,22 @@ def config(self, s): To view what is configurable on a given class, just pass the class name:: - In [2]: %config IPCompleter - IPCompleter(Completer) options - ---------------------------- - IPCompleter.backslash_combining_completions= - Enable unicode completions, e.g. \\alpha . Includes completion of latex - commands, unicode names, and expanding unicode characters back to latex - commands. - Current: True - IPCompleter.debug= - Enable debug for the Completer. Mostly print extra information for - experimental jedi integration. + In [2]: %config LoggingMagics + LoggingMagics(Magics) options + --------------------------- + LoggingMagics.quiet= + Suppress output of log state when logging is enabled Current: False - IPCompleter.disable_matchers=... - List of matchers to disable. - The list should contain matcher identifiers (see - :any:`completion_matcher`). - Current: [] - IPCompleter.greedy= - Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - Current: False - IPCompleter.jedi_compute_type_timeout= - Experimental: restrict time (in milliseconds) during which Jedi can compute types. - Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt - performance by preventing jedi to build its cache. - Current: 400 - IPCompleter.limit_to__all__= - DEPRECATED as of version 5.0. - Instruct the completer to use __all__ for the completion - Specifically, when completing on ``object.``. - When True: only those names in obj.__all__ will be included. - When False [default]: the __all__ attribute is ignored - Current: False - IPCompleter.merge_completions= - Whether to merge completion results into a single list - If False, only the completion results from the first non-empty - completer will be returned. - As of version 8.6.0, setting the value to ``False`` is an alias for: - ``IPCompleter.suppress_competing_matchers = True.``. - Current: True - IPCompleter.omit__names= - Instruct the completer to omit private method names - Specifically, when completing on ``object.``. - When 2 [default]: all names that start with '_' will be excluded. - When 1: all 'magic' names (``__foo__``) will be excluded. - When 0: nothing will be excluded. - Choices: any of [0, 1, 2] - Current: 2 - IPCompleter.profile_completions= - If True, emit profiling data for completion subsystem using cProfile. - Current: False - IPCompleter.profiler_output_dir= - Template for path at which to output profile data for completions. - Current: '.completion_profiles' - IPCompleter.suppress_competing_matchers= - Whether to suppress completions from other *Matchers*. - When set to ``None`` (default) the matchers will attempt to auto-detect - whether suppression of other matchers is desirable. For example, at the - beginning of a line followed by `%` we expect a magic completion to be the - only applicable option, and after ``my_dict['`` we usually expect a - completion with an existing dictionary key. - If you want to disable this heuristic and see completions from all matchers, - set ``IPCompleter.suppress_competing_matchers = False``. To disable the - heuristic for specific matchers provide a dictionary mapping: - ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': - False}``. - Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions - to the set of matchers with the highest priority; this is equivalent to - ``IPCompleter.merge_completions`` and can be beneficial for performance, but - will sometimes omit relevant candidates from matchers further down the - priority list. - Current: None - IPCompleter.use_jedi= - Experimental: Use Jedi to generate autocompletions. Default to True if jedi - is installed. - Current: True but the real use is in setting values:: - In [3]: %config IPCompleter.greedy = True + In [3]: %config LoggingMagics.quiet = True and these values are read from the user_ns if they are variables:: - In [4]: feeling_greedy=False + In [4]: feeling_quiet=False - In [5]: %config IPCompleter.greedy = feeling_greedy + In [5]: %config LoggingMagics.quiet = feeling_quiet """ from traitlets.config.loader import Config diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index b908f2af255..9c98b7a8e2a 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -3,18 +3,18 @@ EvaluationContext, GuardRejection, guarded_eval, - unbind_method, + _unbind_method, ) from IPython.testing import decorators as dec import pytest def limited(**kwargs): - return EvaluationContext(locals_=kwargs, globals_={}, evaluation="limited") + return EvaluationContext(locals=kwargs, globals={}, evaluation="limited") def unsafe(**kwargs): - return EvaluationContext(locals_=kwargs, globals_={}, evaluation="unsafe") + return EvaluationContext(locals=kwargs, globals={}, evaluation="unsafe") @dec.skip_without("pandas") @@ -206,7 +206,7 @@ def test_access_builtins(): def test_subscript(): context = EvaluationContext( - locals_={}, globals_={}, evaluation="limited", in_subscript=True + locals={}, globals={}, evaluation="limited", in_subscript=True ) empty_slice = slice(None, None, None) assert guarded_eval("", context) == tuple() @@ -221,8 +221,8 @@ def index(self, k): return "CUSTOM" x = X() - assert unbind_method(x.index) is X.index - assert unbind_method([].index) is list.index + assert _unbind_method(x.index) is X.index + assert _unbind_method([].index) is list.index def test_assumption_instance_attr_do_not_matter(): From 8f4e32247605f1bb9180aadd6e02a1be62d011bc Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sat, 3 Dec 2022 22:38:42 +0000 Subject: [PATCH 038/122] Add guards for binary, unary operators and comparators --- IPython/core/guarded_eval.py | 181 +++++++++++++++++++----- IPython/core/tests/test_guarded_eval.py | 70 +++++++++ 2 files changed, 214 insertions(+), 37 deletions(-) diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index 637d329a17e..215a36fab05 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -1,6 +1,7 @@ from typing import ( Any, Callable, + Dict, Set, Tuple, NamedTuple, @@ -9,10 +10,11 @@ Union, TYPE_CHECKING, ) +import ast import builtins import collections +import operator import sys -import ast from functools import cached_property from dataclasses import dataclass, field @@ -84,6 +86,7 @@ class EvaluationPolicy: allow_item_access: bool = False allow_attr_access: bool = False allow_builtins_access: bool = False + allow_all_operations: bool = False allow_any_calls: bool = False allowed_calls: Set[Callable] = field(default_factory=set) @@ -93,6 +96,10 @@ def can_get_item(self, value, item): def can_get_attr(self, value, attr): return self.allow_attr_access + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + if self.allow_all_operations: + return True + def can_call(self, func): if self.allow_any_calls: return True @@ -160,9 +167,17 @@ def _has_original_dunder( class SelectivePolicy(EvaluationPolicy): allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) + allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) + allowed_operations: Set = field(default_factory=set) + allowed_operations_external: Set[Tuple[str, ...]] = field(default_factory=set) + + _operation_methods_cache: Dict[str, Set[Callable]] = field( + default_factory=dict, init=False + ) + def can_get_attr(self, value, attr): has_original_attribute = _has_original_dunder( value, @@ -199,6 +214,27 @@ def can_get_item(self, value, item): method_name="__getitem__", ) + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + return all( + [ + _has_original_dunder( + a, + allowed_types=self.allowed_operations, + allowed_methods=self._dunder_methods(dunder), + allowed_external=self.allowed_operations_external, + method_name=dunder, + ) + for dunder in dunders + ] + ) + + def _dunder_methods(self, dunder: str) -> Set[Callable]: + if dunder not in self._operation_methods_cache: + self._operation_methods_cache[dunder] = self._safe_get_methods( + self.allowed_operations, dunder + ) + return self._operation_methods_cache[dunder] + @cached_property def _getitem_methods(self) -> Set[Callable]: return self._safe_get_methods(self.allowed_getitem, "__getitem__") @@ -291,6 +327,50 @@ def guarded_eval(code: str, context: EvaluationContext): return eval_node(expression, context) +BINARY_OP_DUNDERS: Dict[Type[ast.operator], Tuple[str]] = { + ast.Add: ("__add__",), + ast.Sub: ("__sub__",), + ast.Mult: ("__mul__",), + ast.Div: ("__truediv__",), + ast.FloorDiv: ("__floordiv__",), + ast.Mod: ("__mod__",), + ast.Pow: ("__pow__",), + ast.LShift: ("__lshift__",), + ast.RShift: ("__rshift__",), + ast.BitOr: ("__or__",), + ast.BitXor: ("__xor__",), + ast.BitAnd: ("__and__",), + ast.MatMult: ("__matmul__",), +} + +COMP_OP_DUNDERS: Dict[Type[ast.cmpop], Tuple[str, ...]] = { + ast.Eq: ("__eq__",), + ast.NotEq: ("__ne__", "__eq__"), + ast.Lt: ("__lt__", "__gt__"), + ast.LtE: ("__le__", "__ge__"), + ast.Gt: ("__gt__", "__lt__"), + ast.GtE: ("__ge__", "__le__"), + ast.In: ("__contains__",), + # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially +} + +UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = { + ast.USub: ("__neg__",), + ast.UAdd: ("__pos__",), + # we have to check both __inv__ and __invert__! + ast.Invert: ("__invert__", "__inv__"), + ast.Not: ("__not__",), +} + + +def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]: + dunder = None + for op, candidate_dunder in dunders.items(): + if isinstance(node_op, op): + dunder = candidate_dunder + return dunder + + def eval_node(node: Union[ast.AST, None], context: EvaluationContext): """Evaluate AST node in provided context. @@ -324,35 +404,55 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if isinstance(node, ast.Expression): return eval_node(node.body, context) if isinstance(node, ast.BinOp): - # TODO: add guards left = eval_node(node.left, context) right = eval_node(node.right, context) - if isinstance(node.op, ast.Add): - return left + right - if isinstance(node.op, ast.Sub): - return left - right - if isinstance(node.op, ast.Mult): - return left * right - if isinstance(node.op, ast.Div): - return left / right - if isinstance(node.op, ast.FloorDiv): - return left // right - if isinstance(node.op, ast.Mod): - return left % right - if isinstance(node.op, ast.Pow): - return left**right - if isinstance(node.op, ast.LShift): - return left << right - if isinstance(node.op, ast.RShift): - return left >> right - if isinstance(node.op, ast.BitOr): - return left | right - if isinstance(node.op, ast.BitXor): - return left ^ right - if isinstance(node.op, ast.BitAnd): - return left & right - if isinstance(node.op, ast.MatMult): - return left @ right + dunders = _find_dunder(node.op, BINARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, left, right): + return getattr(left, dunders[0])(right) + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Compare): + left = eval_node(node.left, context) + all_true = True + negate = False + for op, right in zip(node.ops, node.comparators): + right = eval_node(right, context) + dunder = None + dunders = _find_dunder(op, COMP_OP_DUNDERS) + if not dunders: + if isinstance(op, ast.NotIn): + dunders = COMP_OP_DUNDERS[ast.In] + negate = True + if isinstance(op, ast.Is): + dunder = "is_" + if isinstance(op, ast.IsNot): + dunder = "is_" + negate = True + if not dunder and dunders: + dunder = dunders[0] + if dunder: + a, b = (right, left) if dunder == "__contains__" else (left, right) + if dunder == "is_" or dunders and policy.can_operate(dunders, a, b): + result = getattr(operator, dunder)(a, b) + if negate: + result = not result + if not result: + all_true = False + left = right + else: + raise GuardRejection( + f"Comparison (`{dunder}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + else: + raise ValueError(f"Comparison `{dunder}` not supported") + return all_true if isinstance(node, ast.Constant): return node.value if isinstance(node, ast.Index): @@ -379,16 +479,17 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if isinstance(node, ast.ExtSlice): return tuple([eval_node(dim, context) for dim in node.dims]) if isinstance(node, ast.UnaryOp): - # TODO: add guards value = eval_node(node.operand, context) - if isinstance(node.op, ast.USub): - return -value - if isinstance(node.op, ast.UAdd): - return +value - if isinstance(node.op, ast.Invert): - return ~value - if isinstance(node.op, ast.Not): - return not value + dunders = _find_dunder(node.op, UNARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, value): + return getattr(value, dunders[0])() + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(value), + f"not allowed in {context.evaluation} mode", + ) raise ValueError("Unhandled unary operation:", node.op) if isinstance(node, ast.Subscript): value = eval_node(node.value, context) @@ -527,6 +628,9 @@ def _list_methods(cls, source=None): method_descriptor, } + +BUILTIN_OPERATIONS = {int, float, complex, *BUILTIN_GETATTR} + EVALUATION_POLICIES = { "minimal": EvaluationPolicy( allow_builtins_access=True, @@ -536,6 +640,7 @@ def _list_methods(cls, source=None): allow_attr_access=False, allowed_calls=set(), allow_any_calls=False, + allow_all_operations=False, ), "limited": SelectivePolicy( # TODO: @@ -548,6 +653,7 @@ def _list_methods(cls, source=None): ("pandas", "DataFrame"), ("pandas", "Series"), }, + allowed_operations=BUILTIN_OPERATIONS, allow_builtins_access=True, allow_locals_access=True, allow_globals_access=True, @@ -560,6 +666,7 @@ def _list_methods(cls, source=None): allow_attr_access=True, allow_item_access=True, allow_any_calls=True, + allow_all_operations=True, ), } diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 9c98b7a8e2a..94f58298665 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -199,6 +199,76 @@ def test_literals(code, expected): assert guarded_eval(code, context) == expected +@pytest.mark.parametrize( + "code,expected", + [ + ["-5", -5], + ["+5", +5], + ["~5", -6], + ], +) +def test_unary_operations(code, expected): + context = limited() + assert guarded_eval(code, context) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1 + 1", 2], + ["3 - 1", 2], + ["2 * 3", 6], + ["5 // 2", 2], + ["5 / 2", 2.5], + ["5**2", 25], + ["2 >> 1", 1], + ["2 << 1", 4], + ["1 | 2", 3], + ["1 & 1", 1], + ["1 & 2", 0], + ], +) +def test_binary_operations(code, expected): + context = limited() + assert guarded_eval(code, context) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["2 > 1", True], + ["2 < 1", False], + ["2 <= 1", False], + ["2 <= 2", True], + ["1 >= 2", False], + ["2 >= 2", True], + ["2 == 2", True], + ["1 == 2", False], + ["1 != 2", True], + ["1 != 1", False], + ["1 < 4 < 3", False], + ["(1 < 4) < 3", True], + ["4 > 3 > 2 > 1", True], + ["4 > 3 > 2 > 9", False], + ["1 < 2 < 3 < 4", True], + ["9 < 2 < 3 < 4", False], + ["1 < 2 > 1 > 0 > -1 < 1", True], + ["1 in [1] in [[1]]", True], + ["1 in [1] in [[2]]", False], + ["1 in [1]", True], + ["0 in [1]", False], + ["1 not in [1]", False], + ["0 not in [1]", True], + ["True is True", True], + ["False is False", True], + ["True is False", False], + ], +) +def test_comparisons(code, expected): + context = limited() + assert guarded_eval(code, context) == expected + + def test_access_builtins(): context = limited() assert guarded_eval("round", context) == round From 6250931515354a158dbda57ea082d65aec8a308b Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 00:25:52 +0000 Subject: [PATCH 039/122] Increase coverage of `guard_eval` --- IPython/core/completer.py | 3 +- IPython/core/guarded_eval.py | 49 ++++-- IPython/core/tests/test_guarded_eval.py | 201 +++++++++++++++++++++--- 3 files changed, 215 insertions(+), 38 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 7dd585bce36..f2853d3c48a 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -996,7 +996,8 @@ class Completer(Configurable): - ``forbidden``: no evaluation of code is permitted, - ``minimal``: evaluation of literals and access to built-in namespace; - no item/attribute evaluation nor access to locals/globals, + no item/attribute evaluationm no access to locals/globals, + no evaluation of any operations or comparisons. - ``limited``: access to all namespaces, evaluation of hard-coded methods (for example: :any:`dict.keys`, :any:`object.__getattr__`, :any:`object.__getitem__`) on allow-listed objects (for example: diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index 215a36fab05..f71a73bea58 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -108,6 +108,7 @@ def can_call(self, func): return True owner_method = _unbind_method(func) + if owner_method and owner_method in self.allowed_calls: return True @@ -127,6 +128,10 @@ def _has_original_dunder_external( value_type = type(value) if type(value) == member_type: return True + if method_name == "__getattribute__": + # we have to short-circuit here due to an unresolved issue in + # `isinstance` implementation: https://bugs.python.org/issue32683 + return False if isinstance(value, member_type): method = getattr(value_type, method_name, None) member_method = getattr(member_type, method_name, None) @@ -149,7 +154,7 @@ def _has_original_dunder( method = getattr(value_type, method_name, None) - if not method: + if method is None: return None if method in allowed_methods: @@ -193,6 +198,7 @@ def can_get_attr(self, value, attr): allowed_external=self.allowed_getattr_external, method_name="__getattr__", ) + # Many objects do not have `__getattr__`, this is fine if has_original_attr is None and has_original_attribute: return True @@ -200,10 +206,6 @@ def can_get_attr(self, value, attr): # Accept objects without modifications to `__getattr__` and `__getattribute__` return has_original_attr and has_original_attribute - def get_attr(self, value, attr): - if self.can_get_attr(value, attr): - return getattr(value, attr) - def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" return _has_original_dunder( @@ -215,20 +217,24 @@ def can_get_item(self, value, item): ) def can_operate(self, dunders: Tuple[str, ...], a, b=None): + objects = [a] + if b is not None: + objects.append(b) return all( [ _has_original_dunder( - a, + obj, allowed_types=self.allowed_operations, - allowed_methods=self._dunder_methods(dunder), + allowed_methods=self._operator_dunder_methods(dunder), allowed_external=self.allowed_operations_external, method_name=dunder, ) for dunder in dunders + for obj in objects ] ) - def _dunder_methods(self, dunder: str) -> Set[Callable]: + def _operator_dunder_methods(self, dunder: str) -> Set[Callable]: if dunder not in self._operation_methods_cache: self._operation_methods_cache[dunder] = self._safe_get_methods( self.allowed_operations, dunder @@ -257,7 +263,7 @@ def _safe_get_methods(self, classes, name) -> Set[Callable]: class _DummyNamedTuple(NamedTuple): - pass + """Used internally to retrieve methods of named tuple instance.""" class EvaluationContext(NamedTuple): @@ -451,12 +457,15 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): f"not allowed in {context.evaluation} mode", ) else: - raise ValueError(f"Comparison `{dunder}` not supported") + raise ValueError( + f"Comparison `{dunder}` not supported" + ) # pragma: no cover return all_true if isinstance(node, ast.Constant): return node.value if isinstance(node, ast.Index): - return eval_node(node.value, context) + # deprecated since Python 3.9 + return eval_node(node.value, context) # pragma: no cover if isinstance(node, ast.Tuple): return tuple(eval_node(e, context) for e in node.elts) if isinstance(node, ast.List): @@ -477,7 +486,8 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): eval_node(node.step, context), ) if isinstance(node, ast.ExtSlice): - return tuple([eval_node(dim, context) for dim in node.dims]) + # deprecated since Python 3.9 + return tuple([eval_node(dim, context) for dim in node.dims]) # pragma: no cover if isinstance(node, ast.UnaryOp): value = eval_node(node.operand, context) dunders = _find_dunder(node.op, UNARY_OP_DUNDERS) @@ -490,7 +500,6 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): type(value), f"not allowed in {context.evaluation} mode", ) - raise ValueError("Unhandled unary operation:", node.op) if isinstance(node, ast.Subscript): value = eval_node(node.value, context) slice_ = eval_node(node.slice, context) @@ -507,14 +516,14 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): if policy.allow_globals_access and node.id in context.globals: return context.globals[node.id] if policy.allow_builtins_access and hasattr(builtins, node.id): - # note: do not use __builtins__, it is implementation detail of Python + # note: do not use __builtins__, it is implementation detail of cPython return getattr(builtins, node.id) if not policy.allow_globals_access and not policy.allow_locals_access: raise GuardRejection( f"Namespace access not allowed in {context.evaluation} mode" ) else: - raise NameError(f"{node.id} not found in locals nor globals") + raise NameError(f"{node.id} not found in locals, globals, nor builtins") if isinstance(node, ast.Attribute): value = eval_node(node.value, context) if policy.can_get_attr(value, node.attr): @@ -540,7 +549,7 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): func, # not joined to avoid calling `repr` f"not allowed in {context.evaluation} mode", ) - raise ValueError("Unhandled node", node) + raise ValueError("Unhandled node", ast.dump(node)) SUPPORTED_EXTERNAL_GETITEM = { @@ -552,6 +561,7 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext): ("numpy", "void"), } + BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { dict, str, @@ -583,6 +593,8 @@ def _list_methods(cls, source=None): dict_keys: Type[collections.abc.KeysView] = type({}.keys()) method_descriptor: Any = type(list.copy) +NUMERICS = {int, float, complex} + ALLOWED_CALLS = { bytes, *_list_methods(bytes), @@ -600,6 +612,8 @@ def _list_methods(cls, source=None): *_list_methods(str), tuple, *_list_methods(tuple), + *NUMERICS, + *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)], collections.deque, *_list_methods(collections.deque, list_non_mutating_methods), collections.defaultdict, @@ -624,12 +638,13 @@ def _list_methods(cls, source=None): frozenset, object, type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + *NUMERICS, dict_keys, method_descriptor, } -BUILTIN_OPERATIONS = {int, float, complex, *BUILTIN_GETATTR} +BUILTIN_OPERATIONS = {*BUILTIN_GETATTR} EVALUATION_POLICIES = { "minimal": EvaluationPolicy( diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 94f58298665..8d3495a3d6e 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -1,4 +1,5 @@ from typing import NamedTuple +from functools import partial from IPython.core.guarded_eval import ( EvaluationContext, GuardRejection, @@ -9,12 +10,19 @@ import pytest -def limited(**kwargs): - return EvaluationContext(locals=kwargs, globals={}, evaluation="limited") +def create_context(evaluation: str, **kwargs): + return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation) -def unsafe(**kwargs): - return EvaluationContext(locals=kwargs, globals={}, evaluation="unsafe") +forbidden = partial(create_context, "forbidden") +minimal = partial(create_context, "minimal") +limited = partial(create_context, "limited") +unsafe = partial(create_context, "unsafe") +dangerous = partial(create_context, "dangerous") + +LIMITED_OR_HIGHER = [limited, unsafe, dangerous] + +MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] @dec.skip_without("pandas") @@ -142,7 +150,7 @@ def test_set_literal(): assert guarded_eval('{"a"}', context) == {"a"} -def test_if_expression(): +def test_evaluates_if_expression(): context = limited() assert guarded_eval("2 if True else 3", context) == 2 assert guarded_eval("4 if False else 5", context) == 5 @@ -178,7 +186,7 @@ def test_method_descriptor(): [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True], ], ) -def test_calls(data, good, bad, expected): +def test_evaluates_calls(data, good, bad, expected): context = limited(data=data) assert guarded_eval(good, context) == expected @@ -194,9 +202,26 @@ def test_calls(data, good, bad, expected): ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]], ], ) -def test_literals(code, expected): - context = limited() - assert guarded_eval(code, context) == expected +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_complex_cases(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1", 1], + ["1.0", 1.0], + ["0xdeedbeef", 0xDEEDBEEF], + ["True", True], + ["None", None], + ["{}", {}], + ["[]", []], + ], +) +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_evaluates_literals(code, expected, context): + assert guarded_eval(code, context()) == expected @pytest.mark.parametrize( @@ -207,9 +232,9 @@ def test_literals(code, expected): ["~5", -6], ], ) -def test_unary_operations(code, expected): - context = limited() - assert guarded_eval(code, context) == expected +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_unary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected @pytest.mark.parametrize( @@ -228,9 +253,9 @@ def test_unary_operations(code, expected): ["1 & 2", 0], ], ) -def test_binary_operations(code, expected): - context = limited() - assert guarded_eval(code, context) == expected +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_binary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected @pytest.mark.parametrize( @@ -262,16 +287,152 @@ def test_binary_operations(code, expected): ["True is True", True], ["False is False", True], ["True is False", False], + ["True is not True", False], + ["False is not True", True], ], ) -def test_comparisons(code, expected): - context = limited() - assert guarded_eval(code, context) == expected +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_comparisons(code, expected, context): + assert guarded_eval(code, context()) == expected + + +def test_guards_comparisons(): + class GoodEq(int): + pass + + class BadEq(int): + def __eq__(self, other): + assert False + + context = limited(bad=BadEq(1), good=GoodEq(1)) + + with pytest.raises(GuardRejection): + guarded_eval("bad == 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad != 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 == bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 != bad", context) + + assert guarded_eval("good == 1", context) is True + assert guarded_eval("good != 1", context) is False + assert guarded_eval("1 == good", context) is True + assert guarded_eval("1 != good", context) is False + + +def test_guards_unary_operations(): + class GoodOp(int): + pass + + class BadOpInv(int): + def __inv__(self, other): + assert False + + class BadOpInverse(int): + def __inv__(self, other): + assert False + + context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1)) + + with pytest.raises(GuardRejection): + guarded_eval("~bad1", context) + + with pytest.raises(GuardRejection): + guarded_eval("~bad2", context) + + +def test_guards_binary_operations(): + class GoodOp(int): + pass + class BadOp(int): + def __add__(self, other): + assert False -def test_access_builtins(): + context = limited(good=GoodOp(1), bad=BadOp(1)) + + with pytest.raises(GuardRejection): + guarded_eval("1 + bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad + 1", context) + + assert guarded_eval("good + 1", context) == 2 + assert guarded_eval("1 + good", context) == 2 + + +def test_guards_attributes(): + class GoodAttr(float): + pass + + class BadAttr1(float): + def __getattr__(self, key): + assert False + + class BadAttr2(float): + def __getattribute__(self, key): + assert False + + context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5)) + + with pytest.raises(GuardRejection): + guarded_eval("bad1.as_integer_ratio", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad2.as_integer_ratio", context) + + assert guarded_eval("good.as_integer_ratio()", context) == (1, 2) + + +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_access_builtins(context): + assert guarded_eval("round", context()) == round + + +def test_access_builtins_fails(): context = limited() - assert guarded_eval("round", context) == round + with pytest.raises(NameError): + guarded_eval("this_is_not_builtin", context) + + +def test_rejects_forbidden(): + context = forbidden() + with pytest.raises(GuardRejection): + guarded_eval("1", context) + + +def test_guards_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal" + ) + + with pytest.raises(GuardRejection): + guarded_eval("local_a", context) + + with pytest.raises(GuardRejection): + guarded_eval("global_b", context) + + +def test_access_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited" + ) + assert guarded_eval("local_a", context) == "a" + assert guarded_eval("global_b", context) == "b" + + +@pytest.mark.parametrize( + "code", + ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"], +) +@pytest.mark.parametrize("context", [minimal(), limited(), unsafe()]) +def test_rejects_side_effect_syntax(code, context): + with pytest.raises(SyntaxError): + guarded_eval(code, context) def test_subscript(): From add04498bd106451679b75e0bc3a782ad5a13ea5 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 01:55:03 +0000 Subject: [PATCH 040/122] Fix code cov coverage reporting --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53ccb6f78ed..2f4677fb4d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,4 +77,7 @@ jobs: run: | pytest --color=yes -raXxs ${{ startsWith(matrix.python-version, 'pypy') && ' ' || '--cov --cov-report=xml' }} - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 + with: + name: Test + files: /home/runner/work/ipython/ipython/coverage.xml From a6e74d58a693917677bfa4f35d94523a77cd3e35 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 11:31:50 +0000 Subject: [PATCH 041/122] Describe code style checks and working with docs locally --- CONTRIBUTING.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5826baf599c..10bf1efff8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,8 +66,9 @@ Some guidelines on contributing to IPython: If you're making functional changes, you can clean up the specific pieces of code you're working on. -[Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing -IPython and Pull Requests, but it may make sense to manually perform tests, +[GitHub Actions](https://github.com/ipython/ipython/actions/workflows/test.yml) does +a pretty good job testing IPython and Pull Requests, +but it may make sense to manually perform tests, particularly for PRs that affect `IPython.parallel` or Windows. For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow). @@ -88,3 +89,30 @@ Only a single test (for example **test_alias_lifecycle**) within a single file c ```shell pytest IPython/core/tests/test_alias.py::test_alias_lifecycle ``` + +## Code style + +* Before committing run `darker -r 60625f241f298b5039cb2debc365db38aa7bb522 ` to apply selective `black` formatting on modified regions using [darker](https://github.com/akaihola/darker) +* For newly added modules or refactors, please enable static typing analysis with `mypy` for the modified module by adding the file path in [`mypy.yml`](https://github.com/ipython/ipython/blob/main/.github/workflows/mypy.yml) workflow. +* As described in pull requests section, please avoid excessive formatting changes; if formatting-only commit is necessary consider adding its hash to [`.git-blame-ignore-revs`](https://github.com/ipython/ipython/blob/main/.git-blame-ignore-revs) file + +## Documentation + +Sphinx documentation can be built locally using standard sphinx `make` commands. To build HTML documentation from the root of the project, execute: + +```shell +pip install -r docs/requirements.txt # only needed once +make -C docs/ html SPHINXOPTS="-W" +``` + +To force update of the API documentation, precede the `make` command with: + +```shell +python3 docs/autogen_api.py +``` + +Similarly, to force-update the configuration, run: + +```shell +python3 docs/autogen_config.py +``` From 200dc32e519c12aa0bf9bdf6fce27e41c6a02dd0 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 11:32:23 +0000 Subject: [PATCH 042/122] Fix a typo in GH actions URL in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0371848061e..b004792e0e9 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :target: https://pypi.python.org/pypi/ipython .. image:: https://github.com/ipython/ipython/actions/workflows/test.yml/badge.svg - :target: https://github.com/ipython/ipython/actions/workflows/test.yml) + :target: https://github.com/ipython/ipython/actions/workflows/test.yml .. image:: https://www.codetriage.com/ipython/ipython/badges/users.svg :target: https://www.codetriage.com/ipython/ipython/ From 6938ae1204891a9ebb9989f5b266007f1f18d82f Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 12:03:32 +0000 Subject: [PATCH 043/122] Remove unused pytest ignores (files have been deleted) --- pytest.ini | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pytest.ini b/pytest.ini index 81511e9ce51..5cc977692b8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,18 +14,10 @@ addopts = --durations=10 --ignore=IPython/sphinxext --ignore=IPython/terminal/pt_inputhooks --ignore=IPython/__main__.py - --ignore=IPython/config.py - --ignore=IPython/frontend.py - --ignore=IPython/html.py - --ignore=IPython/nbconvert.py - --ignore=IPython/nbformat.py - --ignore=IPython/parallel.py - --ignore=IPython/qt.py --ignore=IPython/external/qt_for_kernel.py --ignore=IPython/html/widgets/widget_link.py --ignore=IPython/html/widgets/widget_output.py --ignore=IPython/terminal/console.py - --ignore=IPython/terminal/ptshell.py --ignore=IPython/utils/_process_cli.py --ignore=IPython/utils/_process_posix.py --ignore=IPython/utils/_process_win32.py From a96912d4e98ea2da4f0e8d16e9d51714575b2bf2 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 16:16:04 +0000 Subject: [PATCH 044/122] Guard against custom properties --- IPython/core/guarded_eval.py | 65 ++++++++++++++++++++----- IPython/core/tests/test_guarded_eval.py | 49 ++++++++++++++++++- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index f71a73bea58..9f391e35a69 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -3,6 +3,7 @@ Callable, Dict, Set, + Sequence, Tuple, NamedTuple, Type, @@ -113,18 +114,30 @@ def can_call(self, func): return True +def _get_external(module_name: str, access_path: Sequence[str]): + """Get value from external module given a dotted access path. + + Raises: + * `KeyError` if module is removed not found, and + * `AttributeError` if acess path does not match an exported object + """ + member_type = sys.modules[module_name] + for attr in access_path: + member_type = getattr(member_type, attr) + return member_type + + def _has_original_dunder_external( value, - module_name, - access_path, - method_name, + module_name: str, + access_path: Sequence[str], + method_name: str, ): + if module_name not in sys.modules: + # LBYLB as it is faster + return False try: - if module_name not in sys.modules: - return False - member_type = sys.modules[module_name] - for attr in access_path: - member_type = getattr(member_type, attr) + member_type = _get_external(module_name, access_path) value_type = type(value) if type(value) == member_type: return True @@ -199,12 +212,42 @@ def can_get_attr(self, value, attr): method_name="__getattr__", ) + accept = False + # Many objects do not have `__getattr__`, this is fine if has_original_attr is None and has_original_attribute: - return True + accept = True + else: + # Accept objects without modifications to `__getattr__` and `__getattribute__` + accept = has_original_attr and has_original_attribute + + if accept: + # We still need to check for overriden properties. - # Accept objects without modifications to `__getattr__` and `__getattribute__` - return has_original_attr and has_original_attribute + value_class = type(value) + if not hasattr(value_class, attr): + return True + + class_attr_val = getattr(value_class, attr) + is_property = isinstance(class_attr_val, property) + + if not is_property: + return True + + # Properties in allowed types are ok + if type(value) in self.allowed_getattr: + return True + + # Properties in subclasses of allowed types may be ok if not changed + for module_name, *access_path in self.allowed_getattr_external: + try: + external_class = _get_external(module_name, access_path) + external_class_attr_val = getattr(external_class, attr) + except (KeyError, AttributeError): + return False # pragma: no cover + return class_attr_val == external_class_attr_val + + return False def can_get_item(self, value, item): """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 8d3495a3d6e..1ee93ffe744 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import NamedTuple from functools import partial from IPython.core.guarded_eval import ( @@ -25,6 +26,21 @@ def create_context(evaluation: str, **kwargs): MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] +@contextmanager +def module_not_installed(module: str): + import sys + + try: + to_restore = sys.modules[module] + del sys.modules[module] + except KeyError: + to_restore = None + try: + yield + finally: + sys.modules[module] = to_restore + + @dec.skip_without("pandas") def test_pandas_series_iloc(): import pandas as pd @@ -34,6 +50,32 @@ def test_pandas_series_iloc(): assert guarded_eval("data.iloc[0]", context) == 1 +def test_rejects_custom_properties(): + class BadProperty: + @property + def iloc(self): + return [None] + + series = BadProperty() + context = limited(data=series) + + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + +@dec.skip_without("pandas") +def test_accepts_non_overriden_properties(): + import pandas as pd + + class GoodProperty(pd.Series): + pass + + series = GoodProperty([1], index=["a"]) + context = limited(data=series) + + assert guarded_eval("data.iloc[0]", context) == 1 + + @dec.skip_without("pandas") def test_pandas_series(): import pandas as pd @@ -472,9 +514,12 @@ def __getitem__(self, k): def __getattr__(self, k): return "a" + def f(self): + return "b" + t = T() - t.__getitem__ = lambda f: "b" - t.__getattr__ = lambda f: "b" + t.__getitem__ = f + t.__getattr__ = f assert t[1] == "a" assert t[1] == "a" From 52cda652f1a5c71f2117bf84f33f0cc99e59e7da Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 16:16:38 +0000 Subject: [PATCH 045/122] Increase coverage for completer tests --- IPython/core/tests/test_completer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 423979a297f..5e8cb35bc33 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -886,6 +886,12 @@ def match(*args, **kwargs): assert match(keys, "2") == ("", 0, ["21", "22"]) assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"]) + # Should yield on variables + assert match(keys, "a_variable") == ("", 0, []) + + # Should pass over invalid literals + assert match(keys, "'' ''") == ("", 0, []) + def test_match_dict_keys_tuple(self): """ Test that match_dict_keys called with extra prefix works on a couple of use case, @@ -1687,6 +1693,9 @@ def _(expected): ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"], ["0xdeadbeef", "0xdeadbeef"], ["0b_1110_0101", "0b_1110_0101"], + # should not match if in an operation + ["1 + 1", None], + [", 1 + 1", None], ], ) def test_match_numeric_literal_for_dict_key(input, expected): From 7f95861a8736657bd78e36c14157c3962e65421c Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 16:17:33 +0000 Subject: [PATCH 046/122] Remove outdated TODO comment --- IPython/core/guarded_eval.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index 9f391e35a69..c2d88d0473a 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -701,8 +701,6 @@ def _list_methods(cls, source=None): allow_all_operations=False, ), "limited": SelectivePolicy( - # TODO: - # - should reject binary and unary operations if custom methods would be dispatched allowed_getitem=BUILTIN_GETITEM, allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, allowed_getattr=BUILTIN_GETATTR, From abe32e8d2346f2badccfef6f84f7520365ece346 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 4 Dec 2022 16:58:39 +0000 Subject: [PATCH 047/122] Add more tests --- IPython/core/guarded_eval.py | 10 ++++--- IPython/core/tests/test_guarded_eval.py | 35 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py index c2d88d0473a..d60a5c5f1e3 100644 --- a/IPython/core/guarded_eval.py +++ b/IPython/core/guarded_eval.py @@ -60,7 +60,8 @@ class DoesNotHaveGetAttr(Protocol): def _unbind_method(func: Callable) -> Union[Callable, None]: """Get unbound method for given bound method. - Returns None if cannot get unbound method.""" + Returns None if cannot get unbound method, or method is already unbound. + """ owner = getattr(func, "__self__", None) owner_class = type(owner) name = getattr(func, "__name__", None) @@ -214,7 +215,7 @@ def can_get_attr(self, value, attr): accept = False - # Many objects do not have `__getattr__`, this is fine + # Many objects do not have `__getattr__`, this is fine. if has_original_attr is None and has_original_attribute: accept = True else: @@ -234,9 +235,10 @@ def can_get_attr(self, value, attr): if not is_property: return True - # Properties in allowed types are ok + # Properties in allowed types are ok (although we do not include any + # properties in our default allow list currently). if type(value) in self.allowed_getattr: - return True + return True # pragma: no cover # Properties in subclasses of allowed types may be ok if not changed for module_name, *access_path in self.allowed_getattr_external: diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py index 1ee93ffe744..905cf3ab8e3 100644 --- a/IPython/core/tests/test_guarded_eval.py +++ b/IPython/core/tests/test_guarded_eval.py @@ -22,7 +22,6 @@ def create_context(evaluation: str, **kwargs): dangerous = partial(create_context, "dangerous") LIMITED_OR_HIGHER = [limited, unsafe, dangerous] - MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] @@ -41,6 +40,39 @@ def module_not_installed(module: str): sys.modules[module] = to_restore +def test_external_not_installed(): + """ + Because attribute check requires checking if object is not of allowed + external type, this tests logic for absence of external module. + """ + + class Custom: + def __init__(self): + self.test = 1 + + def __getattr__(self, key): + return key + + with module_not_installed("pandas"): + context = limited(x=Custom()) + with pytest.raises(GuardRejection): + guarded_eval("x.test", context) + + +@dec.skip_without("pandas") +def test_external_changed_api(monkeypatch): + """Check that the execution rejects if external API changed paths""" + import pandas as pd + + series = pd.Series([1], index=["a"]) + + with monkeypatch.context() as m: + m.delattr(pd, "Series") + context = limited(data=series) + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + @dec.skip_without("pandas") def test_pandas_series_iloc(): import pandas as pd @@ -496,6 +528,7 @@ def index(self, k): x = X() assert _unbind_method(x.index) is X.index assert _unbind_method([].index) is list.index + assert _unbind_method(list.index) is None def test_assumption_instance_attr_do_not_matter(): From 9632124e4e561e99d96304a672854be0d4cb6e16 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Wed, 7 Dec 2022 10:22:54 -0700 Subject: [PATCH 048/122] Add py.typed to setup.cfg --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 226506f08f0..74bbd95193b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,7 @@ exclude = setupext [options.package_data] +IPython = py.typed IPython.core = profile/README* IPython.core.tests = *.png, *.jpg, daft_extension/*.py IPython.lib.tests = *.wav From 86e1bdf022a2b181a690d77e97a78f20bd8ecab9 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 11:07:08 -0700 Subject: [PATCH 049/122] Refactor Inspector._get_info to make it easier to subclass and modify behavior. This factors out the logic to append an info field to the inspect_reply mimebundle and the logic for what information is in the mimebundle into separate functions that can easily be overridden by an inspector subclass. This allows a subclass to easily: * format information into yet another mimetype besides text/plain and text/html * modify or add to the default information without having to copy the default implementation. --- IPython/core/oinspect.py | 143 +++++++++++++++++++++------------------ 1 file changed, 78 insertions(+), 65 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index f1c454b2604..13d6268256d 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -552,56 +552,46 @@ def _mime_format(self, text:str, formatter=None) -> dict: def format_mime(self, bundle): - + """Format a mimebundle being created by _make_info_unformatted into a real mimebundle""" + # First, format the field names and values for the text/plain field text_plain = bundle['text/plain'] + if isinstance(text_plain, (list, tuple)): + text = '' + heads, bodies = list(zip(*text_plain)) + _len = max(len(h) for h in heads) - text = '' - heads, bodies = list(zip(*text_plain)) - _len = max(len(h) for h in heads) + for head, body in zip(heads, bodies): + body = body.strip('\n') + delim = '\n' if '\n' in body else ' ' + text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n' - for head, body in zip(heads, bodies): - body = body.strip('\n') - delim = '\n' if '\n' in body else ' ' - text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n' + bundle['text/plain'] = text - bundle['text/plain'] = text + # Next format the text/html value by joining strings if it is a list of strings + if isinstance(bundle['text/html'], (list, tuple)): + bundle['text/html'] = '\n'.join(bundle['text/html']) return bundle - def _get_info( - self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() - ): - """Retrieve an info dict and format it. - - Parameters - ---------- - obj : any - Object to inspect and return info from - oname : str (default: ''): - Name of the variable pointing to `obj`. - formatter : callable - info - already computed information - detail_level : integer - Granularity of detail level, if set to 1, give more information. - omit_sections : container[str] - Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) - """ - - info = self.info(obj, oname=oname, info=info, detail_level=detail_level) - - _mime = { + def _append_info_field(self, bundle, title:str, key:str, info, omit_sections, formatter): + """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted""" + if title in omit_sections or key in omit_sections: + return + field = info[key] + if field is not None: + formatted_field = self._mime_format(field, formatter) + bundle['text/plain'].append((title, formatted_field['text/plain'])) + bundle['text/html'] += '

' + title + '

\n' + formatted_field['text/html'] + + def _make_info_unformatted(self, info, formatter, detail_level, omit_sections): + """Assemble the mimebundle as unformatted lists of information""" + bundle = { 'text/plain': [], - 'text/html': '', + 'text/html': [], } + # A convenience function to simplify calls below def append_field(bundle, title:str, key:str, formatter=None): - if title in omit_sections or key in omit_sections: - return - field = info[key] - if field is not None: - formatted_field = self._mime_format(field, formatter) - bundle['text/plain'].append((title, formatted_field['text/plain'])) - bundle['text/html'] += '

' + title + '

\n' + formatted_field['text/html'] + '\n' + self._append_info_field(bundle, title=title, key=key, info=info, omit_sections=omit_sections, formatter=formatter) def code_formatter(text): return { @@ -610,56 +600,79 @@ def code_formatter(text): } if info['isalias']: - append_field(_mime, 'Repr', 'string_form') + append_field(bundle, 'Repr', 'string_form') elif info['ismagic']: if detail_level > 0: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, 'Source', 'source', code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) - append_field(_mime, 'File', 'file') + append_field(bundle, 'Docstring', 'docstring', formatter) + append_field(bundle, 'File', 'file') elif info['isclass'] or is_simple_callable(obj): # Functions, methods, classes - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Init signature', 'init_definition', code_formatter) - append_field(_mime, 'Docstring', 'docstring', formatter) + append_field(bundle, 'Signature', 'definition', code_formatter) + append_field(bundle, 'Init signature', 'init_definition', code_formatter) + append_field(bundle, 'Docstring', 'docstring', formatter) if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, 'Source', 'source', code_formatter) else: - append_field(_mime, 'Init docstring', 'init_docstring', formatter) + append_field(bundle, 'Init docstring', 'init_docstring', formatter) - append_field(_mime, 'File', 'file') - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'Subclasses', 'subclasses') + append_field(bundle, 'File', 'file') + append_field(bundle, 'Type', 'type_name') + append_field(bundle, 'Subclasses', 'subclasses') else: # General Python objects - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Call signature', 'call_def', code_formatter) - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'String form', 'string_form') + append_field(bundle, 'Signature', 'definition', code_formatter) + append_field(bundle, 'Call signature', 'call_def', code_formatter) + append_field(bundle, 'Type', 'type_name') + append_field(bundle, 'String form', 'string_form') # Namespace if info['namespace'] != 'Interactive': - append_field(_mime, 'Namespace', 'namespace') + append_field(bundle, 'Namespace', 'namespace') - append_field(_mime, 'Length', 'length') - append_field(_mime, 'File', 'file') + append_field(bundle, 'Length', 'length') + append_field(bundle, 'File', 'file') # Source or docstring, depending on detail level and whether # source found. if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, 'Source', 'source', code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) + append_field(bundle, 'Docstring', 'docstring', formatter) + + append_field(bundle, 'Class docstring', 'class_docstring', formatter) + append_field(bundle, 'Init docstring', 'init_docstring', formatter) + append_field(bundle, 'Call docstring', 'call_docstring', formatter) + return bundle - append_field(_mime, 'Class docstring', 'class_docstring', formatter) - append_field(_mime, 'Init docstring', 'init_docstring', formatter) - append_field(_mime, 'Call docstring', 'call_docstring', formatter) + def _get_info( + self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + ): + """Retrieve an info dict and format it. - return self.format_mime(_mime) + Parameters + ---------- + obj : any + Object to inspect and return info from + oname : str (default: ''): + Name of the variable pointing to `obj`. + formatter : callable + info + already computed information + detail_level : integer + Granularity of detail level, if set to 1, give more information. + omit_sections : container[str] + Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) + """ + + info = self.info(obj, oname=oname, info=info, detail_level=detail_level) + bundle = self._make_info_unformatted(info, formatter, detail_level=detail_level, omit_sections=omit_sections) + return self.format_mime(bundle) def pinfo( self, From eae299e8c9d1471eca2c76f1e8e19765fde556a2 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 11:17:05 -0700 Subject: [PATCH 050/122] Move formatting of inspect reply html into the format_mime function Also simplify the formatting of the inspect reply text/plain logic --- IPython/core/oinspect.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 13d6268256d..3a80f05a4e1 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -553,23 +553,23 @@ def _mime_format(self, text:str, formatter=None) -> dict: def format_mime(self, bundle): """Format a mimebundle being created by _make_info_unformatted into a real mimebundle""" - # First, format the field names and values for the text/plain field - text_plain = bundle['text/plain'] - if isinstance(text_plain, (list, tuple)): - text = '' - heads, bodies = list(zip(*text_plain)) - _len = max(len(h) for h in heads) - - for head, body in zip(heads, bodies): + # Format text/plain mimetype + if isinstance(bundle['text/plain'], (list, tuple)): + # bundle['text/plain'] is a list of (head, formatted body) pairs + lines = [] + _len = max(len(h) for h,_ in bundle['text/plain']) + + for head, body in bundle['text/plain']: body = body.strip('\n') delim = '\n' if '\n' in body else ' ' - text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n' + lines.append(f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}") - bundle['text/plain'] = text + bundle['text/plain'] = '\n'.join(lines) - # Next format the text/html value by joining strings if it is a list of strings + # Format the text/html mimetype if isinstance(bundle['text/html'], (list, tuple)): - bundle['text/html'] = '\n'.join(bundle['text/html']) + # bundle['text/html'] is a list of (head, formatted body) pairs + bundle['text/html'] = '\n'.join((f'

{head}

\n{body}' for (head,body) in bundle['text/html'])) return bundle def _append_info_field(self, bundle, title:str, key:str, info, omit_sections, formatter): @@ -580,7 +580,7 @@ def _append_info_field(self, bundle, title:str, key:str, info, omit_sections, fo if field is not None: formatted_field = self._mime_format(field, formatter) bundle['text/plain'].append((title, formatted_field['text/plain'])) - bundle['text/html'] += '

' + title + '

\n' + formatted_field['text/html'] + bundle['text/html'].append((title, formatted_field['text/html'])) def _make_info_unformatted(self, info, formatter, detail_level, omit_sections): """Assemble the mimebundle as unformatted lists of information""" From 5e7b9d5506ff0f2891e10248255394592cc779b9 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 11:19:17 -0700 Subject: [PATCH 051/122] Lint --- IPython/core/oinspect.py | 105 ++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 3a80f05a4e1..1227fd07c66 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -554,44 +554,57 @@ def _mime_format(self, text:str, formatter=None) -> dict: def format_mime(self, bundle): """Format a mimebundle being created by _make_info_unformatted into a real mimebundle""" # Format text/plain mimetype - if isinstance(bundle['text/plain'], (list, tuple)): + if isinstance(bundle["text/plain"], (list, tuple)): # bundle['text/plain'] is a list of (head, formatted body) pairs lines = [] - _len = max(len(h) for h,_ in bundle['text/plain']) + _len = max(len(h) for h, _ in bundle["text/plain"]) - for head, body in bundle['text/plain']: - body = body.strip('\n') - delim = '\n' if '\n' in body else ' ' - lines.append(f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}") + for head, body in bundle["text/plain"]: + body = body.strip("\n") + delim = "\n" if "\n" in body else " " + lines.append( + f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}" + ) - bundle['text/plain'] = '\n'.join(lines) + bundle["text/plain"] = "\n".join(lines) # Format the text/html mimetype - if isinstance(bundle['text/html'], (list, tuple)): + if isinstance(bundle["text/html"], (list, tuple)): # bundle['text/html'] is a list of (head, formatted body) pairs - bundle['text/html'] = '\n'.join((f'

{head}

\n{body}' for (head,body) in bundle['text/html'])) + bundle["text/html"] = "\n".join( + (f"

{head}

\n{body}" for (head, body) in bundle["text/html"]) + ) return bundle - def _append_info_field(self, bundle, title:str, key:str, info, omit_sections, formatter): + def _append_info_field( + self, bundle, title: str, key: str, info, omit_sections, formatter + ): """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted""" if title in omit_sections or key in omit_sections: return field = info[key] if field is not None: formatted_field = self._mime_format(field, formatter) - bundle['text/plain'].append((title, formatted_field['text/plain'])) - bundle['text/html'].append((title, formatted_field['text/html'])) + bundle["text/plain"].append((title, formatted_field["text/plain"])) + bundle["text/html"].append((title, formatted_field["text/html"])) def _make_info_unformatted(self, info, formatter, detail_level, omit_sections): """Assemble the mimebundle as unformatted lists of information""" bundle = { - 'text/plain': [], - 'text/html': [], + "text/plain": [], + "text/html": [], } # A convenience function to simplify calls below - def append_field(bundle, title:str, key:str, formatter=None): - self._append_info_field(bundle, title=title, key=key, info=info, omit_sections=omit_sections, formatter=formatter) + def append_field(bundle, title: str, key: str, formatter=None): + self._append_info_field( + bundle, + title=title, + key=key, + info=info, + omit_sections=omit_sections, + formatter=formatter, + ) def code_formatter(text): return { @@ -599,54 +612,54 @@ def code_formatter(text): 'text/html': pylight(text) } - if info['isalias']: - append_field(bundle, 'Repr', 'string_form') + if info["isalias"]: + append_field(bundle, "Repr", "string_form") elif info['ismagic']: if detail_level > 0: - append_field(bundle, 'Source', 'source', code_formatter) + append_field(bundle, "Source", "source", code_formatter) else: - append_field(bundle, 'Docstring', 'docstring', formatter) - append_field(bundle, 'File', 'file') + append_field(bundle, "Docstring", "docstring", formatter) + append_field(bundle, "File", "file") elif info['isclass'] or is_simple_callable(obj): # Functions, methods, classes - append_field(bundle, 'Signature', 'definition', code_formatter) - append_field(bundle, 'Init signature', 'init_definition', code_formatter) - append_field(bundle, 'Docstring', 'docstring', formatter) - if detail_level > 0 and info['source']: - append_field(bundle, 'Source', 'source', code_formatter) + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Init signature", "init_definition", code_formatter) + append_field(bundle, "Docstring", "docstring", formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(bundle, 'Init docstring', 'init_docstring', formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) - append_field(bundle, 'File', 'file') - append_field(bundle, 'Type', 'type_name') - append_field(bundle, 'Subclasses', 'subclasses') + append_field(bundle, "File", "file") + append_field(bundle, "Type", "type_name") + append_field(bundle, "Subclasses", "subclasses") else: # General Python objects - append_field(bundle, 'Signature', 'definition', code_formatter) - append_field(bundle, 'Call signature', 'call_def', code_formatter) - append_field(bundle, 'Type', 'type_name') - append_field(bundle, 'String form', 'string_form') + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Call signature", "call_def", code_formatter) + append_field(bundle, "Type", "type_name") + append_field(bundle, "String form", "string_form") # Namespace - if info['namespace'] != 'Interactive': - append_field(bundle, 'Namespace', 'namespace') + if info["namespace"] != "Interactive": + append_field(bundle, "Namespace", "namespace") - append_field(bundle, 'Length', 'length') - append_field(bundle, 'File', 'file') + append_field(bundle, "Length", "length") + append_field(bundle, "File", "file") # Source or docstring, depending on detail level and whether # source found. - if detail_level > 0 and info['source']: - append_field(bundle, 'Source', 'source', code_formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(bundle, 'Docstring', 'docstring', formatter) + append_field(bundle, "Docstring", "docstring", formatter) - append_field(bundle, 'Class docstring', 'class_docstring', formatter) - append_field(bundle, 'Init docstring', 'init_docstring', formatter) - append_field(bundle, 'Call docstring', 'call_docstring', formatter) + append_field(bundle, "Class docstring", "class_docstring", formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) + append_field(bundle, "Call docstring", "call_docstring", formatter) return bundle @@ -671,7 +684,9 @@ def _get_info( """ info = self.info(obj, oname=oname, info=info, detail_level=detail_level) - bundle = self._make_info_unformatted(info, formatter, detail_level=detail_level, omit_sections=omit_sections) + bundle = self._make_info_unformatted( + info, formatter, detail_level=detail_level, omit_sections=omit_sections + ) return self.format_mime(bundle) def pinfo( From 5db69068fe3256b29f575deb68ecca5287cefafc Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 11:24:40 -0700 Subject: [PATCH 052/122] Fix missing argument in _make_info_unformatted --- IPython/core/oinspect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 1227fd07c66..43f6707dd36 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -588,7 +588,7 @@ def _append_info_field( bundle["text/plain"].append((title, formatted_field["text/plain"])) bundle["text/html"].append((title, formatted_field["text/html"])) - def _make_info_unformatted(self, info, formatter, detail_level, omit_sections): + def _make_info_unformatted(self, obj, info, formatter, detail_level, omit_sections): """Assemble the mimebundle as unformatted lists of information""" bundle = { "text/plain": [], @@ -685,7 +685,7 @@ def _get_info( info = self.info(obj, oname=oname, info=info, detail_level=detail_level) bundle = self._make_info_unformatted( - info, formatter, detail_level=detail_level, omit_sections=omit_sections + obj, info, formatter, detail_level=detail_level, omit_sections=omit_sections ) return self.format_mime(bundle) From 99b83f140ac70d6307382b2fd238844baaf5c957 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 13:18:08 -0700 Subject: [PATCH 053/122] Escape html text in inspect replies by default Otherwise, the default docstring was not displayed in html because it was acting as an html tag. --- IPython/core/oinspect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 43f6707dd36..801cb880659 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -16,6 +16,7 @@ import ast import inspect from inspect import signature +import html import linecache import warnings import os @@ -531,7 +532,7 @@ def _mime_format(self, text:str, formatter=None) -> dict: """ defaults = { 'text/plain': text, - 'text/html': '
' + text + '
' + 'text/html': '
' + html.escape(text) + '
' } if formatter is None: From 08b767f70d80677fbd2205014f9b495c568738de Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 12:32:45 -0800 Subject: [PATCH 054/122] Minor formatting changes --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10bf1efff8f..164757fb350 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,9 +92,9 @@ pytest IPython/core/tests/test_alias.py::test_alias_lifecycle ## Code style -* Before committing run `darker -r 60625f241f298b5039cb2debc365db38aa7bb522 ` to apply selective `black` formatting on modified regions using [darker](https://github.com/akaihola/darker) +* Before committing, run `darker -r 60625f241f298b5039cb2debc365db38aa7bb522 ` to apply selective `black` formatting on modified regions using [darker](https://github.com/akaihola/darker). * For newly added modules or refactors, please enable static typing analysis with `mypy` for the modified module by adding the file path in [`mypy.yml`](https://github.com/ipython/ipython/blob/main/.github/workflows/mypy.yml) workflow. -* As described in pull requests section, please avoid excessive formatting changes; if formatting-only commit is necessary consider adding its hash to [`.git-blame-ignore-revs`](https://github.com/ipython/ipython/blob/main/.git-blame-ignore-revs) file +* As described in the pull requests section, please avoid excessive formatting changes; if a formatting-only commit is necessary, consider adding its hash to [`.git-blame-ignore-revs`](https://github.com/ipython/ipython/blob/main/.git-blame-ignore-revs) file. ## Documentation From ea239fb8c60c12e8db051cb64fd09d74e1684bc3 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 9 Dec 2022 13:51:11 -0700 Subject: [PATCH 055/122] Lint + change to f strings --- IPython/core/oinspect.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 801cb880659..bcaa95c97fa 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -531,8 +531,8 @@ def _mime_format(self, text:str, formatter=None) -> dict: """ defaults = { - 'text/plain': text, - 'text/html': '
' + html.escape(text) + '
' + "text/plain": text, + "text/html": f"
{html.escape(text)}
", } if formatter is None: @@ -543,10 +543,7 @@ def _mime_format(self, text:str, formatter=None) -> dict: if not isinstance(formatted, dict): # Handle the deprecated behavior of a formatter returning # a string instead of a mime bundle. - return { - 'text/plain': formatted, - 'text/html': '
' + formatted + '
' - } + return {"text/plain": formatted, "text/html": f"
{formatted}
"} else: return dict(defaults, **formatted) From fdb27756cf4b9bb7bbacb725e9eeb735a14bae42 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 17 Nov 2022 13:26:11 -0700 Subject: [PATCH 056/122] Make inspector class a configurable attribute in InteractiveShell. --- IPython/core/interactiveshell.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 21e428b54d4..fa0614d6139 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -389,6 +389,9 @@ def _import_runner(self, proposal): displayhook_class = Type(DisplayHook) display_pub_class = Type(DisplayPublisher) compiler_class = Type(CachingCompiler) + inspector_class = Type( + oinspect.Inspector, help="Class to use to instantiate the shell inspector" + ).tag(config=True) sphinxify_docstring = Bool(False, help= """ @@ -755,10 +758,12 @@ def init_builtins(self): @observe('colors') def init_inspector(self, changes=None): # Object inspector - self.inspector = oinspect.Inspector(oinspect.InspectColors, - PyColorize.ANSICodeColors, - self.colors, - self.object_info_string_level) + self.inspector = self.inspector_class( + oinspect.InspectColors, + PyColorize.ANSICodeColors, + self.colors, + self.object_info_string_level, + ) def init_io(self): # implemented in subclasses, TerminalInteractiveShell does call From 4182eee632b721ef5d70aa6bd6053920d8c1e42f Mon Sep 17 00:00:00 2001 From: nfgf Date: Sat, 10 Dec 2022 12:04:12 -0500 Subject: [PATCH 057/122] Semicolon at the end silence output of %%time too. --- IPython/core/interactiveshell.py | 9 +++++++++ IPython/core/tests/test_magic.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 12503e9d916..f158ef21c4f 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2423,6 +2423,14 @@ def run_cell_magic(self, magic_name, line, cell): with self.builtin_trap: args = (magic_arg_s, cell) result = fn(*args, **kwargs) + + # The code below prevents the output from being displayed + # when using magics with decodator @output_can_be_silenced + # when the last Python token in the expression is a ';'. + if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): + if DisplayHook.semicolon_at_end_of_expression(cell): + return None + return result def find_line_magic(self, magic_name): @@ -3199,6 +3207,7 @@ def error_before_exec(value): # Execute the user code interactivity = "none" if silent else self.ast_node_interactivity + has_raised = await self.run_ast_nodes(code_ast.body, cell_name, interactivity=interactivity, compiler=compiler, result=result) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 55408d4af1e..e64b959322b 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -422,6 +422,7 @@ def test_time(): def test_time_no_output_with_semicolon(): ip = get_ipython() + # Test %time cases with tt.AssertPrints(" 123456"): with tt.AssertPrints("Wall time: ", suppress=False): with tt.AssertPrints("CPU times: ", suppress=False): @@ -447,6 +448,32 @@ def test_time_no_output_with_semicolon(): with tt.AssertPrints("CPU times: ", suppress=False): ip.run_cell("%time 123000+456 # ;Comment") + # Test %%time cases + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456\n\n\n") + + with tt.AssertNotPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456;\n\n\n") + + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456 # Comment\n\n\n") + + with tt.AssertNotPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456; # Comment\n\n\n") + + with tt.AssertPrints("123456"): + with tt.AssertPrints("Wall time: ", suppress=False): + with tt.AssertPrints("CPU times: ", suppress=False): + ip.run_cell("%%time\n123000+456 # ;Comment\n\n\n") + def test_time_last_not_expression(): ip.run_cell("%%time\n" From 661d6d7c8212dd0b6d35853bf0a1fb7d1aad545e Mon Sep 17 00:00:00 2001 From: Takumasa Nakamura Date: Fri, 16 Dec 2022 12:26:55 +0900 Subject: [PATCH 058/122] Fix paste/cpaste magic --- IPython/core/interactiveshell.py | 6 +++++- IPython/terminal/magics.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 21e428b54d4..e137c922d75 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -3138,8 +3138,12 @@ def error_before_exec(value): else: cell = raw_cell + # Do NOT store paste/cpaste magic history + if "get_ipython().run_line_magic(" in cell and "paste" in cell: + store_history = False + # Store raw and processed history - if store_history and raw_cell.strip(" %") != "paste": + if store_history: self.history_manager.store_inputs(self.execution_count, cell, raw_cell) if not silent: self.logger.log(cell, raw_cell) diff --git a/IPython/terminal/magics.py b/IPython/terminal/magics.py index 66d532511b3..cea53e4a248 100644 --- a/IPython/terminal/magics.py +++ b/IPython/terminal/magics.py @@ -147,7 +147,7 @@ def cpaste(self, parameter_s=''): sentinel = opts.get('s', u'--') block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet)) - self.store_or_execute(block, name, store_history=False) + self.store_or_execute(block, name, store_history=True) @line_magic def paste(self, parameter_s=''): From 3455b5738de94fb4587d507a560424188acabb7e Mon Sep 17 00:00:00 2001 From: azjps Date: Mon, 19 Dec 2022 21:59:21 +1300 Subject: [PATCH 059/122] Set up shell command-line tab-completion for ipython Set up shell command-line tab-completion using argcomplete and ipython/traitlets#811 argcomplete supports following setuptools console_scripts to the corresponding package's __main__.py to look for a PYTHON_ARGCOMPLETE_OK marker. --- IPython/__init__.py | 1 + IPython/__main__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/IPython/__init__.py b/IPython/__init__.py index 03b3116a98a..c224f9a8c90 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK """ IPython: tools for interactive and parallel computing in Python. diff --git a/IPython/__main__.py b/IPython/__main__.py index d5123f33a20..8e9f989a82c 100644 --- a/IPython/__main__.py +++ b/IPython/__main__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK # encoding: utf-8 """Terminal-based IPython entry point. """ From a7cebe885813722fe432cf05d9effe59e05d5856 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Thu, 22 Dec 2022 16:08:13 +0000 Subject: [PATCH 060/122] Fix warning in docs build --- IPython/core/completer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index f2853d3c48a..4edeb93426b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -59,7 +59,8 @@ Both forward and backward completions can be deactivated by setting the -:any:`Completer.backslash_combining_completions` option to ``False``. +:std:configtrait:`Completer.backslash_combining_completions` option to +``False``. Experimental @@ -166,7 +167,7 @@ should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key. The suppression behaviour can is user-configurable via -:any:`IPCompleter.suppress_competing_matchers`. +:std:configtrait:`IPCompleter.suppress_competing_matchers`. """ @@ -972,7 +973,7 @@ class Completer(Configurable): help="""Activate greedy completion. .. deprecated:: 8.8 - Use :any:`Completer.evaluation` and :any:`Completer.auto_close_dict_keys` instead. + Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead. When enabled in IPython 8.8 or newer, changes configuration as follows: From f2b3fab00c6fdb56cd5512d457182d1dda4012ca Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 23 Dec 2022 11:16:53 +0100 Subject: [PATCH 061/122] Test on more recent Python versions. --- .github/workflows/mypy.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e05678f724d..52f3e79ab9c 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ["3.x"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f4677fb4d2..73968555c4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] deps: [test_extra] # Test all on ubuntu, test ends on macos include: @@ -27,15 +27,15 @@ jobs: python-version: "3.8" deps: test_extra - os: macos-latest - python-version: "3.10" + python-version: "3.11" deps: test_extra # Tests minimal dependencies set - os: ubuntu-latest - python-version: "3.10" + python-version: "3.11" deps: test # Tests latest development Python version - os: ubuntu-latest - python-version: "3.11-dev" + python-version: "3.12-dev" deps: test # Installing optional dependencies stuff takes ages on PyPy - os: ubuntu-latest From 6cd25549de0ef4565ef70dfa8170833966baca6d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 23 Dec 2022 11:16:53 +0100 Subject: [PATCH 062/122] Test on more recent Python versions. --- .github/workflows/mypy.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e05678f724d..52f3e79ab9c 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ["3.x"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f4677fb4d2..73968555c4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] deps: [test_extra] # Test all on ubuntu, test ends on macos include: @@ -27,15 +27,15 @@ jobs: python-version: "3.8" deps: test_extra - os: macos-latest - python-version: "3.10" + python-version: "3.11" deps: test_extra # Tests minimal dependencies set - os: ubuntu-latest - python-version: "3.10" + python-version: "3.11" deps: test # Tests latest development Python version - os: ubuntu-latest - python-version: "3.11-dev" + python-version: "3.12-dev" deps: test # Installing optional dependencies stuff takes ages on PyPy - os: ubuntu-latest From 023a594a32450ad02d771de6f83805f3bcd1662a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 23 Dec 2022 19:04:09 +0100 Subject: [PATCH 063/122] extend unicode for Python 3.12 --- IPython/core/completer.py | 2 +- IPython/core/tests/test_completer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 4edeb93426b..5ba8ea0fdbd 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -285,7 +285,7 @@ def cast(type_, obj): # write this). With below range we cover them all, with a density of ~67% # biggest next gap we consider only adds up about 1% density and there are 600 # gaps that would need hard coding. -_UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)] +_UNICODE_RANGES = [(32, 0x323B0), (0xE0001, 0xE01F0)] # Public API __all__ = ["Completer", "IPCompleter"] diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 5e8cb35bc33..7783798eb36 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -99,7 +99,7 @@ def test_unicode_range(): assert len_exp == len_test, message # fail if new unicode symbols have been added. - assert len_exp <= 138552, message + assert len_exp <= 143041, message @contextmanager From 02894af40a93c28073ab84a14f5f396204cbb823 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 23 Dec 2022 19:44:09 +0100 Subject: [PATCH 064/122] skip test on 3.12 --- IPython/extensions/tests/test_autoreload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 88637fbab9c..2c3c9db212d 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -367,7 +367,8 @@ class TestEnum(Enum): self.shell.run_code("assert func2() == 'changed'") self.shell.run_code("t = Test(); assert t.new_func() == 'changed'") self.shell.run_code("assert number == 1") - self.shell.run_code("assert TestEnum.B.value == 'added'") + if sys.version_info < (3, 12): + self.shell.run_code("assert TestEnum.B.value == 'added'") # ----------- TEST IMPORT FROM MODULE -------------------------- From ed7f35f8b721d4b4dcafea173ce724bee25704c7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 11:57:18 +0100 Subject: [PATCH 065/122] Fix tests for pygments > 2.14 Pygments 2.14+ have the bash lexer return some tokens as Text.Whitespace instead of Text, this update the test to support this. --- IPython/lib/tests/test_lexers.py | 52 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/IPython/lib/tests/test_lexers.py b/IPython/lib/tests/test_lexers.py index efa00d601ea..000b8fe6fd9 100644 --- a/IPython/lib/tests/test_lexers.py +++ b/IPython/lib/tests/test_lexers.py @@ -4,11 +4,14 @@ # Distributed under the terms of the Modified BSD License. from unittest import TestCase +from pygments import __version__ as pygments_version from pygments.token import Token from pygments.lexers import BashLexer from .. import lexers +pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) + class TestLexers(TestCase): """Collection of lexers tests""" @@ -18,25 +21,26 @@ def setUp(self): def testIPythonLexer(self): fragment = '!echo $HOME\n' - tokens = [ + bash_tokens = [ (Token.Operator, '!'), ] - tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) + ipylex_token = list(self.lexer.get_tokens(fragment)) + assert bash_tokens[:-1] == ipylex_token[:-1] - fragment_2 = '!' + fragment + fragment_2 = "!" + fragment tokens_2 = [ (Token.Operator, '!!'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = '\t %%!\n' + fragment[1:] tokens_2 = [ (Token.Text, '\t '), (Token.Operator, '%%!'), (Token.Text, '\n'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = 'x = ' + fragment tokens_2 = [ @@ -44,8 +48,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = ' + fragment tokens_2 = [ @@ -54,8 +58,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = %sx ' + fragment[1:] tokens_2 = [ @@ -67,8 +71,10 @@ def testIPythonLexer(self): (Token.Operator, '%'), (Token.Keyword, 'sx'), (Token.Text, ' '), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + if tokens_2[7] == (Token.Text, " ") and pyg214: # pygments 2.14+ + tokens_2[7] = (Token.Text.Whitespace, " ") + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'f = %R function () {}\n' tokens_2 = [ @@ -80,7 +86,7 @@ def testIPythonLexer(self): (Token.Keyword, 'R'), (Token.Text, ' function () {}\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '\t%%xyz\n$foo\n' tokens_2 = [ @@ -89,7 +95,7 @@ def testIPythonLexer(self): (Token.Keyword, 'xyz'), (Token.Text, '\n$foo\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '%system?\n' tokens_2 = [ @@ -98,7 +104,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x != y\n' tokens_2 = [ @@ -109,7 +115,7 @@ def testIPythonLexer(self): (Token.Name, 'y'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = ' ?math.sin\n' tokens_2 = [ @@ -118,7 +124,7 @@ def testIPythonLexer(self): (Token.Text, 'math.sin'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment = ' *int*?\n' tokens = [ @@ -126,7 +132,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment)) fragment = '%%writefile -a foo.py\nif a == b:\n pass' tokens = [ @@ -145,7 +151,9 @@ def testIPythonLexer(self): (Token.Keyword, 'pass'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + if tokens[10] == (Token.Text, "\n") and pyg214: # pygments 2.14+ + tokens[10] = (Token.Text.Whitespace, "\n") + assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1] fragment = '%%timeit\nmath.sin(0)' tokens = [ @@ -173,4 +181,4 @@ def testIPythonLexer(self): (Token.Punctuation, '>'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment)) From 0520f55c8a2cb500ad2428d1c0b36adc57b11309 Mon Sep 17 00:00:00 2001 From: Nir Schulman Date: Sat, 31 Dec 2022 08:47:28 +0200 Subject: [PATCH 066/122] Removed the usage of minor-version entrypoints --- setupbase.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setupbase.py b/setupbase.py index 748b4dd6a8a..3be9e831133 100644 --- a/setupbase.py +++ b/setupbase.py @@ -211,19 +211,16 @@ def find_entry_points(): use, our own build_scripts_entrypt class below parses these and builds command line scripts. - Each of our entry points gets a plain name, e.g. ipython, a name - suffixed with the Python major version number, e.g. ipython3, and - a name suffixed with the Python major.minor version number, eg. ipython3.8. + Each of our entry points gets a plain name, e.g. ipython, and a name + suffixed with the Python major version number, e.g. ipython3. """ ep = [ 'ipython%s = IPython:start_ipython', ] major_suffix = str(sys.version_info[0]) - minor_suffix = ".".join([str(sys.version_info[0]), str(sys.version_info[1])]) return ( [e % "" for e in ep] + [e % major_suffix for e in ep] - + [e % minor_suffix for e in ep] ) class install_lib_symlink(Command): From 34da39cb0d7654f21d6f551864be94cc16f9ad34 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 11:06:33 +0100 Subject: [PATCH 067/122] Please linter --- setupbase.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setupbase.py b/setupbase.py index 3be9e831133..a867c73ecd0 100644 --- a/setupbase.py +++ b/setupbase.py @@ -218,10 +218,8 @@ def find_entry_points(): 'ipython%s = IPython:start_ipython', ] major_suffix = str(sys.version_info[0]) - return ( - [e % "" for e in ep] - + [e % major_suffix for e in ep] - ) + return [e % "" for e in ep] + [e % major_suffix for e in ep] + class install_lib_symlink(Command): user_options = [ From 8af5442ee0d7c34d0066c6651ac2d364790c6d6c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 8 Dec 2022 17:12:57 -0900 Subject: [PATCH 068/122] Fix vi mode escape delay --- IPython/terminal/shortcuts.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 7d6de8b3b04..01072cd99a1 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -68,9 +68,14 @@ def reformat_and_execute(event): reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell) event.current_buffer.validate_and_handle() + @Condition + def ebivim(): + return shell.emacs_bindings_in_vi_insert_mode + kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode + & ebivim ))(reformat_and_execute) kb.add("c-\\")(quit) @@ -333,10 +338,6 @@ def _(event): if sys.platform == "win32": kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste) - @Condition - def ebivim(): - return shell.emacs_bindings_in_vi_insert_mode - focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode @kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode)) From 11dd2521aee4a87d68f2244d6fee86c6e1873fcd Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 12:00:52 +0100 Subject: [PATCH 069/122] please formatter --- IPython/terminal/shortcuts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 01072cd99a1..6ca91ec31ba 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -72,11 +72,11 @@ def reformat_and_execute(event): def ebivim(): return shell.emacs_bindings_in_vi_insert_mode - kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - & ebivim - ))(reformat_and_execute) + kb.add( + "escape", + "enter", + filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim), + )(reformat_and_execute) kb.add("c-\\")(quit) From cc74642003f1ab22415ae0a32a4aa9548e3e398c Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 14:04:45 +0100 Subject: [PATCH 070/122] whats new version8.8 --- docs/source/whatsnew/version8.rst | 66 ++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/source/whatsnew/version8.rst b/docs/source/whatsnew/version8.rst index d3c33704bad..e1d4574e452 100644 --- a/docs/source/whatsnew/version8.rst +++ b/docs/source/whatsnew/version8.rst @@ -2,6 +2,44 @@ 8.x Series ============ +.. _version 8.8.0: + +IPython 8.8.0 +------------- + +First release of IPython in 2023 as there was no release at the end of +December. + +This is an unusually big release (relatively speaking) with more than 15 Pull +Requests merge. + +Of particular interest are: + + - :ghpull:`13852` that replace the greedy completer and improve + completion, in particular for dictionary keys. + - :ghpull:`13858` that adds ``py.typed`` to ``setup.cfg`` to make sure it is + bundled in wheels. + - :ghpull:`13869` that implements tab completions for IPython options in the + shell when using `argcomplete `. I + believe this also needs a recent version of Traitlets. + - :ghpull:`13865` makes the ``inspector`` class of `InteractiveShell` + configurable. + - :ghpull:`13880` that remove minor-version entrypoints as the minor version + entry points that would be included in the wheel would be the one of the + Python version that was used to build the ``whl`` file. + +In no particular order, the rest of the changes update the test suite to be +compatible with Pygments 2.14, various docfixes, testing on more recent python +versions and various updates. + +As usual you can find the full list of PRs on GitHub under `the 8.8 milestone +`__. + +Many thanks to @krassowski for the many PRs and @jasongrout for reviewing and +merging contributions. + +Thanks to the `D. E. Shaw group `__ for sponsoring +work on IPython and related libraries. .. _version 8.7.0: @@ -138,7 +176,7 @@ Here is a non exhaustive list of changes that have been implemented for IPython - Fix paste magic on wayland. :ghpull:`13671` - show maxlen in deque's repr. :ghpull:`13648` -Restore line numbers for Input +Restore line numbers for Input ------------------------------ Line number information in tracebacks from input are restored. @@ -269,7 +307,7 @@ Thanks to the `D. E. Shaw group `__ for sponsoring work on IPython and related libraries. .. _version 8.1.1: - + IPython 8.1.1 ------------- @@ -403,10 +441,10 @@ The 8.x branch started diverging from its predecessor around IPython 7.12 (January 2020). This release contains 250+ pull requests, in addition to many of the features -and backports that have made it to the 7.x branch. Please see the +and backports that have made it to the 7.x branch. Please see the `8.0 milestone `__ for the full list of pull requests. -Please feel free to send pull requests to updates those notes after release, +Please feel free to send pull requests to updates those notes after release, I have likely forgotten a few things reviewing 250+ PRs. Dependencies changes/downstream packaging @@ -421,7 +459,7 @@ looking for help to do so. - minimal Python is now 3.8 - ``nose`` is not a testing requirement anymore - ``pytest`` replaces nose. - - ``iptest``/``iptest3`` cli entrypoints do not exists anymore. + - ``iptest``/``iptest3`` cli entrypoints do not exists anymore. - minimum officially support ``numpy`` version has been bumped, but this should not have much effect on packaging. @@ -443,7 +481,7 @@ deprecation warning: - Please add **since which version** something is deprecated. As a side note, it is much easier to conditionally compare version -numbers rather than using ``try/except`` when functionality changes with a version. +numbers rather than using ``try/except`` when functionality changes with a version. I won't list all the removed features here, but modules like ``IPython.kernel``, which was just a shim module around ``ipykernel`` for the past 8 years, have been @@ -475,7 +513,7 @@ by mypy. Featured changes ---------------- -Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. +Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. Please note as well that many features have been added in the 7.x branch as well (and hence why you want to read the 7.x what's new notes), in particular features contributed by QuantStack (with respect to debugger protocol and Xeus @@ -523,7 +561,7 @@ The error traceback is now correctly formatted, showing the cell number in which ZeroDivisionError: division by zero -The ``stack_data`` package has been integrated, which provides smarter information in the traceback; +The ``stack_data`` package has been integrated, which provides smarter information in the traceback; in particular it will highlight the AST node where an error occurs which can help to quickly narrow down errors. For example in the following snippet:: @@ -563,7 +601,7 @@ and IPython 8.0 is capable of telling you where the index error occurs:: ----> 3 return x[0][i][0] ^^^^^^^ -The corresponding locations marked here with ``^`` will show up highlighted in +The corresponding locations marked here with ``^`` will show up highlighted in the terminal and notebooks. Finally, a colon ``::`` and line number is appended after a filename in @@ -760,7 +798,7 @@ Previously, this was not the case for the Vi-mode prompts:: This is now fixed, and Vi prompt prefixes - ``[ins]`` and ``[nav]`` - are skipped just as the normal ``In`` would be. -IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``, +IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``, You should be able to change mode dynamically with ``%config TerminalInteractiveShell.editing_mode='vi'`` Empty History Ranges @@ -787,8 +825,8 @@ when followed with :kbd:`F2`), send it to `dpaste.org `_ using Windows timing implementation: Switch to process_time ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Timing on Windows, for example with ``%%time``, was changed from being based on ``time.perf_counter`` -(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead +Timing on Windows, for example with ``%%time``, was changed from being based on ``time.perf_counter`` +(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead (which only counts CPU time). This brings it closer to the behavior on Linux. See :ghpull:`12984`. Miscellaneous @@ -813,7 +851,7 @@ Re-added support for XDG config directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ XDG support through the years comes and goes. There is a tension between having -an identical location for configuration in all platforms versus having simple instructions. +an identical location for configuration in all platforms versus having simple instructions. After initial failures a couple of years ago, IPython was modified to automatically migrate XDG config files back into ``~/.ipython``. That migration code has now been removed. IPython now checks the XDG locations, so if you _manually_ move your config @@ -841,7 +879,7 @@ Removing support for older Python versions We are removing support for Python up through 3.7, allowing internal code to use the more -efficient ``pathlib`` and to make better use of type annotations. +efficient ``pathlib`` and to make better use of type annotations. .. image:: ../_images/8.0/pathlib_pathlib_everywhere.jpg :alt: "Meme image of Toy Story with Woody and Buzz, with the text 'pathlib, pathlib everywhere'" From 198bb721ef48a538133fa4d646828e6838856113 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 14:35:34 +0100 Subject: [PATCH 071/122] Misc release process update A few of those things failed during release time, as I was building docs on 3.10 (not 3.11) after recently upgrading. --- IPython/core/completer.py | 2 +- tools/release_helper.sh | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 5ba8ea0fdbd..f0bbb4e5619 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -256,7 +256,7 @@ JEDI_INSTALLED = False -if TYPE_CHECKING or GENERATING_DOCUMENTATION: +if TYPE_CHECKING or GENERATING_DOCUMENTATION and sys.version_info >= (3, 11): from typing import cast from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard else: diff --git a/tools/release_helper.sh b/tools/release_helper.sh index d221f551e66..ebf8098195c 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -2,15 +2,6 @@ # when releasing with bash, simple source it to get asked questions. # misc check before starting - -python -c 'import keyring' -python -c 'import twine' -python -c 'import sphinx' -python -c 'import sphinx_rtd_theme' -python -c 'import pytest' -python -c 'import build' - - BLACK=$(tput setaf 1) RED=$(tput setaf 1) GREEN=$(tput setaf 2) @@ -22,6 +13,22 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Checking all tools are installed..." + +python -c 'import keyring' +python -c 'import twine' +python -c 'import sphinx' +python -c 'import sphinx_rtd_theme' +python -c 'import pytest' +python -c 'import build' +# those are necessary fo building the docs +echo "Checking imports for docs" +python -c 'import numpy' +python -c 'import matplotlib' + + + + echo "Will use $BLUE'$EDITOR'$NOR to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input From add5877a42ba8e3960bc92eb994c15955eacf254 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 15:03:24 +0100 Subject: [PATCH 072/122] release 8.8.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index e2ce2eac2b4..e99b4f3493d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,7 +20,7 @@ _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -# _version_extra = "" # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 01bc9d96e0dd50fad02d7b1e3d2e58113c617b6b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Jan 2023 15:04:11 +0100 Subject: [PATCH 073/122] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index e99b4f3493d..0416637fc7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -16,11 +16,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 8 -_version_minor = 8 +_version_minor = 9 _version_patch = 0 _version_extra = ".dev" # _version_extra = "rc1" -_version_extra = "" # Uncomment this for full releases +# _version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 64e72a955f3ae4eb0ad823936a20364e4475e057 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 00:17:23 +0000 Subject: [PATCH 074/122] Restore shortcuts in documentation, define identifiers --- .../{shortcuts.py => shortcuts/__init__.py} | 373 +++++++++--------- IPython/terminal/shortcuts/auto_match.py | 90 +++++ IPython/terminal/shortcuts/autosuggestions.py | 39 ++ docs/autogen_shortcuts.py | 252 +++++++++--- docs/source/_static/theme_overrides.css | 7 + docs/source/conf.py | 5 +- docs/source/config/shortcuts/index.rst | 29 +- 7 files changed, 525 insertions(+), 270 deletions(-) rename IPython/terminal/{shortcuts.py => shortcuts/__init__.py} (62%) create mode 100644 IPython/terminal/shortcuts/auto_match.py create mode 100644 IPython/terminal/shortcuts/autosuggestions.py create mode 100644 docs/source/_static/theme_overrides.css diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts/__init__.py similarity index 62% rename from IPython/terminal/shortcuts.py rename to IPython/terminal/shortcuts/__init__.py index 6ca91ec31ba..6bc6ec7c39f 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -16,14 +16,29 @@ from prompt_toolkit.application.current import get_app from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER -from prompt_toolkit.filters import (has_focus, has_selection, Condition, - vi_insert_mode, emacs_insert_mode, has_completions, vi_mode) -from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline +from prompt_toolkit.filters import ( + has_focus as has_focus_impl, + has_selection, + Condition, + vi_insert_mode, + emacs_insert_mode, + has_completions, + vi_mode, +) +from prompt_toolkit.key_binding.bindings.completion import ( + display_completions_like_readline, +) from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.key_binding.bindings import named_commands as nc from prompt_toolkit.key_binding.vi_state import InputMode, ViState +from prompt_toolkit.layout.layout import FocusableElement from IPython.utils.decorators import undoc +from . import auto_match as match, autosuggestions + + +__all__ = ["create_ipython_shortcuts"] + @undoc @Condition @@ -32,80 +47,84 @@ def cursor_in_leading_ws(): return (not before) or before.isspace() -# Needed for to accept autosuggestions in vi insert mode -def _apply_autosuggest(event): - """ - Apply autosuggestion if at end of line. - """ - b = event.current_buffer - d = b.document - after_cursor = d.text[d.cursor_position :] - lines = after_cursor.split("\n") - end_of_current_line = lines[0].strip() - suggestion = b.suggestion - if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): - b.insert_text(suggestion.text) - else: - nc.end_of_line(event) +def has_focus(value: FocusableElement): + """Wrapper around has_focus adding a nice `__name__` to tester function""" + tester = has_focus_impl(value).func + tester.__name__ = f"is_focused({value})" + return Condition(tester) -def create_ipython_shortcuts(shell): - """Set up the prompt_toolkit keyboard shortcuts for IPython""" + +def create_ipython_shortcuts(shell, for_all_platforms: bool = False): + """Set up the prompt_toolkit keyboard shortcuts for IPython.""" + # Warning: if possible, do NOT define handler functions in the locals + # scope of this function, instead define functions in the global + # scope, or a separate module, and include a user-friendly docstring + # describing the action. kb = KeyBindings() insert_mode = vi_insert_mode | emacs_insert_mode - if getattr(shell, 'handle_return', None): + if getattr(shell, "handle_return", None): return_handler = shell.handle_return(shell) else: return_handler = newline_or_execute_outer(shell) - kb.add('enter', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - ))(return_handler) - - def reformat_and_execute(event): - reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell) - event.current_buffer.validate_and_handle() + kb.add("enter", filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode))( + return_handler + ) @Condition def ebivim(): return shell.emacs_bindings_in_vi_insert_mode - kb.add( + @kb.add( "escape", "enter", filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim), - )(reformat_and_execute) + ) + def reformat_and_execute(event): + """Reformat code and execute it""" + reformat_text_before_cursor( + event.current_buffer, event.current_buffer.document, shell + ) + event.current_buffer.validate_and_handle() kb.add("c-\\")(quit) - kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)) - )(previous_history_or_previous_completion) + kb.add("c-p", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))( + previous_history_or_previous_completion + ) - kb.add('c-n', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)) - )(next_history_or_next_completion) + kb.add("c-n", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))( + next_history_or_next_completion + ) - kb.add('c-g', filter=(has_focus(DEFAULT_BUFFER) & has_completions) - )(dismiss_completion) + kb.add("c-g", filter=(has_focus(DEFAULT_BUFFER) & has_completions))( + dismiss_completion + ) - kb.add('c-c', filter=has_focus(DEFAULT_BUFFER))(reset_buffer) + kb.add("c-c", filter=has_focus(DEFAULT_BUFFER))(reset_buffer) - kb.add('c-c', filter=has_focus(SEARCH_BUFFER))(reset_search_buffer) + kb.add("c-c", filter=has_focus(SEARCH_BUFFER))(reset_search_buffer) - supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP')) - kb.add('c-z', filter=supports_suspend)(suspend_to_bg) + supports_suspend = Condition(lambda: hasattr(signal, "SIGTSTP")) + kb.add("c-z", filter=supports_suspend)(suspend_to_bg) # Ctrl+I == Tab - kb.add('tab', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & insert_mode - & cursor_in_leading_ws - ))(indent_buffer) - kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode) - )(newline_autoindent_outer(shell.input_transformer_manager)) + kb.add( + "tab", + filter=( + has_focus(DEFAULT_BUFFER) + & ~has_selection + & insert_mode + & cursor_in_leading_ws + ), + )(indent_buffer) + kb.add("c-o", filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode))( + newline_autoindent_outer(shell.input_transformer_manager) + ) - kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor) + kb.add("f2", filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor) @Condition def auto_match(): @@ -146,6 +165,8 @@ def _preceding_text(): before_cursor = app.current_buffer.document.current_line_before_cursor return bool(m.match(before_cursor)) + _preceding_text.__name__ = f"preceding_text({pattern!r})" + condition = Condition(_preceding_text) _preceding_text_cache[pattern] = condition return condition @@ -161,6 +182,8 @@ def _following_text(): app = get_app() return bool(m.match(app.current_buffer.document.current_line_after_cursor)) + _following_text.__name__ = f"following_text({pattern!r})" + condition = Condition(_following_text) _following_text_cache[pattern] = condition return condition @@ -178,151 +201,110 @@ def not_inside_unclosed_string(): return not ('"' in s or "'" in s) # auto match - @kb.add("(", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("()") - event.current_buffer.cursor_left() - - @kb.add("[", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("[]") - event.current_buffer.cursor_left() - - @kb.add("{", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$")) - def _(event): - event.current_buffer.insert_text("{}") - event.current_buffer.cursor_left() + auto_match_parens = {"(": match.parenthesis, "[": match.brackets, "{": match.braces} + for key, cmd in auto_match_parens.items(): + kb.add(key, filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))( + cmd + ) - @kb.add( + kb.add( '"', filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(lambda line: all_quotes_paired('"', line)) & following_text(r"[,)}\]]|$"), - ) - def _(event): - event.current_buffer.insert_text('""') - event.current_buffer.cursor_left() + )(match.double_quote) - @kb.add( + kb.add( "'", filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(lambda line: all_quotes_paired("'", line)) & following_text(r"[,)}\]]|$"), - ) - def _(event): - event.current_buffer.insert_text("''") - event.current_buffer.cursor_left() + )(match.single_quote) - @kb.add( + kb.add( '"', filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(r'^.*""$'), - ) - def _(event): - event.current_buffer.insert_text('""""') - event.current_buffer.cursor_left(3) + )(match.docstring_double_quotes) - @kb.add( + kb.add( "'", filter=focused_insert & auto_match & not_inside_unclosed_string & preceding_text(r"^.*''$"), - ) - def _(event): - event.current_buffer.insert_text("''''") - event.current_buffer.cursor_left(3) + )(match.docstring_single_quotes) # raw string - @kb.add( - "(", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") - ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("()" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) + auto_match_parens_raw_string = { + "(": match.raw_string_parenthesis, + "[": match.raw_string_bracket, + "{": match.raw_string_braces, + } + for key, cmd in auto_match_parens_raw_string.items(): + kb.add( + key, + filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"), + )(cmd) - @kb.add( - "[", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + # just move cursor + kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))( + match.skip_over ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("[]" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) - - @kb.add( - "{", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$") + kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]"))( + match.skip_over + ) + kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}"))( + match.skip_over + ) + kb.add('"', filter=focused_insert & auto_match & following_text('^"'))( + match.skip_over + ) + kb.add("'", filter=focused_insert & auto_match & following_text("^'"))( + match.skip_over ) - def _(event): - matches = re.match( - r".*(r|R)[\"'](-*)", - event.current_buffer.document.current_line_before_cursor, - ) - dashes = matches.group(2) or "" - event.current_buffer.insert_text("{}" + dashes) - event.current_buffer.cursor_left(len(dashes) + 1) - - # just move cursor - @kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)")) - @kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]")) - @kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}")) - @kb.add('"', filter=focused_insert & auto_match & following_text('^"')) - @kb.add("'", filter=focused_insert & auto_match & following_text("^'")) - def _(event): - event.current_buffer.cursor_right() - @kb.add( + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\($") & auto_match & following_text(r"^\)"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\[$") & auto_match & following_text(r"^\]"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*\{$") & auto_match & following_text(r"^\}"), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text('.*"$') & auto_match & following_text('^"'), - ) - @kb.add( + )(match.delete_pair) + kb.add( "backspace", filter=focused_insert & preceding_text(r".*'$") & auto_match & following_text(r"^'"), - ) - def _(event): - event.current_buffer.delete() - event.current_buffer.delete_before_cursor() + )(match.delete_pair) if shell.display_completions == "readlinelike": kb.add( @@ -335,37 +317,22 @@ def _(event): ), )(display_completions_like_readline) - if sys.platform == "win32": + if sys.platform == "win32" or for_all_platforms: kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste) focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode - @kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode)) - def _(event): - _apply_autosuggest(event) - - @kb.add("c-e", filter=focused_insert_vi & ebivim) - def _(event): - _apply_autosuggest(event) - - @kb.add("c-f", filter=focused_insert_vi) - def _(event): - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - b.insert_text(suggestion.text) - else: - nc.forward_char(event) - - @kb.add("escape", "f", filter=focused_insert_vi & ebivim) - def _(event): - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - t = re.split(r"(\S+\s+)", suggestion.text) - b.insert_text(next((x for x in t if x), "")) - else: - nc.forward_word(event) + # autosuggestions + kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))( + autosuggestions.accept_in_vi_insert_mode + ) + kb.add("c-e", filter=focused_insert_vi & ebivim)( + autosuggestions.accept_in_vi_insert_mode + ) + kb.add("c-f", filter=focused_insert_vi)(autosuggestions.accept) + kb.add("escape", "f", filter=focused_insert_vi & ebivim)( + autosuggestions.accept_word + ) # Simple Control keybindings key_cmd_dict = { @@ -423,7 +390,7 @@ def set_input_mode(self, mode): def reformat_text_before_cursor(buffer, document, shell): - text = buffer.delete_before_cursor(len(document.text[:document.cursor_position])) + text = buffer.delete_before_cursor(len(document.text[: document.cursor_position])) try: formatted_text = shell.reformat_handler(text) buffer.insert_text(formatted_text) @@ -432,7 +399,6 @@ def reformat_text_before_cursor(buffer, document, shell): def newline_or_execute_outer(shell): - def newline_or_execute(event): """When the user presses return, insert a newline or execute the code.""" b = event.current_buffer @@ -451,34 +417,38 @@ def newline_or_execute(event): if d.line_count == 1: check_text = d.text else: - check_text = d.text[:d.cursor_position] + check_text = d.text[: d.cursor_position] status, indent = shell.check_complete(check_text) - + # if all we have after the cursor is whitespace: reformat current text # before cursor - after_cursor = d.text[d.cursor_position:] + after_cursor = d.text[d.cursor_position :] reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) reformatted = True - if not (d.on_last_line or - d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() - ): + if not ( + d.on_last_line + or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() + ): if shell.autoindent: - b.insert_text('\n' + indent) + b.insert_text("\n" + indent) else: - b.insert_text('\n') + b.insert_text("\n") return - if (status != 'incomplete') and b.accept_handler: + if (status != "incomplete") and b.accept_handler: if not reformatted: reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: - b.insert_text('\n' + indent) + b.insert_text("\n" + indent) else: - b.insert_text('\n') + b.insert_text("\n") + + newline_or_execute.__qualname__ = "newline_or_execute" + return newline_or_execute @@ -501,12 +471,14 @@ def next_history_or_next_completion(event): def dismiss_completion(event): + """Dismiss completion""" b = event.current_buffer if b.complete_state: b.cancel_completion() def reset_buffer(event): + """Reset buffer""" b = event.current_buffer if b.complete_state: b.cancel_completion() @@ -515,16 +487,22 @@ def reset_buffer(event): def reset_search_buffer(event): + """Reset search buffer""" if event.current_buffer.document.text: event.current_buffer.reset() else: event.app.layout.focus(DEFAULT_BUFFER) + def suspend_to_bg(event): + """Suspend to background""" event.app.suspend_to_background() + def quit(event): """ + Quit application with ``SIGQUIT`` if supported or ``sys.exit`` otherwise. + On platforms that support SIGQUIT, send SIGQUIT to the current process. On other platforms, just exit the process with a message. """ @@ -534,8 +512,11 @@ def quit(event): else: sys.exit("Quit") + def indent_buffer(event): - event.current_buffer.insert_text(' ' * 4) + """Indent buffer""" + event.current_buffer.insert_text(" " * 4) + @undoc def newline_with_copy_margin(event): @@ -547,9 +528,12 @@ def newline_with_copy_margin(event): Preserve margin and cursor position when using Control-O to insert a newline in EMACS mode """ - warnings.warn("`newline_with_copy_margin(event)` is deprecated since IPython 6.0. " - "see `newline_autoindent_outer(shell)(event)` for a replacement.", - DeprecationWarning, stacklevel=2) + warnings.warn( + "`newline_with_copy_margin(event)` is deprecated since IPython 6.0. " + "see `newline_autoindent_outer(shell)(event)` for a replacement.", + DeprecationWarning, + stacklevel=2, + ) b = event.current_buffer cursor_start_pos = b.document.cursor_position_col @@ -560,6 +544,7 @@ def newline_with_copy_margin(event): pos_diff = cursor_start_pos - cursor_end_pos b.cursor_right(count=pos_diff) + def newline_autoindent_outer(inputsplitter) -> Callable[..., None]: """ Return a function suitable for inserting a indented newline after the cursor. @@ -571,28 +556,33 @@ def newline_autoindent_outer(inputsplitter) -> Callable[..., None]: """ def newline_autoindent(event): - """insert a newline after the cursor indented appropriately.""" + """Insert a newline after the cursor indented appropriately.""" b = event.current_buffer d = b.document if b.complete_state: b.cancel_completion() - text = d.text[:d.cursor_position] + '\n' + text = d.text[: d.cursor_position] + "\n" _, indent = inputsplitter.check_complete(text) - b.insert_text('\n' + (' ' * (indent or 0)), move_cursor=False) + b.insert_text("\n" + (" " * (indent or 0)), move_cursor=False) + + newline_autoindent.__qualname__ = "newline_autoindent" return newline_autoindent def open_input_in_editor(event): + """Open code from input in external editor""" event.app.current_buffer.open_in_editor() -if sys.platform == 'win32': +if sys.platform == "win32": from IPython.core.error import TryNext - from IPython.lib.clipboard import (ClipboardEmpty, - win32_clipboard_get, - tkinter_clipboard_get) + from IPython.lib.clipboard import ( + ClipboardEmpty, + win32_clipboard_get, + tkinter_clipboard_get, + ) @undoc def win_paste(event): @@ -606,3 +596,10 @@ def win_paste(event): except ClipboardEmpty: return event.current_buffer.insert_text(text.replace("\t", " " * 4)) + +else: + + @undoc + def win_paste(event): + """Stub used when auto-generating shortcuts for documentation""" + pass diff --git a/IPython/terminal/shortcuts/auto_match.py b/IPython/terminal/shortcuts/auto_match.py new file mode 100644 index 00000000000..0976bb20336 --- /dev/null +++ b/IPython/terminal/shortcuts/auto_match.py @@ -0,0 +1,90 @@ +import re +from prompt_toolkit.key_binding import KeyPressEvent + + +def parenthesis(event: KeyPressEvent): + """Auto-close parenthesis""" + event.current_buffer.insert_text("()") + event.current_buffer.cursor_left() + + +def brackets(event: KeyPressEvent): + """Auto-close brackets""" + event.current_buffer.insert_text("[]") + event.current_buffer.cursor_left() + + +def braces(event: KeyPressEvent): + """Auto-close braces""" + event.current_buffer.insert_text("{}") + event.current_buffer.cursor_left() + + +def double_quote(event: KeyPressEvent): + """Auto-close double quotes""" + event.current_buffer.insert_text('""') + event.current_buffer.cursor_left() + + +def single_quote(event: KeyPressEvent): + """Auto-close single quotes""" + event.current_buffer.insert_text("''") + event.current_buffer.cursor_left() + + +def docstring_double_quotes(event: KeyPressEvent): + """Auto-close docstring (double quotes)""" + event.current_buffer.insert_text('""""') + event.current_buffer.cursor_left(3) + + +def docstring_single_quotes(event: KeyPressEvent): + """Auto-close docstring (single quotes)""" + event.current_buffer.insert_text("''''") + event.current_buffer.cursor_left(3) + + +def raw_string_parenthesis(event: KeyPressEvent): + """Auto-close parenthesis in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) or "" + event.current_buffer.insert_text("()" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def raw_string_bracket(event: KeyPressEvent): + """Auto-close bracker in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) or "" + event.current_buffer.insert_text("[]" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def raw_string_braces(event: KeyPressEvent): + """Auto-close braces in raw strings""" + matches = re.match( + r".*(r|R)[\"'](-*)", + event.current_buffer.document.current_line_before_cursor, + ) + dashes = matches.group(2) or "" + event.current_buffer.insert_text("{}" + dashes) + event.current_buffer.cursor_left(len(dashes) + 1) + + +def skip_over(event: KeyPressEvent): + """Skip over automatically added parenthesis. + + (rather than adding another parenthesis)""" + event.current_buffer.cursor_right() + + +def delete_pair(event: KeyPressEvent): + """Delete auto-closed parenthesis""" + event.current_buffer.delete() + event.current_buffer.delete_before_cursor() diff --git a/IPython/terminal/shortcuts/autosuggestions.py b/IPython/terminal/shortcuts/autosuggestions.py new file mode 100644 index 00000000000..fe1e8d07b7c --- /dev/null +++ b/IPython/terminal/shortcuts/autosuggestions.py @@ -0,0 +1,39 @@ +import re +from prompt_toolkit.key_binding import KeyPressEvent +from prompt_toolkit.key_binding.bindings import named_commands as nc + + +# Needed for to accept autosuggestions in vi insert mode +def accept_in_vi_insert_mode(event: KeyPressEvent): + """Apply autosuggestion if at end of line.""" + b = event.current_buffer + d = b.document + after_cursor = d.text[d.cursor_position :] + lines = after_cursor.split("\n") + end_of_current_line = lines[0].strip() + suggestion = b.suggestion + if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): + b.insert_text(suggestion.text) + else: + nc.end_of_line(event) + + +def accept(event): + """Accept suggestion""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion: + b.insert_text(suggestion.text) + else: + nc.forward_char(event) + + +def accept_word(event): + """Fill partial suggestion by word""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion: + t = re.split(r"(\S+\s+)", suggestion.text) + b.insert_text(next((x for x in t if x), "")) + else: + nc.forward_word(event) diff --git a/docs/autogen_shortcuts.py b/docs/autogen_shortcuts.py index db7fe8d4917..b5886ffa576 100755 --- a/docs/autogen_shortcuts.py +++ b/docs/autogen_shortcuts.py @@ -1,45 +1,98 @@ +from dataclasses import dataclass +from inspect import getsource from pathlib import Path +from typing import cast, Callable, List, Union +from html import escape as html_escape +import re + +from prompt_toolkit.keys import KEY_ALIASES +from prompt_toolkit.key_binding import KeyBindingsBase +from prompt_toolkit.filters import Filter, Condition +from prompt_toolkit.shortcuts import PromptSession from IPython.terminal.shortcuts import create_ipython_shortcuts -def name(c): - s = c.__class__.__name__ - if s == '_Invert': - return '(Not: %s)' % name(c.filter) - if s in log_filters.keys(): - return '(%s: %s)' % (log_filters[s], ', '.join(name(x) for x in c.filters)) - return log_filters[s] if s in log_filters.keys() else s +@dataclass +class Shortcut: + #: a sequence of keys (each element on the list corresponds to pressing one or more keys) + keys_sequence: list[str] + filter: str -def sentencize(s): - """Extract first sentence - """ - s = s.replace('\n', ' ').strip().split('.') - s = s[0] if len(s) else s - try: - return " ".join(s.split()) - except AttributeError: - return s +@dataclass +class Handler: + description: str + identifier: str -def most_common(lst, n=3): - """Most common elements occurring more then `n` times - """ - from collections import Counter - c = Counter(lst) - return [k for (k, v) in c.items() if k and v > n] +@dataclass +class Binding: + handler: Handler + shortcut: Shortcut -def multi_filter_str(flt): - """Yield readable conditional filter - """ - assert hasattr(flt, 'filters'), 'Conditional filter required' - yield name(flt) +class _NestedFilter(Filter): + """Protocol reflecting non-public prompt_toolkit's `_AndList` and `_OrList`.""" + + filters: List[Filter] + + +class _Invert(Filter): + """Protocol reflecting non-public prompt_toolkit's `_Invert`.""" + + filter: Filter + + +conjunctions_labels = {"_AndList": "and", "_OrList": "or"} +ATOMIC_CLASSES = {"Never", "Always", "Condition"} + + +def format_filter( + filter_: Union[Filter, _NestedFilter, Condition, _Invert], + is_top_level=True, + skip=None, +) -> str: + """Create easily readable description of the filter.""" + s = filter_.__class__.__name__ + if s == "Condition": + func = cast(Condition, filter_).func + name = func.__name__ + if name == "": + source = getsource(func) + return source.split("=")[0].strip() + return func.__name__ + elif s == "_Invert": + operand = cast(_Invert, filter_).filter + if operand.__class__.__name__ in ATOMIC_CLASSES: + return f"not {format_filter(operand, is_top_level=False)}" + return f"not ({format_filter(operand, is_top_level=False)})" + elif s in conjunctions_labels: + filters = cast(_NestedFilter, filter_).filters + conjunction = conjunctions_labels[s] + glue = f" {conjunction} " + result = glue.join(format_filter(x, is_top_level=False) for x in filters) + if len(filters) > 1 and not is_top_level: + result = f"({result})" + return result + elif s in ["Never", "Always"]: + return s.lower() + else: + raise ValueError(f"Unknown filter type: {filter_}") + + +def sentencize(s) -> str: + """Extract first sentence""" + s = re.split(r"\.\W", s.replace("\n", " ").strip()) + s = s[0] if len(s) else "" + if not s.endswith("."): + s += "." + try: + return " ".join(s.split()) + except AttributeError: + return s -log_filters = {'_AndList': 'And', '_OrList': 'Or'} -log_invert = {'_Invert'} class _DummyTerminal: """Used as a buffer to get prompt_toolkit bindings @@ -50,47 +103,118 @@ class _DummyTerminal: editing_mode = "emacs" -ipy_bindings = create_ipython_shortcuts(_DummyTerminal()).bindings - -dummy_docs = [] # ignore bindings without proper documentation - -common_docs = most_common([kb.handler.__doc__ for kb in ipy_bindings]) -if common_docs: - dummy_docs.extend(common_docs) +def create_identifier(handler: Callable): + parts = handler.__module__.split(".") + name = handler.__name__ + package = parts[0] + if len(parts) > 1: + final_module = parts[-1] + return f"{package}:{final_module}.{name}" + else: + return f"{package}:{name}" + + +def bindings_from_prompt_toolkit(prompt_bindings: KeyBindingsBase) -> List[Binding]: + """Collect bindings to a simple format that does not depend on prompt-toolkit internals""" + bindings: List[Binding] = [] + + for kb in prompt_bindings.bindings: + bindings.append( + Binding( + handler=Handler( + description=kb.handler.__doc__ or "", + identifier=create_identifier(kb.handler), + ), + shortcut=Shortcut( + keys_sequence=[ + str(k.value) if hasattr(k, "value") else k for k in kb.keys + ], + filter=format_filter(kb.filter, skip={"has_focus_filter"}), + ), + ) + ) + return bindings + + +INDISTINGUISHABLE_KEYS = {**KEY_ALIASES, **{v: k for k, v in KEY_ALIASES.items()}} + + +def format_prompt_keys(keys: str, add_alternatives=True) -> str: + """Format prompt toolkit key with modifier into an RST representation.""" + + def to_rst(key): + escaped = key.replace("\\", "\\\\") + return f":kbd:`{escaped}`" + + keys_to_press: list[str] + + prefixes = { + "c-s-": [to_rst("ctrl"), to_rst("shift")], + "s-c-": [to_rst("ctrl"), to_rst("shift")], + "c-": [to_rst("ctrl")], + "s-": [to_rst("shift")], + } + + for prefix, modifiers in prefixes.items(): + if keys.startswith(prefix): + remainder = keys[len(prefix) :] + keys_to_press = [*modifiers, to_rst(remainder)] + break + else: + keys_to_press = [to_rst(keys)] -dummy_docs = list(set(dummy_docs)) + result = " + ".join(keys_to_press) -single_filter = {} -multi_filter = {} -for kb in ipy_bindings: - doc = kb.handler.__doc__ - if not doc or doc in dummy_docs: - continue + if keys in INDISTINGUISHABLE_KEYS and add_alternatives: + alternative = INDISTINGUISHABLE_KEYS[keys] - shortcut = ' '.join([k if isinstance(k, str) else k.name for k in kb.keys]) - shortcut += shortcut.endswith('\\') and '\\' or '' - if hasattr(kb.filter, 'filters'): - flt = ' '.join(multi_filter_str(kb.filter)) - multi_filter[(shortcut, flt)] = sentencize(doc) - else: - single_filter[(shortcut, name(kb.filter))] = sentencize(doc) + result = ( + result + + " (or " + + format_prompt_keys(alternative, add_alternatives=False) + + ")" + ) + return result if __name__ == '__main__': here = Path(__file__).parent dest = here / "source" / "config" / "shortcuts" - def sort_key(item): - k, v = item - shortcut, flt = k - return (str(shortcut), str(flt)) - - for filters, output_filename in [ - (single_filter, "single_filtered"), - (multi_filter, "multi_filtered"), - ]: - with (dest / "{}.csv".format(output_filename)).open( - "w", encoding="utf-8" - ) as csv: - for (shortcut, flt), v in sorted(filters.items(), key=sort_key): - csv.write(":kbd:`{}`\t{}\t{}\n".format(shortcut, flt, v)) + ipy_bindings = create_ipython_shortcuts(_DummyTerminal(), for_all_platforms=True) + + session = PromptSession(key_bindings=ipy_bindings) + prompt_bindings = session.app.key_bindings + + assert prompt_bindings + # Ensure that we collected the default shortcuts + assert len(prompt_bindings.bindings) > len(ipy_bindings.bindings) + + bindings = bindings_from_prompt_toolkit(prompt_bindings) + + def sort_key(binding: Binding): + return binding.handler.identifier, binding.shortcut.filter + + filters = [] + with (dest / "table.tsv").open("w", encoding="utf-8") as csv: + for binding in sorted(bindings, key=sort_key): + sequence = ", ".join( + [format_prompt_keys(keys) for keys in binding.shortcut.keys_sequence] + ) + if binding.shortcut.filter == "always": + condition_label = "-" + else: + # we cannot fit all the columns as the filters got too complex over time + condition_label = "ⓘ" + + csv.write( + "\t".join( + [ + sequence, + sentencize(binding.handler.description) + + f" :raw-html:`
` `{binding.handler.identifier}`", + f':raw-html:`{condition_label}`', + ] + ) + + "\n" + ) diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000000..156db8c24b0 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,7 @@ +/* + Needed to revert problematic lack of wrapping in sphinx_rtd_theme, see: + https://github.com/readthedocs/sphinx_rtd_theme/issues/117 +*/ +.wy-table-responsive table.shortcuts td, .wy-table-responsive table.shortcuts th { + white-space: normal!important; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index d04d4637ba7..868c0d0e346 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -211,7 +211,6 @@ def filter(self, record): # given in html_static_path. # html_style = 'default.css' - # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None @@ -327,6 +326,10 @@ def filter(self, record): modindex_common_prefix = ['IPython.'] +def setup(app): + app.add_css_file("theme_overrides.css") + + # Cleanup # ------- # delete release info to avoid pickling errors from sphinx diff --git a/docs/source/config/shortcuts/index.rst b/docs/source/config/shortcuts/index.rst index 4103d92a7bd..e361ec26c5d 100755 --- a/docs/source/config/shortcuts/index.rst +++ b/docs/source/config/shortcuts/index.rst @@ -4,28 +4,23 @@ IPython shortcuts Available shortcuts in an IPython terminal. -.. warning:: +.. note:: - This list is automatically generated, and may not hold all available - shortcuts. In particular, it may depend on the version of ``prompt_toolkit`` - installed during the generation of this page. + This list is automatically generated. Key bindings defined in ``prompt_toolkit`` may differ + between installations depending on the ``prompt_toolkit`` version. -Single Filtered shortcuts -========================= - -.. csv-table:: - :header: Shortcut,Filter,Description - :widths: 30, 30, 100 - :delim: tab - :file: single_filtered.csv +* Comma-separated keys, e.g. :kbd:`Esc`, :kbd:`f`, indicate a sequence which can be activated by pressing the listed keys in succession. +* Plus-separated keys, e.g. :kbd:`Esc` + :kbd:`f` indicate a combination which requires pressing all keys simultaneously. +* Hover over the ⓘ icon in the filter column to see when the shortcut is active.g +.. role:: raw-html(raw) + :format: html -Multi Filtered shortcuts -======================== .. csv-table:: - :header: Shortcut,Filter,Description - :widths: 30, 30, 100 + :header: Shortcut,Description and identifier,Filter :delim: tab - :file: multi_filtered.csv + :class: shortcuts + :file: table.tsv + :widths: 20 75 5 From f6cb59f82a748f6225e8f9460722c8169a00d665 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 14:45:38 +0000 Subject: [PATCH 075/122] Fix mypy job, fix issues detected by mypy --- .github/workflows/mypy.yml | 2 ++ IPython/terminal/shortcuts/__init__.py | 12 ++++++------ IPython/terminal/shortcuts/auto_match.py | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 52f3e79ab9c..c7fa22c7210 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -29,11 +29,13 @@ jobs: pip install mypy pyflakes flake8 - name: Lint with mypy run: | + set -e mypy -p IPython.terminal mypy -p IPython.core.magics mypy -p IPython.core.guarded_eval mypy -p IPython.core.completer - name: Lint with pyflakes run: | + set -e flake8 IPython/core/magics/script.py flake8 IPython/core/magics/packaging.py diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index 6bc6ec7c39f..68eaf65ff60 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -11,7 +11,7 @@ import sys import re import os -from typing import Callable +from typing import Callable, Dict, Union from prompt_toolkit.application.current import get_app @@ -143,10 +143,10 @@ def all_quotes_paired(quote, buf): return paired focused_insert = (vi_insert_mode | emacs_insert_mode) & has_focus(DEFAULT_BUFFER) - _preceding_text_cache = {} - _following_text_cache = {} + _preceding_text_cache: Dict[Union[str, Callable], Condition] = {} + _following_text_cache: Dict[Union[str, Callable], Condition] = {} - def preceding_text(pattern): + def preceding_text(pattern: Union[str, Callable]): if pattern in _preceding_text_cache: return _preceding_text_cache[pattern] @@ -383,8 +383,8 @@ def set_input_mode(self, mode): self._input_mode = mode if shell.editing_mode == "vi" and shell.modal_cursor: - ViState._input_mode = InputMode.INSERT - ViState.input_mode = property(get_input_mode, set_input_mode) + ViState._input_mode = InputMode.INSERT # type: ignore + ViState.input_mode = property(get_input_mode, set_input_mode) # type: ignore return kb diff --git a/IPython/terminal/shortcuts/auto_match.py b/IPython/terminal/shortcuts/auto_match.py index 0976bb20336..bb0ca8b3169 100644 --- a/IPython/terminal/shortcuts/auto_match.py +++ b/IPython/terminal/shortcuts/auto_match.py @@ -50,7 +50,7 @@ def raw_string_parenthesis(event: KeyPressEvent): r".*(r|R)[\"'](-*)", event.current_buffer.document.current_line_before_cursor, ) - dashes = matches.group(2) or "" + dashes = matches.group(2) if matches else "" event.current_buffer.insert_text("()" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) @@ -61,7 +61,7 @@ def raw_string_bracket(event: KeyPressEvent): r".*(r|R)[\"'](-*)", event.current_buffer.document.current_line_before_cursor, ) - dashes = matches.group(2) or "" + dashes = matches.group(2) if matches else "" event.current_buffer.insert_text("[]" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) @@ -72,7 +72,7 @@ def raw_string_braces(event: KeyPressEvent): r".*(r|R)[\"'](-*)", event.current_buffer.document.current_line_before_cursor, ) - dashes = matches.group(2) or "" + dashes = matches.group(2) if matches else "" event.current_buffer.insert_text("{}" + dashes) event.current_buffer.cursor_left(len(dashes) + 1) From 1e51d378075a54559c257d91ff2e516bfced6cbc Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 17:58:04 +0000 Subject: [PATCH 076/122] Implement token-by-token autosuggestions --- IPython/terminal/shortcuts/__init__.py | 1 + IPython/terminal/shortcuts/autosuggestions.py | 52 +++++++++- IPython/terminal/tests/test_shortcuts.py | 94 +++++++++++++++++++ IPython/tests/test_shortcuts.py | 40 -------- 4 files changed, 145 insertions(+), 42 deletions(-) create mode 100644 IPython/terminal/tests/test_shortcuts.py delete mode 100644 IPython/tests/test_shortcuts.py diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index 68eaf65ff60..f25a8c1a869 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -333,6 +333,7 @@ def not_inside_unclosed_string(): kb.add("escape", "f", filter=focused_insert_vi & ebivim)( autosuggestions.accept_word ) + kb.add("c-right", filter=has_focus(DEFAULT_BUFFER))(autosuggestions.accept_token) # Simple Control keybindings key_cmd_dict = { diff --git a/IPython/terminal/shortcuts/autosuggestions.py b/IPython/terminal/shortcuts/autosuggestions.py index fe1e8d07b7c..158d988acae 100644 --- a/IPython/terminal/shortcuts/autosuggestions.py +++ b/IPython/terminal/shortcuts/autosuggestions.py @@ -1,7 +1,13 @@ import re +import tokenize +from io import StringIO +from typing import List, Optional + from prompt_toolkit.key_binding import KeyPressEvent from prompt_toolkit.key_binding.bindings import named_commands as nc +from IPython.utils.tokenutil import generate_tokens + # Needed for to accept autosuggestions in vi insert mode def accept_in_vi_insert_mode(event: KeyPressEvent): @@ -18,7 +24,7 @@ def accept_in_vi_insert_mode(event: KeyPressEvent): nc.end_of_line(event) -def accept(event): +def accept(event: KeyPressEvent): """Accept suggestion""" b = event.current_buffer suggestion = b.suggestion @@ -28,7 +34,7 @@ def accept(event): nc.forward_char(event) -def accept_word(event): +def accept_word(event: KeyPressEvent): """Fill partial suggestion by word""" b = event.current_buffer suggestion = b.suggestion @@ -37,3 +43,45 @@ def accept_word(event): b.insert_text(next((x for x in t if x), "")) else: nc.forward_word(event) + + +def accept_token(event: KeyPressEvent): + """Fill partial suggestion by token""" + b = event.current_buffer + suggestion = b.suggestion + + if suggestion: + prefix = b.text + text = prefix + suggestion.text + + tokens: List[Optional[str]] = [None, None, None] + substings = [""] + i = 0 + + for token in generate_tokens(StringIO(text).readline): + if token.type == tokenize.NEWLINE: + index = len(text) + else: + index = text.index(token[1], len(substings[-1])) + substings.append(text[:index]) + tokenized_so_far = substings[-1] + if tokenized_so_far.startswith(prefix): + if i == 0 and len(tokenized_so_far) > len(prefix): + tokens[0] = tokenized_so_far[len(prefix) :] + substings.append(tokenized_so_far) + i += 1 + tokens[i] = token[1] + if i == 2: + break + i += 1 + + if tokens[0]: + to_insert: str + insert_text = substings[-2] + if tokens[1] and len(tokens[1]) == 1: + insert_text = substings[-1] + to_insert = insert_text[len(prefix) :] + b.insert_text(to_insert) + return + + nc.forward_word(event) diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py new file mode 100644 index 00000000000..92242f75d38 --- /dev/null +++ b/IPython/terminal/tests/test_shortcuts.py @@ -0,0 +1,94 @@ +import pytest +from IPython.terminal.shortcuts.autosuggestions import ( + accept_in_vi_insert_mode, + accept_token, +) + +from unittest.mock import patch, Mock + + +def make_event(text, cursor, suggestion): + event = Mock() + event.current_buffer = Mock() + event.current_buffer.suggestion = Mock() + event.current_buffer.text = text + event.current_buffer.cursor_position = cursor + event.current_buffer.suggestion.text = suggestion + event.current_buffer.document = Mock() + event.current_buffer.document.get_end_of_line_position = Mock(return_value=0) + event.current_buffer.document.text = text + event.current_buffer.document.cursor_position = cursor + return event + + +@pytest.mark.parametrize( + "text, cursor, suggestion, called", + [ + ("123456", 6, "123456789", True), + ("123456", 3, "123456789", False), + ("123456 \n789", 6, "123456789", True), + ], +) +def test_autosuggest_at_EOL(text, cursor, suggestion, called): + """ + test that autosuggest is only applied at end of line. + """ + + event = make_event(text, cursor, suggestion) + event.current_buffer.insert_text = Mock() + accept_in_vi_insert_mode(event) + if called: + event.current_buffer.insert_text.assert_called() + else: + event.current_buffer.insert_text.assert_not_called() + # event.current_buffer.document.get_end_of_line_position.assert_called() + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def "), + ("d", "ef out(tag: str, n=50):", "ef "), + ("de ", "f out(tag: str, n=50):", "f "), + ("def", " out(tag: str, n=50):", " "), + ("def ", "out(tag: str, n=50):", "out("), + ("def o", "ut(tag: str, n=50):", "ut("), + ("def ou", "t(tag: str, n=50):", "t("), + ("def out", "(tag: str, n=50):", "("), + ("def out(", "tag: str, n=50):", "tag: "), + ("def out(t", "ag: str, n=50):", "ag: "), + ("def out(ta", "g: str, n=50):", "g: "), + ("def out(tag", ": str, n=50):", ": "), + ("def out(tag:", " str, n=50):", " "), + ("def out(tag: ", "str, n=50):", "str, "), + ("def out(tag: s", "tr, n=50):", "tr, "), + ("def out(tag: st", "r, n=50):", "r, "), + ("def out(tag: str", ", n=50):", ", n"), + ("def out(tag: str,", " n=50):", " n"), + ("def out(tag: str, ", "n=50):", "n="), + ("def out(tag: str, n", "=50):", "="), + ("def out(tag: str, n=", "50):", "50)"), + ("def out(tag: str, n=5", "0):", "0)"), + ("def out(tag: str, n=50", "):", "):"), + ("def out(tag: str, n=50)", ":", ":"), + ], +) +def test_autosuggest_token(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_token(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +def test_autosuggest_token_empty(): + full = "def out(tag: str, n=50):" + event = make_event(full, len(full), "") + event.current_buffer.insert_text = Mock() + + with patch( + "prompt_toolkit.key_binding.bindings.named_commands.forward_word" + ) as forward_word: + accept_token(event) + assert not event.current_buffer.insert_text.called + assert forward_word.called diff --git a/IPython/tests/test_shortcuts.py b/IPython/tests/test_shortcuts.py deleted file mode 100644 index 42edb92ba58..00000000000 --- a/IPython/tests/test_shortcuts.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest -from IPython.terminal.shortcuts import _apply_autosuggest - -from unittest.mock import Mock - - -def make_event(text, cursor, suggestion): - event = Mock() - event.current_buffer = Mock() - event.current_buffer.suggestion = Mock() - event.current_buffer.cursor_position = cursor - event.current_buffer.suggestion.text = suggestion - event.current_buffer.document = Mock() - event.current_buffer.document.get_end_of_line_position = Mock(return_value=0) - event.current_buffer.document.text = text - event.current_buffer.document.cursor_position = cursor - return event - - -@pytest.mark.parametrize( - "text, cursor, suggestion, called", - [ - ("123456", 6, "123456789", True), - ("123456", 3, "123456789", False), - ("123456 \n789", 6, "123456789", True), - ], -) -def test_autosuggest_at_EOL(text, cursor, suggestion, called): - """ - test that autosuggest is only applied at end of line. - """ - - event = make_event(text, cursor, suggestion) - event.current_buffer.insert_text = Mock() - _apply_autosuggest(event) - if called: - event.current_buffer.insert_text.assert_called() - else: - event.current_buffer.insert_text.assert_not_called() - # event.current_buffer.document.get_end_of_line_position.assert_called() From 2a5f51a098e49ab34b45a00070aa65f672b1c9c6 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:21:58 +0000 Subject: [PATCH 077/122] Implement traversal of autosuggestions and by-character fill --- IPython/terminal/interactiveshell.py | 20 +- IPython/terminal/shortcuts/__init__.py | 37 ++- IPython/terminal/shortcuts/auto_suggest.py | 255 ++++++++++++++++++ IPython/terminal/shortcuts/autosuggestions.py | 87 ------ IPython/terminal/tests/test_shortcuts.py | 2 +- docs/autogen_shortcuts.py | 1 + 6 files changed, 303 insertions(+), 99 deletions(-) create mode 100644 IPython/terminal/shortcuts/auto_suggest.py delete mode 100644 IPython/terminal/shortcuts/autosuggestions.py diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index c867b553f2e..0abff28db90 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -4,6 +4,7 @@ import os import sys from warnings import warn +from typing import Union as UnionType from IPython.core.async_helpers import get_asyncio_loop from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC @@ -49,6 +50,7 @@ from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook from .ptutils import IPythonPTCompleter, IPythonPTLexer from .shortcuts import create_ipython_shortcuts +from .shortcuts.auto_suggest import NavigableAutoSuggestFromHistory PTK3 = ptk_version.startswith('3.') @@ -183,7 +185,7 @@ class TerminalInteractiveShell(InteractiveShell): 'menus, decrease for short and wide.' ).tag(config=True) - pt_app = None + pt_app: UnionType[PromptSession, None] = None debugger_history = None debugger_history_file = Unicode( @@ -376,18 +378,25 @@ def _displayhook_class_default(self): ).tag(config=True) autosuggestions_provider = Unicode( - "AutoSuggestFromHistory", + "NavigableAutoSuggestFromHistory", help="Specifies from which source automatic suggestions are provided. " - "Can be set to `'AutoSuggestFromHistory`' or `None` to disable" - "automatic suggestions. Default is `'AutoSuggestFromHistory`'.", + "Can be set to ``'NavigableAutoSuggestFromHistory'`` (:kbd:`up` and " + ":kbd:`down` swap suggestions), ``'AutoSuggestFromHistory'``, " + " or ``None`` to disable automatic suggestions. " + "Default is `'NavigableAutoSuggestFromHistory`'.", allow_none=True, ).tag(config=True) def _set_autosuggestions(self, provider): + # disconnect old handler + if self.auto_suggest and isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory): + self.auto_suggest.disconnect() if provider is None: self.auto_suggest = None elif provider == "AutoSuggestFromHistory": self.auto_suggest = AutoSuggestFromHistory() + elif provider == "NavigableAutoSuggestFromHistory": + self.auto_suggest = NavigableAutoSuggestFromHistory() else: raise ValueError("No valid provider.") if self.pt_app: @@ -462,6 +471,8 @@ def prompt(): tempfile_suffix=".py", **self._extra_prompt_options() ) + if isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory): + self.auto_suggest.connect(self.pt_app) def _make_style_from_name_or_cls(self, name_or_cls): """ @@ -649,6 +660,7 @@ def init_alias(self): def __init__(self, *args, **kwargs): super(TerminalInteractiveShell, self).__init__(*args, **kwargs) + self.auto_suggest: UnionType[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] = None self._set_autosuggestions(self.autosuggestions_provider) self.init_prompt_toolkit_cli() self.init_term_title() diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index f25a8c1a869..3bb39066ec1 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -34,12 +34,24 @@ from prompt_toolkit.layout.layout import FocusableElement from IPython.utils.decorators import undoc -from . import auto_match as match, autosuggestions +from . import auto_match as match, auto_suggest __all__ = ["create_ipython_shortcuts"] +try: + # only added in 3.0.30 + from prompt_toolkit.filters import has_suggestion +except ImportError: + + @undoc + @Condition + def has_suggestion(): + buffer = get_app().current_buffer + return buffer.suggestion is not None and buffer.suggestion.text != "" + + @undoc @Condition def cursor_in_leading_ws(): @@ -324,16 +336,27 @@ def not_inside_unclosed_string(): # autosuggestions kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))( - autosuggestions.accept_in_vi_insert_mode + auto_suggest.accept_in_vi_insert_mode ) kb.add("c-e", filter=focused_insert_vi & ebivim)( - autosuggestions.accept_in_vi_insert_mode + auto_suggest.accept_in_vi_insert_mode + ) + kb.add("c-f", filter=focused_insert_vi)(auto_suggest.accept) + kb.add("escape", "f", filter=focused_insert_vi & ebivim)(auto_suggest.accept_word) + kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_token + ) + from functools import partial + + kb.add("up", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.swap_autosuggestion_up(shell.auto_suggest) + ) + kb.add("down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.swap_autosuggestion_down(shell.auto_suggest) ) - kb.add("c-f", filter=focused_insert_vi)(autosuggestions.accept) - kb.add("escape", "f", filter=focused_insert_vi & ebivim)( - autosuggestions.accept_word + kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_character ) - kb.add("c-right", filter=has_focus(DEFAULT_BUFFER))(autosuggestions.accept_token) # Simple Control keybindings key_cmd_dict = { diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py new file mode 100644 index 00000000000..0e8533f6572 --- /dev/null +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -0,0 +1,255 @@ +import re +import tokenize +from io import StringIO +from typing import Callable, List, Optional, Union + +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.key_binding import KeyPressEvent +from prompt_toolkit.key_binding.bindings import named_commands as nc +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory, Suggestion +from prompt_toolkit.document import Document +from prompt_toolkit.history import History +from prompt_toolkit.shortcuts import PromptSession + +from IPython.utils.tokenutil import generate_tokens + + +def _get_query(document: Document): + return document.text.rsplit("\n", 1)[-1] + + +class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory): + """ """ + + def __init__( + self, + ): + self.skip_lines = 0 + self._connected_apps = [] + + def reset_history_position(self, _: Buffer): + self.skip_lines = 0 + + def disconnect(self): + for pt_app in self._connected_apps: + text_insert_event = pt_app.default_buffer.on_text_insert + text_insert_event.remove_handler(self.reset_history_position) + + def connect(self, pt_app: PromptSession): + self._connected_apps.append(pt_app) + pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position) + + def get_suggestion( + self, buffer: Buffer, document: Document + ) -> Optional[Suggestion]: + text = _get_query(document) + + if text.strip(): + for suggestion, _ in self._find_next_match( + text, self.skip_lines, buffer.history + ): + return Suggestion(suggestion) + + return None + + def _find_match( + self, text: str, skip_lines: float, history: History, previous: bool + ): + line_number = -1 + + for string in reversed(list(history.get_strings())): + for line in reversed(string.splitlines()): + line_number += 1 + if not previous and line_number < skip_lines: + continue + # do not return empty suggestions as these + # close the auto-suggestion overlay (and are useless) + if line.startswith(text) and len(line) > len(text): + yield line[len(text) :], line_number + if previous and line_number >= skip_lines: + return + + def _find_next_match(self, text: str, skip_lines: float, history: History): + return self._find_match(text, skip_lines, history, previous=False) + + def _find_previous_match(self, text: str, skip_lines: float, history: History): + return reversed( + list(self._find_match(text, skip_lines, history, previous=True)) + ) + + def up(self, query: str, other_than: str, history: History): + for suggestion, line_number in self._find_next_match( + query, self.skip_lines, history + ): + # if user has history ['very.a', 'very', 'very.b'] and typed 'very' + # we want to switch from 'very.b' to 'very.a' because a) if they + # suggestion equals current text, prompt-toolit aborts suggesting + # b) user likely would not be interested in 'very' anyways (they + # already typed it). + if query + suggestion != other_than: + self.skip_lines = line_number + break + else: + # no matches found, cycle back to beginning + self.skip_lines = 0 + + def down(self, query: str, other_than: str, history: History): + for suggestion, line_number in self._find_previous_match( + query, self.skip_lines, history + ): + if query + suggestion != other_than: + self.skip_lines = line_number + break + else: + # no matches found, cycle to end + for suggestion, line_number in self._find_previous_match( + query, float("Inf"), history + ): + if query + suggestion != other_than: + self.skip_lines = line_number + break + + +# Needed for to accept autosuggestions in vi insert mode +def accept_in_vi_insert_mode(event: KeyPressEvent): + """Apply autosuggestion if at end of line.""" + b = event.current_buffer + d = b.document + after_cursor = d.text[d.cursor_position :] + lines = after_cursor.split("\n") + end_of_current_line = lines[0].strip() + suggestion = b.suggestion + if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): + b.insert_text(suggestion.text) + else: + nc.end_of_line(event) + + +def accept(event: KeyPressEvent): + """Accept autosuggestion""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion: + b.insert_text(suggestion.text) + else: + nc.forward_char(event) + + +def accept_word(event: KeyPressEvent): + """Fill partial autosuggestion by word""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion: + t = re.split(r"(\S+\s+)", suggestion.text) + b.insert_text(next((x for x in t if x), "")) + else: + nc.forward_word(event) + + +def accept_character(event: KeyPressEvent): + """Fill partial autosuggestion by character""" + b = event.current_buffer + suggestion = b.suggestion + if suggestion and suggestion.text: + b.insert_text(suggestion.text[0]) + + +def accept_token(event: KeyPressEvent): + """Fill partial autosuggestion by token""" + b = event.current_buffer + suggestion = b.suggestion + + if suggestion: + prefix = _get_query(b.document) + text = prefix + suggestion.text + + tokens: List[Optional[str]] = [None, None, None] + substrings = [""] + i = 0 + + for token in generate_tokens(StringIO(text).readline): + if token.type == tokenize.NEWLINE: + index = len(text) + else: + index = text.index(token[1], len(substrings[-1])) + substrings.append(text[:index]) + tokenized_so_far = substrings[-1] + if tokenized_so_far.startswith(prefix): + if i == 0 and len(tokenized_so_far) > len(prefix): + tokens[0] = tokenized_so_far[len(prefix) :] + substrings.append(tokenized_so_far) + i += 1 + tokens[i] = token[1] + if i == 2: + break + i += 1 + + if tokens[0]: + to_insert: str + insert_text = substrings[-2] + if tokens[1] and len(tokens[1]) == 1: + insert_text = substrings[-1] + to_insert = insert_text[len(prefix) :] + b.insert_text(to_insert) + return + + nc.forward_word(event) + + +Provider = Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] + + +def _swap_autosuggestion( + buffer: Buffer, + provider: NavigableAutoSuggestFromHistory, + direction_method: Callable, +): + """ + We skip most recent history entry (in either direction) if it equals the + current autosuggestion because if user cycles when auto-suggestion is shown + they most likely want something else than what was suggested (othewrise + they would have accepted the suggestion). + """ + suggestion = buffer.suggestion + if not suggestion: + return + + query = _get_query(buffer.document) + current = query + suggestion.text + + direction_method(query=query, other_than=current, history=buffer.history) + + new_suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = new_suggestion + + +def swap_autosuggestion_up(provider: Provider): + def swap_autosuggestion_up(event: KeyPressEvent): + """Get next autosuggestion from history.""" + if not isinstance(provider, NavigableAutoSuggestFromHistory): + return + + return _swap_autosuggestion( + buffer=event.current_buffer, provider=provider, direction_method=provider.up + ) + + swap_autosuggestion_up.__name__ = "swap_autosuggestion_up" + return swap_autosuggestion_up + + +def swap_autosuggestion_down( + provider: Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] +): + def swap_autosuggestion_down(event: KeyPressEvent): + """Get previous autosuggestion from history.""" + if not isinstance(provider, NavigableAutoSuggestFromHistory): + return + + return _swap_autosuggestion( + buffer=event.current_buffer, + provider=provider, + direction_method=provider.down, + ) + + swap_autosuggestion_down.__name__ = "swap_autosuggestion_down" + return swap_autosuggestion_down diff --git a/IPython/terminal/shortcuts/autosuggestions.py b/IPython/terminal/shortcuts/autosuggestions.py deleted file mode 100644 index 158d988acae..00000000000 --- a/IPython/terminal/shortcuts/autosuggestions.py +++ /dev/null @@ -1,87 +0,0 @@ -import re -import tokenize -from io import StringIO -from typing import List, Optional - -from prompt_toolkit.key_binding import KeyPressEvent -from prompt_toolkit.key_binding.bindings import named_commands as nc - -from IPython.utils.tokenutil import generate_tokens - - -# Needed for to accept autosuggestions in vi insert mode -def accept_in_vi_insert_mode(event: KeyPressEvent): - """Apply autosuggestion if at end of line.""" - b = event.current_buffer - d = b.document - after_cursor = d.text[d.cursor_position :] - lines = after_cursor.split("\n") - end_of_current_line = lines[0].strip() - suggestion = b.suggestion - if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): - b.insert_text(suggestion.text) - else: - nc.end_of_line(event) - - -def accept(event: KeyPressEvent): - """Accept suggestion""" - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - b.insert_text(suggestion.text) - else: - nc.forward_char(event) - - -def accept_word(event: KeyPressEvent): - """Fill partial suggestion by word""" - b = event.current_buffer - suggestion = b.suggestion - if suggestion: - t = re.split(r"(\S+\s+)", suggestion.text) - b.insert_text(next((x for x in t if x), "")) - else: - nc.forward_word(event) - - -def accept_token(event: KeyPressEvent): - """Fill partial suggestion by token""" - b = event.current_buffer - suggestion = b.suggestion - - if suggestion: - prefix = b.text - text = prefix + suggestion.text - - tokens: List[Optional[str]] = [None, None, None] - substings = [""] - i = 0 - - for token in generate_tokens(StringIO(text).readline): - if token.type == tokenize.NEWLINE: - index = len(text) - else: - index = text.index(token[1], len(substings[-1])) - substings.append(text[:index]) - tokenized_so_far = substings[-1] - if tokenized_so_far.startswith(prefix): - if i == 0 and len(tokenized_so_far) > len(prefix): - tokens[0] = tokenized_so_far[len(prefix) :] - substings.append(tokenized_so_far) - i += 1 - tokens[i] = token[1] - if i == 2: - break - i += 1 - - if tokens[0]: - to_insert: str - insert_text = substings[-2] - if tokens[1] and len(tokens[1]) == 1: - insert_text = substings[-1] - to_insert = insert_text[len(prefix) :] - b.insert_text(to_insert) - return - - nc.forward_word(event) diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index 92242f75d38..39cc6e24819 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -1,5 +1,5 @@ import pytest -from IPython.terminal.shortcuts.autosuggestions import ( +from IPython.terminal.shortcuts.auto_suggest import ( accept_in_vi_insert_mode, accept_token, ) diff --git a/docs/autogen_shortcuts.py b/docs/autogen_shortcuts.py index b5886ffa576..f8fd17bb51f 100755 --- a/docs/autogen_shortcuts.py +++ b/docs/autogen_shortcuts.py @@ -101,6 +101,7 @@ class _DummyTerminal: input_transformer_manager = None display_completions = None editing_mode = "emacs" + auto_suggest = None def create_identifier(handler: Callable): From 43d6a9b11a1aa2843edaf0d3ca1c3a4c4f699182 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:49:19 +0000 Subject: [PATCH 078/122] Accepting suggestions with cursor in place and resume on backspace --- IPython/terminal/shortcuts/__init__.py | 6 +++ IPython/terminal/shortcuts/auto_suggest.py | 49 +++++++++++++++++----- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index 3bb39066ec1..84d599f237a 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -357,6 +357,12 @@ def not_inside_unclosed_string(): kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_character ) + kb.add("left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_and_keep_cursor + ) + kb.add("backspace", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.backspace_and_resume_hint + ) # Simple Control keybindings key_cmd_dict = { diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 0e8533f6572..8124f5c4b5d 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -37,6 +37,8 @@ def disconnect(self): def connect(self, pt_app: PromptSession): self._connected_apps.append(pt_app) + # note: `on_text_changed` could be used for a bit different behaviour + # on character deletion (i.e. reseting history position on backspace) pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position) def get_suggestion( @@ -113,35 +115,35 @@ def down(self, query: str, other_than: str, history: History): # Needed for to accept autosuggestions in vi insert mode def accept_in_vi_insert_mode(event: KeyPressEvent): """Apply autosuggestion if at end of line.""" - b = event.current_buffer - d = b.document + buffer = event.current_buffer + d = buffer.document after_cursor = d.text[d.cursor_position :] lines = after_cursor.split("\n") end_of_current_line = lines[0].strip() - suggestion = b.suggestion + suggestion = buffer.suggestion if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""): - b.insert_text(suggestion.text) + buffer.insert_text(suggestion.text) else: nc.end_of_line(event) def accept(event: KeyPressEvent): """Accept autosuggestion""" - b = event.current_buffer - suggestion = b.suggestion + buffer = event.current_buffer + suggestion = buffer.suggestion if suggestion: - b.insert_text(suggestion.text) + buffer.insert_text(suggestion.text) else: nc.forward_char(event) def accept_word(event: KeyPressEvent): """Fill partial autosuggestion by word""" - b = event.current_buffer - suggestion = b.suggestion + buffer = event.current_buffer + suggestion = buffer.suggestion if suggestion: t = re.split(r"(\S+\s+)", suggestion.text) - b.insert_text(next((x for x in t if x), "")) + buffer.insert_text(next((x for x in t if x), "")) else: nc.forward_word(event) @@ -154,6 +156,33 @@ def accept_character(event: KeyPressEvent): b.insert_text(suggestion.text[0]) +def accept_and_keep_cursor(event: KeyPressEvent): + """Accept autosuggestion and keep cursor in place""" + buffer = event.current_buffer + old_position = buffer.cursor_position + suggestion = buffer.suggestion + if suggestion: + buffer.insert_text(suggestion.text) + buffer.cursor_position = old_position + else: + nc.backward_char(event) + + +def backspace_and_resume_hint(event: KeyPressEvent): + """Resume autosuggestions after deleting last character""" + current_buffer = event.current_buffer + + def resume_hinting(buffer: Buffer): + if buffer.auto_suggest: + suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document) + if suggestion: + buffer.suggestion = suggestion + current_buffer.on_text_changed.remove_handler(resume_hinting) + + current_buffer.on_text_changed.add_handler(resume_hinting) + nc.backward_delete_char(event) + + def accept_token(event: KeyPressEvent): """Fill partial autosuggestion by token""" b = event.current_buffer From dfb5353105b3d33647bb943df32ffbad13f98978 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:56:22 +0000 Subject: [PATCH 079/122] Accept with cursor in place with ctrl + down, move left after accepting --- IPython/terminal/shortcuts/__init__.py | 3 +++ IPython/terminal/shortcuts/auto_suggest.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index 84d599f237a..c4af1378df8 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -358,6 +358,9 @@ def not_inside_unclosed_string(): auto_suggest.accept_character ) kb.add("left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + auto_suggest.accept_and_move_cursor_left + ) + kb.add("c-down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_and_keep_cursor ) kb.add("backspace", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 8124f5c4b5d..19ecb865378 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -164,8 +164,12 @@ def accept_and_keep_cursor(event: KeyPressEvent): if suggestion: buffer.insert_text(suggestion.text) buffer.cursor_position = old_position - else: - nc.backward_char(event) + + +def accept_and_move_cursor_left(event: KeyPressEvent): + """Accept autosuggestion and move cursor left""" + accept_and_keep_cursor(event) + nc.backward_char(event) def backspace_and_resume_hint(event: KeyPressEvent): From 039c83ae7ba2e4dfa737692bd90c3b03c534c2c7 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Mon, 9 Jan 2023 00:33:09 +0000 Subject: [PATCH 080/122] Lint and add more tests --- IPython/terminal/interactiveshell.py | 8 +- IPython/terminal/shortcuts/auto_suggest.py | 3 +- IPython/terminal/tests/test_shortcuts.py | 139 +++++++++++++++++++++ 3 files changed, 146 insertions(+), 4 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 0abff28db90..be81c51a120 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -389,7 +389,9 @@ def _displayhook_class_default(self): def _set_autosuggestions(self, provider): # disconnect old handler - if self.auto_suggest and isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory): + if self.auto_suggest and isinstance( + self.auto_suggest, NavigableAutoSuggestFromHistory + ): self.auto_suggest.disconnect() if provider is None: self.auto_suggest = None @@ -660,7 +662,9 @@ def init_alias(self): def __init__(self, *args, **kwargs): super(TerminalInteractiveShell, self).__init__(*args, **kwargs) - self.auto_suggest: UnionType[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] = None + self.auto_suggest: UnionType[ + AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None + ] = None self._set_autosuggestions(self.autosuggestions_provider) self.init_prompt_toolkit_cli() self.init_term_title() diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 19ecb865378..988853f25b3 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -58,7 +58,6 @@ def _find_match( self, text: str, skip_lines: float, history: History, previous: bool ): line_number = -1 - for string in reversed(list(history.get_strings())): for line in reversed(string.splitlines()): line_number += 1 @@ -167,7 +166,7 @@ def accept_and_keep_cursor(event: KeyPressEvent): def accept_and_move_cursor_left(event: KeyPressEvent): - """Accept autosuggestion and move cursor left""" + """Accept autosuggestion and move cursor left in place""" accept_and_keep_cursor(event) nc.backward_char(event) diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index 39cc6e24819..da8e841eddf 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -2,8 +2,18 @@ from IPython.terminal.shortcuts.auto_suggest import ( accept_in_vi_insert_mode, accept_token, + accept_character, + accept_word, + accept_and_keep_cursor, + NavigableAutoSuggestFromHistory, + swap_autosuggestion_up, + swap_autosuggestion_down, ) +from prompt_toolkit.history import InMemoryHistory +from prompt_toolkit.shortcuts import PromptSession +from prompt_toolkit.buffer import Buffer + from unittest.mock import patch, Mock @@ -81,6 +91,59 @@ def test_autosuggest_token(text, suggestion, expected): assert event.current_buffer.insert_text.call_args[0] == (expected,) +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "d"), + ("d", "ef out(tag: str, n=50):", "e"), + ("de ", "f out(tag: str, n=50):", "f"), + ("def", " out(tag: str, n=50):", " "), + ], +) +def test_accept_character(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_character(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def "), + ("d", "ef out(tag: str, n=50):", "ef "), + ("de", "f out(tag: str, n=50):", "f "), + ("def", " out(tag: str, n=50):", " "), + # (this is why we also have accept_token) + ("def ", "out(tag: str, n=50):", "out(tag: "), + ], +) +def test_accept_word(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + event.current_buffer.insert_text = Mock() + accept_word(event) + assert event.current_buffer.insert_text.called + assert event.current_buffer.insert_text.call_args[0] == (expected,) + + +@pytest.mark.parametrize( + "text, suggestion, expected, cursor", + [ + ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):", 0), + ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):", 4), + ], +) +def test_accept_and_keep_cursor(text, suggestion, expected, cursor): + event = make_event(text, cursor, suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + accept_and_keep_cursor(event) + assert buffer.insert_text.called + assert buffer.insert_text.call_args[0] == (expected,) + assert buffer.cursor_position == cursor + + def test_autosuggest_token_empty(): full = "def out(tag: str, n=50):" event = make_event(full, len(full), "") @@ -92,3 +155,79 @@ def test_autosuggest_token_empty(): accept_token(event) assert not event.current_buffer.insert_text.called assert forward_word.called + + +async def test_navigable_provider(): + provider = NavigableAutoSuggestFromHistory() + history = InMemoryHistory(history_strings=["very_a", "very", "very_b", "very_c"]) + buffer = Buffer(history=history) + + async for _ in history.load(): + pass + + buffer.cursor_position = 5 + buffer.text = "very" + + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + + event = Mock() + event.current_buffer = buffer + + def get_suggestion(): + suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + return suggestion + + assert get_suggestion().text == "_c" + + # should go up + up(event) + assert get_suggestion().text == "_b" + + # should skip over 'very' which is identical to buffer content + up(event) + assert get_suggestion().text == "_a" + + # should cycle back to beginning + up(event) + assert get_suggestion().text == "_c" + + # should cycle back through end boundary + down(event) + assert get_suggestion().text == "_a" + + down(event) + assert get_suggestion().text == "_b" + + down(event) + assert get_suggestion().text == "_c" + + down(event) + assert get_suggestion().text == "_a" + + +def test_navigable_provider_connection(): + provider = NavigableAutoSuggestFromHistory() + provider.skip_lines = 1 + + session_1 = PromptSession() + provider.connect(session_1) + + assert provider.skip_lines == 1 + session_1.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 0 + + session_2 = PromptSession() + provider.connect(session_2) + provider.skip_lines = 2 + + assert provider.skip_lines == 2 + session_2.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 0 + + provider.skip_lines = 3 + provider.disconnect() + session_1.default_buffer.on_text_insert.fire() + session_2.default_buffer.on_text_insert.fire() + assert provider.skip_lines == 3 From c97747aab90214f1c9b682aebe3f384a136caee0 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Mon, 9 Jan 2023 00:43:53 +0000 Subject: [PATCH 081/122] Mock session to avoid Windows issues on CI. Instantiating a real session on Windows CI leads to exception: ``` prompt_toolkit.output.win32.NoConsoleScreenBufferError: No Windows console found. Are you running cmd.exe? ``` with the following traceback: ``` IPython\terminal\tests\test_shortcuts.py:214: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\shortcuts\prompt.py:476: in __init__ self.app = self._create_application(editing_mode, erase_when_done) C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\shortcuts\prompt.py:765: in _create_application output=self._output, C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\application\application.py:282: in __init__ self.output = output or session.output C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\application\current.py:71: in output self._output = create_output() C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\output\defaults.py:85: in create_output return Win32Output(stdout, default_color_depth=color_depth_from_env) C:\hostedtoolcache\windows\PyPy\3.8.15\x86\lib\site-packages\prompt_toolkit\output\win32.py:114: in __init__ info = self.get_win32_screen_buffer_info() ``` --- IPython/terminal/tests/test_shortcuts.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index da8e841eddf..21434b74d18 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -11,7 +11,6 @@ ) from prompt_toolkit.history import InMemoryHistory -from prompt_toolkit.shortcuts import PromptSession from prompt_toolkit.buffer import Buffer from unittest.mock import patch, Mock @@ -207,18 +206,24 @@ def get_suggestion(): assert get_suggestion().text == "_a" +def create_session_mock(): + session = Mock() + session.default_buffer = Buffer() + return session + + def test_navigable_provider_connection(): provider = NavigableAutoSuggestFromHistory() provider.skip_lines = 1 - session_1 = PromptSession() + session_1 = create_session_mock() provider.connect(session_1) assert provider.skip_lines == 1 session_1.default_buffer.on_text_insert.fire() assert provider.skip_lines == 0 - session_2 = PromptSession() + session_2 = create_session_mock() provider.connect(session_2) provider.skip_lines = 2 From dda97ea16fcecb56af542268031d6c02afb1d17a Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Mon, 9 Jan 2023 00:59:31 +0000 Subject: [PATCH 082/122] Add two final tests to increase coverage --- IPython/terminal/tests/test_shortcuts.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index 21434b74d18..9f158434487 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -1,5 +1,6 @@ import pytest from IPython.terminal.shortcuts.auto_suggest import ( + accept, accept_in_vi_insert_mode, accept_token, accept_character, @@ -12,6 +13,7 @@ from prompt_toolkit.history import InMemoryHistory from prompt_toolkit.buffer import Buffer +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from unittest.mock import patch, Mock @@ -30,6 +32,22 @@ def make_event(text, cursor, suggestion): return event +@pytest.mark.parametrize( + "text, suggestion, expected", + [ + ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):"), + ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):"), + ], +) +def test_accept(text, suggestion, expected): + event = make_event(text, len(text), suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + accept(event) + assert buffer.insert_text.called + assert buffer.insert_text.call_args[0] == (expected,) + + @pytest.mark.parametrize( "text, cursor, suggestion, called", [ @@ -156,6 +174,17 @@ def test_autosuggest_token_empty(): assert forward_word.called +def test_other_providers(): + """Ensure that swapping autosuggestions does not break with other providers""" + provider = AutoSuggestFromHistory() + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + event = Mock() + event.current_buffer = Buffer() + assert up(event) is None + assert down(event) is None + + async def test_navigable_provider(): provider = NavigableAutoSuggestFromHistory() history = InMemoryHistory(history_strings=["very_a", "very", "very_b", "very_c"]) From 2e0a730903561765a90957e5ce652f74790890ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Mon, 9 Jan 2023 02:29:12 +0000 Subject: [PATCH 083/122] Fix typos in a comment --- IPython/terminal/shortcuts/auto_suggest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 988853f25b3..8c699d8a23d 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -83,8 +83,8 @@ def up(self, query: str, other_than: str, history: History): query, self.skip_lines, history ): # if user has history ['very.a', 'very', 'very.b'] and typed 'very' - # we want to switch from 'very.b' to 'very.a' because a) if they - # suggestion equals current text, prompt-toolit aborts suggesting + # we want to switch from 'very.b' to 'very.a' because a) if the + # suggestion equals current text, prompt-toolkit aborts suggesting # b) user likely would not be interested in 'very' anyways (they # already typed it). if query + suggestion != other_than: From 73fd29a13a83b077f3828d88daabdff23a8e74a8 Mon Sep 17 00:00:00 2001 From: Garland Zhang Date: Wed, 11 Jan 2023 16:31:59 -0800 Subject: [PATCH 084/122] Update script.py Handle non utf-8 characters during decoding for %%bash --- IPython/core/magics/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index 9fd2fc6c0dd..e0615c0ca85 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -210,7 +210,7 @@ def in_thread(coro): async def _handle_stream(stream, stream_arg, file_object): while True: - line = (await stream.readline()).decode("utf8") + line = (await stream.readline()).decode("utf8", errors="replace") if not line: break if stream_arg: From 462382b3e30a006427d71c7e046b6e6164bda0a2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 16 Jan 2023 17:16:00 +0100 Subject: [PATCH 085/122] MAINT: Remove usage of traitlets. Now this is mostly validate at typechecheck time, instead of runtime. We don't use any validation logic so I'm unsure it is really necessary. --- IPython/core/application.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/IPython/core/application.py b/IPython/core/application.py index 26c061661ac..2aa0f104399 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -123,9 +123,8 @@ def load_subconfig(self, fname, path=None, profile=None): return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path) class BaseIPythonApplication(Application): - - name = u'ipython' - description = Unicode(u'IPython: an enhanced interactive Python shell.') + name = "ipython" + description = "IPython: an enhanced interactive Python shell." version = Unicode(release.version) aliases = base_aliases From 46c503d07bb4def76e824c54f4712a5fa4e7538d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 16 Jan 2023 17:15:26 +0100 Subject: [PATCH 086/122] MISC docs, cleanup and typing (in progress). --- IPython/terminal/ipapp.py | 4 ++-- IPython/terminal/shortcuts/__init__.py | 4 ++-- IPython/terminal/shortcuts/auto_suggest.py | 27 +++++++++++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index df4648b8914..6280bce3b20 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -156,7 +156,7 @@ def make_report(self,traceback): flags.update(frontend_flags) aliases = dict(base_aliases) -aliases.update(shell_aliases) +aliases.update(shell_aliases) # type: ignore[arg-type] #----------------------------------------------------------------------------- # Main classes and functions @@ -180,7 +180,7 @@ def start(self): class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): name = u'ipython' description = usage.cl_usage - crash_handler_class = IPAppCrashHandler + crash_handler_class = IPAppCrashHandler # typing: ignore[assignment] examples = _examples flags = flags diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index c4af1378df8..bc0c95c6f0b 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -163,11 +163,11 @@ def preceding_text(pattern: Union[str, Callable]): return _preceding_text_cache[pattern] if callable(pattern): - def _preceding_text(): app = get_app() before_cursor = app.current_buffer.document.current_line_before_cursor - return bool(pattern(before_cursor)) + # mypy can't infer if(callable): https://github.com/python/mypy/issues/3603 + return bool(pattern(before_cursor)) # type: ignore[operator] else: m = re.compile(pattern) diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 8c699d8a23d..f623f8e1acf 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -1,7 +1,7 @@ import re import tokenize from io import StringIO -from typing import Callable, List, Optional, Union +from typing import Callable, List, Optional, Union, Generator, Tuple from prompt_toolkit.buffer import Buffer from prompt_toolkit.key_binding import KeyPressEvent @@ -19,7 +19,11 @@ def _get_query(document: Document): class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory): - """ """ + """ + A subclass of AutoSuggestFromHistory that allow navigation to next/previous + suggestion from history. To do so it remembers the current position, but it + state need to carefully be cleared on the right events. + """ def __init__( self, @@ -56,7 +60,24 @@ def get_suggestion( def _find_match( self, text: str, skip_lines: float, history: History, previous: bool - ): + ) -> Generator[Tuple[str, float], None, None]: + """ + text: str + + skip_lines: float + float is used as the base value is +inf + + Yields + ------ + Tuple with: + str: + current suggestion. + float: + will actually yield only ints, which is passed back via skip_lines, + which may be a +inf (float) + + + """ line_number = -1 for string in reversed(list(history.get_strings())): for line in reversed(string.splitlines()): From 577c3b1225aae1bf9d2a71f64c7cbb848fbbfc54 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 17 Jan 2023 09:04:55 +0100 Subject: [PATCH 087/122] Misc review cleanup. Uniformize import, and put minimal prompt toolkit to 3.30 --- IPython/terminal/shortcuts/__init__.py | 56 +++++++++++----------- IPython/terminal/shortcuts/auto_suggest.py | 2 +- setup.cfg | 2 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index bc0c95c6f0b..a8b07026d31 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -6,52 +6,38 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -import warnings +import os +import re import signal import sys -import re -import os +import warnings from typing import Callable, Dict, Union - from prompt_toolkit.application.current import get_app from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER +from prompt_toolkit.filters import Condition, emacs_insert_mode, has_completions +from prompt_toolkit.filters import has_focus as has_focus_impl from prompt_toolkit.filters import ( - has_focus as has_focus_impl, has_selection, - Condition, + has_suggestion, vi_insert_mode, - emacs_insert_mode, - has_completions, vi_mode, ) +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.bindings import named_commands as nc from prompt_toolkit.key_binding.bindings.completion import ( display_completions_like_readline, ) -from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.key_binding.bindings import named_commands as nc from prompt_toolkit.key_binding.vi_state import InputMode, ViState from prompt_toolkit.layout.layout import FocusableElement +from IPython.terminal.shortcuts import auto_match as match +from IPython.terminal.shortcuts import auto_suggest from IPython.utils.decorators import undoc -from . import auto_match as match, auto_suggest - __all__ = ["create_ipython_shortcuts"] -try: - # only added in 3.0.30 - from prompt_toolkit.filters import has_suggestion -except ImportError: - - @undoc - @Condition - def has_suggestion(): - buffer = get_app().current_buffer - return buffer.suggestion is not None and buffer.suggestion.text != "" - - @undoc @Condition def cursor_in_leading_ws(): @@ -66,8 +52,24 @@ def has_focus(value: FocusableElement): return Condition(tester) -def create_ipython_shortcuts(shell, for_all_platforms: bool = False): - """Set up the prompt_toolkit keyboard shortcuts for IPython.""" +def create_ipython_shortcuts(shell, for_all_platforms: bool = False) -> KeyBindings: + """Set up the prompt_toolkit keyboard shortcuts for IPython. + + Parameters + ---------- + shell: InteractiveShell + The current IPython shell Instance + for_all_platforms: bool (default false) + This parameter is mostly used in generating the documentation + to create the shortcut binding for all the platforms, and export + them. + + Returns + ------- + KeyBindings + the keybinding instance for prompt toolkit. + + """ # Warning: if possible, do NOT define handler functions in the locals # scope of this function, instead define functions in the global # scope, or a separate module, and include a user-friendly docstring @@ -613,8 +615,8 @@ def open_input_in_editor(event): from IPython.core.error import TryNext from IPython.lib.clipboard import ( ClipboardEmpty, - win32_clipboard_get, tkinter_clipboard_get, + win32_clipboard_get, ) @undoc diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index f623f8e1acf..f7fb47e8cb5 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -260,7 +260,7 @@ def _swap_autosuggestion( """ We skip most recent history entry (in either direction) if it equals the current autosuggestion because if user cycles when auto-suggestion is shown - they most likely want something else than what was suggested (othewrise + they most likely want something else than what was suggested (otherwise they would have accepted the suggestion). """ suggestion = buffer.suggestion diff --git a/setup.cfg b/setup.cfg index de327aba545..d196214d19a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = matplotlib-inline pexpect>4.3; sys_platform != "win32" pickleshare - prompt_toolkit>=3.0.11,<3.1.0 + prompt_toolkit>=3.0.30,<3.1.0 pygments>=2.4.0 stack_data traitlets>=5 From ad41919ea90c23c024a8c0e58c2224d21815de80 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 17 Jan 2023 09:10:49 +0100 Subject: [PATCH 088/122] add docs --- IPython/terminal/shortcuts/auto_match.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/IPython/terminal/shortcuts/auto_match.py b/IPython/terminal/shortcuts/auto_match.py index bb0ca8b3169..c29f97a268c 100644 --- a/IPython/terminal/shortcuts/auto_match.py +++ b/IPython/terminal/shortcuts/auto_match.py @@ -1,3 +1,9 @@ +""" +Utilities function for keybinding with prompt toolkit. + +This will be bound to specific key press and filter modes, +like whether we are in edit mode, and whether the completer is open. +""" import re from prompt_toolkit.key_binding import KeyPressEvent From 3c529d3cbc6fa3eb736544860ad984c0198bcb35 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 17 Jan 2023 09:21:51 +0100 Subject: [PATCH 089/122] misc typing --- IPython/terminal/interactiveshell.py | 10 +++++---- IPython/terminal/shortcuts/auto_suggest.py | 26 +++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index be81c51a120..7213809e5fd 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -144,6 +144,10 @@ class PtkHistoryAdapter(History): """ + auto_suggest: UnionType[ + AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None + ] + def __init__(self, shell): super().__init__() self.shell = shell @@ -660,11 +664,9 @@ def init_alias(self): self.alias_manager.soft_define_alias(cmd, cmd) - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super(TerminalInteractiveShell, self).__init__(*args, **kwargs) - self.auto_suggest: UnionType[ - AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None - ] = None + self.auto_suggest = None self._set_autosuggestions(self.autosuggestions_provider) self.init_prompt_toolkit_cli() self.init_term_title() diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index f7fb47e8cb5..a8e1d7bfeac 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -1,7 +1,7 @@ import re import tokenize from io import StringIO -from typing import Callable, List, Optional, Union, Generator, Tuple +from typing import Callable, List, Optional, Union, Generator, Tuple, Sequence from prompt_toolkit.buffer import Buffer from prompt_toolkit.key_binding import KeyPressEvent @@ -62,10 +62,18 @@ def _find_match( self, text: str, skip_lines: float, history: History, previous: bool ) -> Generator[Tuple[str, float], None, None]: """ - text: str - - skip_lines: float - float is used as the base value is +inf + text : str + Text content to find a match for, the user cursor is most of the + time at the end of this text. + skip_lines : float + number of items to skip in the search, this is used to indicate how + far in the list the user has navigated by pressing up or down. + The float type is used as the base value is +inf + history : History + prompt_toolkit History instance to fetch previous entries from. + previous : bool + Direction of the search, whether we are looking previous match + (True), or next match (False). Yields ------ @@ -91,7 +99,9 @@ def _find_match( if previous and line_number >= skip_lines: return - def _find_next_match(self, text: str, skip_lines: float, history: History): + def _find_next_match( + self, text: str, skip_lines: float, history: History + ) -> Generator[Tuple[str, float], None, None]: return self._find_match(text, skip_lines, history, previous=False) def _find_previous_match(self, text: str, skip_lines: float, history: History): @@ -99,7 +109,7 @@ def _find_previous_match(self, text: str, skip_lines: float, history: History): list(self._find_match(text, skip_lines, history, previous=True)) ) - def up(self, query: str, other_than: str, history: History): + def up(self, query: str, other_than: str, history: History) -> None: for suggestion, line_number in self._find_next_match( query, self.skip_lines, history ): @@ -115,7 +125,7 @@ def up(self, query: str, other_than: str, history: History): # no matches found, cycle back to beginning self.skip_lines = 0 - def down(self, query: str, other_than: str, history: History): + def down(self, query: str, other_than: str, history: History) -> None: for suggestion, line_number in self._find_previous_match( query, self.skip_lines, history ): From 3e2655952703c7426f10b9e9d86fde2a8cd0391e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 17 Jan 2023 09:36:11 +0100 Subject: [PATCH 090/122] Reformat and move some mapping to global location. --- IPython/terminal/shortcuts/__init__.py | 23 +++++++++-------------- IPython/terminal/shortcuts/auto_match.py | 8 ++++++++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index a8b07026d31..d287368272a 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -165,6 +165,7 @@ def preceding_text(pattern: Union[str, Callable]): return _preceding_text_cache[pattern] if callable(pattern): + def _preceding_text(): app = get_app() before_cursor = app.current_buffer.document.current_line_before_cursor @@ -215,12 +216,18 @@ def not_inside_unclosed_string(): return not ('"' in s or "'" in s) # auto match - auto_match_parens = {"(": match.parenthesis, "[": match.brackets, "{": match.braces} - for key, cmd in auto_match_parens.items(): + for key, cmd in match.auto_match_parens.items(): kb.add(key, filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))( cmd ) + # raw string + for key, cmd in match.auto_match_parens_raw_string.items(): + kb.add( + key, + filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"), + )(cmd) + kb.add( '"', filter=focused_insert @@ -255,18 +262,6 @@ def not_inside_unclosed_string(): & preceding_text(r"^.*''$"), )(match.docstring_single_quotes) - # raw string - auto_match_parens_raw_string = { - "(": match.raw_string_parenthesis, - "[": match.raw_string_bracket, - "{": match.raw_string_braces, - } - for key, cmd in auto_match_parens_raw_string.items(): - kb.add( - key, - filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"), - )(cmd) - # just move cursor kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))( match.skip_over diff --git a/IPython/terminal/shortcuts/auto_match.py b/IPython/terminal/shortcuts/auto_match.py index c29f97a268c..46cb1bd8754 100644 --- a/IPython/terminal/shortcuts/auto_match.py +++ b/IPython/terminal/shortcuts/auto_match.py @@ -94,3 +94,11 @@ def delete_pair(event: KeyPressEvent): """Delete auto-closed parenthesis""" event.current_buffer.delete() event.current_buffer.delete_before_cursor() + + +auto_match_parens = {"(": parenthesis, "[": brackets, "{": braces} +auto_match_parens_raw_string = { + "(": raw_string_parenthesis, + "[": raw_string_bracket, + "{": raw_string_braces, +} From e0fc2386083aed55b44d4c50661fb53bd4170a3f Mon Sep 17 00:00:00 2001 From: nfgf Date: Sat, 21 Jan 2023 13:03:39 -0500 Subject: [PATCH 091/122] Documentation update --- IPython/core/magic.py | 2 +- docs/source/config/custommagics.rst | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 95653dc7893..4f9e4e548f7 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -280,7 +280,7 @@ def no_var_expand(magic_func): def output_can_be_silenced(magic_func): """Mark a magic function so its output may be silenced. - The output is silenced if the Python expression used as a parameter of + The output is silenced if the Python code used as a parameter of the magic ends in a semicolon, not counting a Python comment that can follow it. """ diff --git a/docs/source/config/custommagics.rst b/docs/source/config/custommagics.rst index 99d4068773c..0a37b858a4c 100644 --- a/docs/source/config/custommagics.rst +++ b/docs/source/config/custommagics.rst @@ -139,13 +139,26 @@ Accessing user namespace and local scope ======================================== When creating line magics, you may need to access surrounding scope to get user -variables (e.g when called inside functions). IPython provide the +variables (e.g when called inside functions). IPython provides the ``@needs_local_scope`` decorator that can be imported from ``IPython.core.magics``. When decorated with ``@needs_local_scope`` a magic will be passed ``local_ns`` as an argument. As a convenience ``@needs_local_scope`` can also be applied to cell magics even if cell magics cannot appear at local scope context. +Silencing the magic output +========================== + +Sometimes it may be useful to define a magic that can be silenced the same way +that non-magic expressions can, i.e., by appending a semicolon at the end of the Python +code to be executed. That can be achieved by decorating the magic function with +the decorator ``@output_can_be_silenced`` that can be imported from +``IPython.core.magics``. When this decorator is used, IPython will parse the Python +code used by the magic and, if the last token is a ``;``, the output created by the +magic will not show up on the screen. If you want to see an example of this decorator +in action, take a look on the ``time`` magic defined in +``IPython.core.magics.execution.py``. + Complete Example ================ From fc700ddd1756a4302707abacf62198d1c3809738 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 22 Jan 2023 13:14:48 +0000 Subject: [PATCH 092/122] Discard auto-suggestion on `Esc` --- IPython/terminal/shortcuts/__init__.py | 5 +++-- IPython/terminal/shortcuts/auto_suggest.py | 6 ++++++ IPython/terminal/tests/test_shortcuts.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index d287368272a..ad4dc39d870 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -343,8 +343,9 @@ def not_inside_unclosed_string(): kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_token ) - from functools import partial - + kb.add("escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER), eager=True)( + auto_suggest.discard + ) kb.add("up", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.swap_autosuggestion_up(shell.auto_suggest) ) diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index a8e1d7bfeac..733a46d416f 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -167,6 +167,12 @@ def accept(event: KeyPressEvent): nc.forward_char(event) +def discard(event: KeyPressEvent): + """Discard autosuggestion""" + buffer = event.current_buffer + buffer.suggestion = None + + def accept_word(event: KeyPressEvent): """Fill partial autosuggestion by word""" buffer = event.current_buffer diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index 9f158434487..a43a4ba2605 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -6,6 +6,7 @@ accept_character, accept_word, accept_and_keep_cursor, + discard, NavigableAutoSuggestFromHistory, swap_autosuggestion_up, swap_autosuggestion_down, @@ -48,6 +49,22 @@ def test_accept(text, suggestion, expected): assert buffer.insert_text.call_args[0] == (expected,) +@pytest.mark.parametrize( + "text, suggestion", + [ + ("", "def out(tag: str, n=50):"), + ("def ", "out(tag: str, n=50):"), + ], +) +def test_discard(text, suggestion): + event = make_event(text, len(text), suggestion) + buffer = event.current_buffer + buffer.insert_text = Mock() + discard(event) + assert not buffer.insert_text.called + assert buffer.suggestion is None + + @pytest.mark.parametrize( "text, cursor, suggestion, called", [ From 9c028ecc7fcd9ec8ba89cc60f4b49096dffa86ca Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 22 Jan 2023 20:01:51 +0000 Subject: [PATCH 093/122] Autosuggest: only navigate on edges of doc, show in multi-line doc --- IPython/terminal/interactiveshell.py | 51 ++++++++++++------ IPython/terminal/shortcuts/__init__.py | 44 +++++++++++++--- IPython/terminal/shortcuts/auto_suggest.py | 60 ++++++++++++++++++++-- IPython/terminal/tests/test_shortcuts.py | 42 +++++++++++++-- 4 files changed, 166 insertions(+), 31 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 7213809e5fd..c61024c124d 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -50,7 +50,10 @@ from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook from .ptutils import IPythonPTCompleter, IPythonPTLexer from .shortcuts import create_ipython_shortcuts -from .shortcuts.auto_suggest import NavigableAutoSuggestFromHistory +from .shortcuts.auto_suggest import ( + NavigableAutoSuggestFromHistory, + AppendAutoSuggestionInAnyLine, +) PTK3 = ptk_version.startswith('3.') @@ -577,23 +580,39 @@ def get_message(): get_message = get_message() options = { - 'complete_in_thread': False, - 'lexer':IPythonPTLexer(), - 'reserve_space_for_menu':self.space_for_menu, - 'message': get_message, - 'prompt_continuation': ( - lambda width, lineno, is_soft_wrap: - PygmentsTokens(self.prompts.continuation_prompt_tokens(width))), - 'multiline': True, - 'complete_style': self.pt_complete_style, - + "complete_in_thread": False, + "lexer": IPythonPTLexer(), + "reserve_space_for_menu": self.space_for_menu, + "message": get_message, + "prompt_continuation": ( + lambda width, lineno, is_soft_wrap: PygmentsTokens( + self.prompts.continuation_prompt_tokens(width) + ) + ), + "multiline": True, + "complete_style": self.pt_complete_style, + "input_processors": [ # Highlight matching brackets, but only when this setting is # enabled, and only when the DEFAULT_BUFFER has the focus. - 'input_processors': [ConditionalProcessor( - processor=HighlightMatchingBracketProcessor(chars='[](){}'), - filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() & - Condition(lambda: self.highlight_matching_brackets))], - } + ConditionalProcessor( + processor=HighlightMatchingBracketProcessor(chars="[](){}"), + filter=HasFocus(DEFAULT_BUFFER) + & ~IsDone() + & Condition(lambda: self.highlight_matching_brackets), + ), + # Show auto-suggestion in lines other than the last line. + ConditionalProcessor( + processor=AppendAutoSuggestionInAnyLine(), + filter=HasFocus(DEFAULT_BUFFER) + & ~IsDone() + & Condition( + lambda: isinstance( + self.auto_suggest, NavigableAutoSuggestFromHistory + ) + ), + ), + ], + } if not PTK3: options['inputhook'] = self.inputhook diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index ad4dc39d870..fc870f0c01c 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -52,6 +52,18 @@ def has_focus(value: FocusableElement): return Condition(tester) +@Condition +def has_line_below() -> bool: + document = get_app().current_buffer.document + return document.cursor_position_row < len(document.lines) - 1 + + +@Condition +def has_line_above() -> bool: + document = get_app().current_buffer.document + return document.cursor_position_row != 0 + + def create_ipython_shortcuts(shell, for_all_platforms: bool = False) -> KeyBindings: """Set up the prompt_toolkit keyboard shortcuts for IPython. @@ -332,6 +344,12 @@ def not_inside_unclosed_string(): focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode # autosuggestions + @Condition + def navigable_suggestions(): + return isinstance( + shell.auto_suggest, auto_suggest.NavigableAutoSuggestFromHistory + ) + kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))( auto_suggest.accept_in_vi_insert_mode ) @@ -343,19 +361,33 @@ def not_inside_unclosed_string(): kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_token ) - kb.add("escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER), eager=True)( + kb.add("escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.discard ) - kb.add("up", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( - auto_suggest.swap_autosuggestion_up(shell.auto_suggest) + kb.add( + "up", + filter=navigable_suggestions + & ~has_line_above + & has_suggestion + & has_focus(DEFAULT_BUFFER), + )(auto_suggest.swap_autosuggestion_up(shell.auto_suggest)) + kb.add( + "down", + filter=navigable_suggestions + & ~has_line_below + & has_suggestion + & has_focus(DEFAULT_BUFFER), + )(auto_suggest.swap_autosuggestion_down(shell.auto_suggest)) + kb.add("up", filter=navigable_suggestions & has_focus(DEFAULT_BUFFER))( + auto_suggest.up_and_update_hint ) - kb.add("down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( - auto_suggest.swap_autosuggestion_down(shell.auto_suggest) + kb.add("down", filter=navigable_suggestions & has_focus(DEFAULT_BUFFER))( + auto_suggest.down_and_update_hint ) kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_character ) - kb.add("left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( + kb.add("c-left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_and_move_cursor_left ) kb.add("c-down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( diff --git a/IPython/terminal/shortcuts/auto_suggest.py b/IPython/terminal/shortcuts/auto_suggest.py index 733a46d416f..7898a5514d6 100644 --- a/IPython/terminal/shortcuts/auto_suggest.py +++ b/IPython/terminal/shortcuts/auto_suggest.py @@ -10,12 +10,43 @@ from prompt_toolkit.document import Document from prompt_toolkit.history import History from prompt_toolkit.shortcuts import PromptSession +from prompt_toolkit.layout.processors import ( + Processor, + Transformation, + TransformationInput, +) from IPython.utils.tokenutil import generate_tokens def _get_query(document: Document): - return document.text.rsplit("\n", 1)[-1] + return document.lines[document.cursor_position_row] + + +class AppendAutoSuggestionInAnyLine(Processor): + """ + Append the auto suggestion to lines other than the last (appending to the + last line is natively supported by the prompt toolkit). + """ + + def __init__(self, style: str = "class:auto-suggestion") -> None: + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + is_last_line = ti.lineno == ti.document.line_count - 1 + is_active_line = ti.lineno == ti.document.cursor_position_row + + if not is_last_line and is_active_line: + buffer = ti.buffer_control.buffer + + if buffer.suggestion and ti.document.is_cursor_at_the_end_of_line: + suggestion = buffer.suggestion.text + else: + suggestion = "" + + return Transformation(fragments=ti.fragments + [(self.style, suggestion)]) + else: + return Transformation(fragments=ti.fragments) class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory): @@ -208,21 +239,40 @@ def accept_and_move_cursor_left(event: KeyPressEvent): nc.backward_char(event) +def _update_hint(buffer: Buffer): + if buffer.auto_suggest: + suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + + def backspace_and_resume_hint(event: KeyPressEvent): """Resume autosuggestions after deleting last character""" current_buffer = event.current_buffer def resume_hinting(buffer: Buffer): - if buffer.auto_suggest: - suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document) - if suggestion: - buffer.suggestion = suggestion + _update_hint(buffer) current_buffer.on_text_changed.remove_handler(resume_hinting) current_buffer.on_text_changed.add_handler(resume_hinting) nc.backward_delete_char(event) +def up_and_update_hint(event: KeyPressEvent): + """Go up and update hint""" + current_buffer = event.current_buffer + + current_buffer.auto_up(count=event.arg) + _update_hint(current_buffer) + + +def down_and_update_hint(event: KeyPressEvent): + """Go down and update hint""" + current_buffer = event.current_buffer + + current_buffer.auto_down(count=event.arg) + _update_hint(current_buffer) + + def accept_token(event: KeyPressEvent): """Fill partial autosuggestion by token""" b = event.current_buffer diff --git a/IPython/terminal/tests/test_shortcuts.py b/IPython/terminal/tests/test_shortcuts.py index a43a4ba2605..309205d4f54 100644 --- a/IPython/terminal/tests/test_shortcuts.py +++ b/IPython/terminal/tests/test_shortcuts.py @@ -14,6 +14,7 @@ from prompt_toolkit.history import InMemoryHistory from prompt_toolkit.buffer import Buffer +from prompt_toolkit.document import Document from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from unittest.mock import patch, Mock @@ -26,10 +27,7 @@ def make_event(text, cursor, suggestion): event.current_buffer.text = text event.current_buffer.cursor_position = cursor event.current_buffer.suggestion.text = suggestion - event.current_buffer.document = Mock() - event.current_buffer.document.get_end_of_line_position = Mock(return_value=0) - event.current_buffer.document.text = text - event.current_buffer.document.cursor_position = cursor + event.current_buffer.document = Document(text=text, cursor_position=cursor) return event @@ -252,6 +250,42 @@ def get_suggestion(): assert get_suggestion().text == "_a" +async def test_navigable_provider_multiline_entries(): + provider = NavigableAutoSuggestFromHistory() + history = InMemoryHistory(history_strings=["very_a\nvery_b", "very_c"]) + buffer = Buffer(history=history) + + async for _ in history.load(): + pass + + buffer.cursor_position = 5 + buffer.text = "very" + up = swap_autosuggestion_up(provider) + down = swap_autosuggestion_down(provider) + + event = Mock() + event.current_buffer = buffer + + def get_suggestion(): + suggestion = provider.get_suggestion(buffer, buffer.document) + buffer.suggestion = suggestion + return suggestion + + assert get_suggestion().text == "_c" + + up(event) + assert get_suggestion().text == "_b" + + up(event) + assert get_suggestion().text == "_a" + + down(event) + assert get_suggestion().text == "_b" + + down(event) + assert get_suggestion().text == "_c" + + def create_session_mock(): session = Mock() session.default_buffer = Buffer() From 11011b059a2bbd95006ce3a57e8d1cdc1e481eea Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Sun, 22 Jan 2023 20:29:33 +0000 Subject: [PATCH 094/122] Add missing line_below/line_above conditions --- IPython/terminal/shortcuts/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/IPython/terminal/shortcuts/__init__.py b/IPython/terminal/shortcuts/__init__.py index fc870f0c01c..6d68c17824d 100644 --- a/IPython/terminal/shortcuts/__init__.py +++ b/IPython/terminal/shortcuts/__init__.py @@ -52,12 +52,14 @@ def has_focus(value: FocusableElement): return Condition(tester) +@undoc @Condition def has_line_below() -> bool: document = get_app().current_buffer.document return document.cursor_position_row < len(document.lines) - 1 +@undoc @Condition def has_line_above() -> bool: document = get_app().current_buffer.document @@ -378,12 +380,13 @@ def navigable_suggestions(): & has_suggestion & has_focus(DEFAULT_BUFFER), )(auto_suggest.swap_autosuggestion_down(shell.auto_suggest)) - kb.add("up", filter=navigable_suggestions & has_focus(DEFAULT_BUFFER))( - auto_suggest.up_and_update_hint - ) - kb.add("down", filter=navigable_suggestions & has_focus(DEFAULT_BUFFER))( - auto_suggest.down_and_update_hint - ) + kb.add( + "up", filter=has_line_above & navigable_suggestions & has_focus(DEFAULT_BUFFER) + )(auto_suggest.up_and_update_hint) + kb.add( + "down", + filter=has_line_below & navigable_suggestions & has_focus(DEFAULT_BUFFER), + )(auto_suggest.down_and_update_hint) kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))( auto_suggest.accept_character ) From b811ef130cbf93100b9e64a26cce28659b663669 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 23 Jan 2023 09:45:11 +0100 Subject: [PATCH 095/122] DOC: add some what's new with respect to #13888 --- docs/source/_images/autosuggest.gif | Bin 0 -> 137790 bytes docs/source/whatsnew/version8.rst | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 docs/source/_images/autosuggest.gif diff --git a/docs/source/_images/autosuggest.gif b/docs/source/_images/autosuggest.gif new file mode 100644 index 0000000000000000000000000000000000000000..ee105489432284124c225cd13cdf6787e39250b9 GIT binary patch literal 137790 zcmV)5K*_&HNk%w1VIKrN0`~v_0E7YnhX4U<5CV7x16URVVi5y_00#pG2TB_VZ3PYt z4G=IU5ItBCLunC2cM(N?5lD&=N)r)Q3=v%j5q1C)1`QHMg%U`I6EQ&(Hck^eW)nw> z6+LzrWdRut8W|)Y86-3rG-w$^h#5`_95;C#HWeQgARiniA1!1bJB1+!7$glLBp5Oz zN(3aAP9&UQB%)^}en2INPbHdACRi~hnq?-SXD2X!C>u;DIVUJV6DUqEDGVYh4<{); z2Pu7EDTHDwW<@IrA}bXjDQPurtXJk%lFj*(s?3#xjg_6Wn6TEIK`ES7KAqV5o}Hea&heo{DW#>Qs;R51O)0Cbtgu@& zu;==*va+(xilw1985^!&HDwzs*sxL`K9c~`uhhrYbLz_6;pX*k2g!^6qS#q<2e z$H&La%*oBo%+1ZsdPL8qf6vd)(bLn_*4Nj+tk>Dr+T7aP_WRzOYTnVt-r(Tik5k~` z+vS^G<-U;R-PGmz{O9NA=%!}r?C!3 zPC#fxHvj+tA^!_bMO0HmK~P09E-(WD0000X`2+ zoJq5$&6_xL>fFh*r_Y~2g9;r=w5ZXeNRujE%CxD|r%fOuNY$>^LVf7tMxUk{Fh!ZPz2)FU9 z#gHRQo=my2<(`h=Uf#^Pv**vCJ!gI#y0q!js8g$!I(oJ1*RW&Do-ML9?c2C>>)wsp zHt*lSg9{(t);ICv$dfBy9@IGV=g^}|pU%-a_3PNPYlm)qyZ7(l!@K|PF220^^XL(i zPtU%+`}bSd!=F#TzWwHz@9W>szkf{m{sk!DfCMs@Ux5fFs9=HsF6iKc5SG_qgcMe2 zA#)R6sNsej-j-p9AciR7XCIDe;)y68R$_`Qw&j5M-#V2wOop#=^gkg>#R zPn04|9c18Wq+u~yq=XGHGzkL@5NW}rlP>(|Q3)LkV88$gL|}rCNLi7B10WD41rZQH zfPoYhIYGey2!PpF8s5-D$dPysX4@4&OzGqcIdHV33`GP{L=HwgR7Igh6ahn(9<>m_ zq?7`1f|W*HkwTgctZx9f8a~ zmpMXX&Y_&FAOOO;wtTD#QW2g+p5F-zyMGY=Ee320? z5G()>KN?Yn%|uy2LIXV}9EB7|T+nYpNGpA>M@=s+)2gjn(0`feb(NEMoLT8h7mWS|4)}vPae$bOjeHixGwxDv$s1twA%7yER3UdM&3@F`u08 z(@ZbEujBp#%w|Pokc(5-1Vki7ANlBV&mv@Sz6-BwOVsw;sGH?n6+p~AP!~ZoTZ6bX z^l&&sbsN-0$wiU@#L7^30XWSzchUL|5II3GrOXGQbrfEdFoBi;1iMkT%idZ5Wxi|n1NCKuYn0fmeik6pRIH#tqBlMASuX(}7-I7%g@D2N>Ja;Igg#_31yImq6Y}_kujastaJ1tZ z!KlN#rVxl-gkv7&=!P)vbh<0>p&arEg%DE5NUFrmbx|+^8em|97XZOoFo;1AaaS^g zkYQv>0G<~%892)g1Z7Q{!4MvzyycBiYes<87y+;VfwZ7{%p1U^kl@N#O^SR*REQLA zut5+c3VsmiR2lNszr9&?jNm(|JmINJR0Lr+KD@bA=kEljAvVe?i93mg2h=L)#!4G|W;~o4U$0Bg` zgjn!{ANat=H|h}&Le}X{SHOce@)3%}HS(xTSzYS_MUY7fP6}pl7NQ=KJB8eBWnd6q zL1tDnrY-4Freug0Mj*aPnNI>EkN`wuNksAizywP`0;LA9z3R1#mjEb$ECsMVuQ3E^ z4ipIdF2H~lY%@&`2muMw4P_gaP0R9@g&I@oL1oM<1!lrOq--bZ34B&wd zuW$ngZW1A5C97Kvicr0pwXd1$+5ptGymlEuu9Kpx7rh$=ix$kSCWx*qshHLgEWoG* zxq>Emr3rheu@#5VM=PRWHhtug3~s;$KekaMDpX7!uE1Cnx`2ouX>6tPIK^0yO5vt_ zY^elU0vcX0t|%-4l|e9LL8^+7D->a6XK+Fpj8;mAElLJCcgVZ7n`We%?vd8 zA#D0_5zA%dAtrgr1mFSYj3DIYC0Uk19KDXaSd445hq4Rg3$lZ=6g z2|{AKop=mq*a3XGx-}U$q{a?oD=i0tt07v-N`)hUk-Y%q@&-Tvy%r|>8Y@~27WT0H zQY}3_h2>36mT5_o+9?Ar!YC{-nJlP652}DBG8%^d%jXL2W^XzBu!-5Yf`yh)v?fqBi`6pcc$v=IYY{89>-6yX+8*p8yN-+jz$_0P&RzWX&1K@(vdQpISS!Mw_r#P&_0XDRH5)pg%_i{@HcmgN{!IygklYe_>R=!t%z{fBVkp;NG zYM&$!n==p0=X|ZuX3;l&piq6+=MD0(3-6E*vao&Kw}qH-Bff%R17UW!Qhw4#Rh>jH zp@eNYAZG$0cN;N(RZvD4tIuHRLKn4P+d4$J-hj%cEClCra5UQtvn+GuX zqKMKYWxN0PDUS679FQgqfGbp%5G;fPLx3rsHw8GrKOE0pddP?GHflm>OAJGYxac)Y@Oub>Ifv#D&1ZzC#0T%t3}4W6$u|%NCUwzw zbqEn`k#L2sz=m9SjWUsJWjB6fn097pG6bOoV$d?$wh-KA5HA*E5&?%lCx-f_5~h2Uzm+hz)^hK0+==c8>y) zkC^|1FaYR0omMdN7zOl5F79X$KAc*Gjb5lyVAyQjXqM*pjR%p2X8;CZ@B?o7137?o@FzGY^HnY|cn`sj z6-8tR0bT)+0DbjtD^QbhSc~d25YLoYvW0kzc##|EP9cy%A22z{WLz0kW`Y$zm?;8f zwoU?cTqLjoPxCK{)mpM;f|vPQnQ0|Va6n}u5S~X6p_l=WqFoHP%;YC12IVpnKU7q5L2O&mul>l&(l*s>J zET?3ZSJz*`AP7A$2LeW5WH5aEfM9jNCRN}Eb+7~)Lk3%Um7x%p{wbD@;VaA1EYA`x z)3O9+V19{0hUXUrY$pZ3!!i=;H^b9acbQ776cK#Modr=ixwK36ND%qR5b#8S010`J zxdaCnLIV+s7`QYp3IV}&Fa88VwmEX`15L3h5HUKVhIm>kNK+f|Z@w~mLQta3WKaHc zS_njDIwV;01cMHtX=G%W`1o#;rctznD;_#m6vd%+XI>2BhX}C+)j*yDVU@x{27JI$ z_z(^-B~t_81~$bHfbIU3gNK@p|U9pvJ%0vNa$oO3uY7HZO#AYvJC-WFiR36 zv$Gzy1eoJ%Mq{%U0kUSMb|6V^^x}s_;k=r#BsJ5ZyF0v|+q*82 zjzmkm$}76YTf58qyt3Q8&^x`<3lh>>z1W+*7ID4W+r8e)5ZwP;5XgHF*CM{&cTp)0&{M!~Yf4w1wN(FCs$Q!`~jT*<`AOT}o+6IT2g5c6~tV=)SL5ngN% zQ_u-9l?V>Am0g)qY23#`vBsVeHe*vZXS0;V@>9a3Yxn&KD|6M)Pa zo6|X;6FT`YI+zg23Ngph^a@POhEE(Z?>EV)JQJ0i5v}{W4xuY3MVD6)2?I7PZm6i0JZM|YG*d(=mN6i9;< z1%-4-iKJs}P!4GNDTj11seH~gvC0==ySVERZ`=$7!E{b{G;dH1{9q2#pbl603AhqA zH)RggKn>w=4s;*|hlHlhhf*ZrQ6ME!BxOfGD?|r&2k|TQ zkPU9#+JvgoLvh9{+l*6ZaH3obDk{pdKoAt854ar#f*r>NVT|!WByHr|XH2FOF?2tfDh6z3?Lrj#Bd7p{SM|#zLii9%|Hfh@D4le$rV9_MmQ`< z7~oAf;L=BZ%bayoXoX%$5JF%!?d@cJFb~j3;Yhv_aLp2RTzy2C4{~JN+V@FRP7w5s z5Vd9xi69Bq-4IFb)J_f6A~uYmQ;f%$jITB_wEc|PEsfL|GbPPy9W{;coaA6DiZl~TFcRk_buDcXAc;{w41)m;k=(c8Oy=dw-_3Cz-YNxzR`zqYRH_zMxk zrw`%a+XP`Y_`u5ua|LMd(*oh>^dbl9U=Hsv4vg+(Fm4b^xiLs+o@IcZ>dBt9{GMSf zpY&;;_<7X(nMf9N2=y=yba0$zN{q7}?+;PyN+7`RYw!I_!2hf75rK3t1<4kK2ljl= z-=GeLz3e6~+y=4OYjhIdjqL)Vrk~WNI0dI-Jg0Sfr+K=ki0o#53aA2c1IVDzm?#PF zaOCoy@(Xdn5bVlZHNh_*!GQk*^AJGR6jWde*O1Hk&)E)@|D57}R0Yc2lPuMtb1&CsMO=u@as zrB0<<)#_EOS+#EE+SThwDj{<^j$X_0VGN;KV8 zY8G7iLX|1FL^1zX*s^IyvCUPAoO1UpEu2z`Upzs#Q7>oS-1&3p(WO%jJ!|^)imkCb zRIQw$_NiHV?fXPlJbAKKUNI%Zun8`Gw*+seU*GQ$=#u-a^Fi49@R#_$-b>=b08A7hmWWffbD6E)?z_Vx$R>EoLnqca9 z?1@!&`No}jy5WV8$Lg9Rm2lp{ryF8iIglYNR?(%M4=2$joNG?m?FtrMgi%JM0@=kI ze5Bdsopb6~f{k)ci=sP&R6OXx2q&x%A}RWKhaWoYt zG_yMzZN&dkM;;HFLd$SG>_(WMq9F68hNR%5OxO1F^ixnn6?N3A`lC+@C18MIh9M-v z0u_FmIf#l^@VTX~H)1kp8g<^0=aa|QqSG8|s0rsBIy5Z@(}Kj4EXBJl;=>z$^!aAa zqK-;~T716wrXHexx?);>@X;n5aN}|aHhiv$W=?b7{ndzZskIhcqYzX|rfcbpi_CcF zXb{bU)WlU+UVR1jAt~NShM98OO=eNQ(2yBXww^i#Gb`{7l^rRU2S%fi8%yID{iQ z41)jZ9!!=XP@iD>vVyTN3w|daK*(tCiZ*y7@YyJgc*C2UN_@!REs){54l>-pg~TYf zrDu^Pu7Cp>e1K{12Ew}zoR5-LV`vHS>`8Q0!Fpt^`|iB=W{?b+xK8#7RvI6Z z(}7J-Gg^bT?plbi`3d`>_JTH>jNLYmv2(t4$6If{X^+Bovv)HTZ8*sr1H#`c; z`H{z!wg8h$C<7g;V#HgK_WpbD!xx_{rMIi12pMv4N+Ou{!Q||z!J#9;y&NY38S+qS zP40r2(5q(zO^rd&G82L%!H;ox#3}77kKcYm1_h0S4D{;=ZG=EDhD2@=BC?#N%8~z# zG8{00b%_E4uTY74Y=aI$Xj^xVmpXNct$mE2o+euYLMp&J|rOn zC9*?=yfArjabQ6ph(I#@rX0T#f?Rr25CUP1hYXQgJ+d)AFosc#V zD_nbMrH@*yaXqp)$gAL}Kh`0Haq+-gUf`HUh^UEHwyGm43fDKd{bUKP86+?cg29Do z@Pr*C(>4a#y@I@|aD%Kxk6c)efWgplQ4r)D3wcK!jz?$)xnvG!V;kp8=ZKnd&I5h2 zBcdFPIkTxvI&l*sK|=^Xk>XE>7DS!R;9mw+nGhxFl8L-@Cp=H&or1uG9)fs?5##{H zW~Nl7D`n|to_Vxk{)&EI#AcUNm_e$5LW;5hOkyHnr{uZCiL#TUQFt42 z_$ilqCPb2zpyWID*^7M4^P#aiszE-993EoOpf<(nO?6t)NM7_HU6m+Av8vU!uCtb_ zGzh@@vc!|Lld0Z7j{o?9qJ;>dZq{U2ALbFvmL^uQi)E};UTU-{FsA<)xe?kMFLyh$ z9i*pLt7al3(MF0KZ&tOc#jFhJ$aXHFsdW$vK1%CV)ip$tfe?iyj8I8VmWYkOXeDf! zxU5QHP_(R^;}tNAR@thRxUMzrE1Ap6(;iW|21x^OeVbQ~s+O;U_$6ZPsmd)bBth?x zRb$P2Ui7BdjAHT1`O*hSpQ2BF3UTjz6H;IJLgZ48a0jv^Dv5f4gQyDGEN_@lIbOU1 zYh_T(7gBKzBPpc1#WWy4#8I`72C;$JeJ66Rz!(*VHMUW3FoX}JK`VNYK?l4rXN7pc z*=h77!_1+K2Q%SH88E;DUPy@}+|cP(xWyTMv4t@VL|Ycb$3FkVW|9p;<4-_!m&{Tv zJTIuQ9wFqw&uxeu>8M^Xhgr-keXLcO=CmICObJbML7EYA<~FOD&2FBEdh`(vI3A>K z_@Iy!0*8jO$*@yzmV-LX@s0#4<_dzsnsU~5xhmwrBf1kr59Z)Gv+1?Nf@qjGn3A?H zY=UZXS@fJ5-K<^^_8^rc1EnpE=~ffY8j0Y+Z>T1O=*`+f7<}{b8SQ;5Wzx;3X!OH|?%N42W!s!y|#3m+aYibU~1 zt$BRI<8kS9xwO%*7)HV3!^-%^4L+=zC>$v(CnvJ$Dp8r!{F)r&meeCoCze|~*|_4? z&*!w!Vm6iFZm1bQJoJrnXuLs)W1Dy-@ec38TkdnGyME+d6)gZrKnlrwfELpCy(a|l zARz)S+ISlpVWS)#lEx|4$MCw4^5fB6%qdg?D9Am~MJ5Gnj9LOCkZl;;1irFyu7NUliRK^)9^H}L=^iC?@q z8V|_VV_vG)Mo^VP4=9OR|MmxoK9d=m5Gny*_(e(^>8%_EGH4+ay9iRdgz+Xa6vx8I zQ-~4ket-PuA3Ap@8oLYPm)fWzijW$70HB5Nvjaql_K*SqG>8((vbH%0E0PN-D1!!B zG+?SI1pJ5xbO_u@BPl?@_P`yV5y84>!2jc+B@n=$0V&}r2nckdZ(@iIoQM!qK=dnv z1IaSm=t11u!1lNzEFv6=nva8Mt!5}b{aZpNY{JXnKlHi+gL65{m;ykUlELX9C#(!9 zxH)|&hATh_D=>v{iNP&ALo`f7sDMJoq5}U@xP}PHs{C?`ZJCK^2nVyXhCx$9uSkle zXbOWELw=x#H{nA=JVZo<2sT`-CWr+lk%myHLCbK1REQ9nkOt!OzeKExagm4N@To}P zh6-`PO)N!IY{Eq}FCn}PBiIeOXvI_X3Mps<_-cbPFvD2PMO~!3RNO^hY>rj@MPVF9 zFzZEPJVsWYg{JhL;0-qXa75Q_hg=@X@3ia88OkNimN7)ZIJ3%e+w{7A@yAW6vk zh==3~b?OFYfCfBlwy21##(+RSY&($LNuJ!lGcv`M#151sFmK9% zksirCAQ6(`OG|@Dk|k-9Cy5f@)06E(5G~;nFBz)SIg}CGEWb>?h}af+pal+@6p)F6 z8?cdg=)6%Z2n^8>4)G8WvC99XL<$9=Dci(N-Bdp4847KZzJj=v9Q&C~G0C~CPV3Z) zy2Olsv$u$Nl~@@qLTVU<>6LjAn_(%IV@Z}xDW-E;Olip&e9;zefthfDx~@Z)l#?yP za~F7#7hvHykJ%VM&#IdeTwY73^;=t zd7PSf%&1^V$tbbPBO0oo8^WJib%=X>vTBeF9?CMII*t7qz!4b~naMJ8 z^`>aT8VeahElP<)@S;+^SA3O-Q`Jt2SfkT&qxFa*%Q~evqE&a?qdq#(TIk4yC?!JL z%;-ujMyeHBqSOC%k-bS;R(U6`(RxZ*s+@xL)}?xtd8k-4$OKx* z1!71AciTH{Eih^+}r#y-%1o0=O3OjgWD;vAmWW^_qy(-D*C$7S(PQ9+w zq`zw|psdweg=z@BN~(*XmO-gXgAF(mHJdB=0>&7qz||(gEr=&Dyk-c)gAgf_atM<` zsi@uDsGVBrh^bGQDO4k?vO256r7xf&1EC5NxYVq%eJi9g+mLfAx;4UMovNyWTevFQ zxy>qi3R(ZK8rjnwU1=yQv)ZW#U86REh{JWF$cU6DFX~n zpxfD&;O#kML56@hY$Ljo}^jTC+?vg zBSrr#rmm$jAptrN6TU0RTVviWQEZZoT!mu}j@$xVh%I}qE;CaZ9^@RQVH#XBG?TML z*0(uR5ezsR@Hf}?XPWvD+mNw0}gozg$6NU zCI++&Av@H+Jqf+#o2#W z>A32KVd&X}lZR7aAu5Pzn3Tz}Ne%+x2bj!=hnZZ1U^^&vJGk>ybe?K-ROi3byQ{`a z!NY34ECyCH*xa>oP zytS4nN5BSfk%?<)-V)JlbeRdy2AzW&lyj(5`4b0kL2q%B;-7z4JD201QTkAP{^g!rMT?#LjL1KF8f2A1SP+Sf&gs z#6r7)ICDhdFQnW+B*S(6ZwasJ^c6EVgu{1`!_26|Zn(odY)EJHLymm#K`g`xKk;Y` z@b78FM~p<^<_b!*MCZ%Ifb7IiEQl%y#oa`49(P6+7a#kk3|Qn17?f&goW)wa#U5Ys zW&Cj_Kd&Hvaw*qEC!caFzj7=mp9;@%F7I+LUydsOaxousGIxkDFLVDiPjfLpb2V>s zHI=^!~mti~4b3X6$d);%YxXZxvb3!llQv`IW*iLvhbVhG));D~c=XQ?g zX>WFH&-UvG^TF$CtwxA$-|BDIc5*Lw(s*@qPj_{9re|MwcYpWALU(wdcY0?Id9QbT z&v(tZcYW`7f7f#M`gegJ_;_n~fSxjRZuV-}+|q`L6(a zp(lEUaDt;x0123Y1Q37~pn2waf(@X0u&_yTa14cvjHl*$yeA{C_X@Bd`iL-s7BB#i zSI$;s`?i03!>5XKC~yM7M|%IHUx37q0xduR5wHLb$bcR| zi7H3|3&4QEu!0R}fF3vsDD)to1c+c{$V4e<@F2p33KueL=3WB`{#j7&6fbg#bEt1}Gq;pbwrp z0empx#DD>zI~N!+^03JU2oE?|*wDZPnI%+2JOEK}N(u`gmK1DafdkqN8#JtlfcmGH4z1>RH{)k@Id%vLk6b^@lLIJH7i83XgU9=Wh?kfC4R-qG?bF1*S=2(uN>mX z?;a+CFGrIuZTd9o)T&prZteOt?ATFSx|D4(rVSV_no4Xjv_OZXJ%1uG0HCJ?3daYm z@G@dV0-XXzNTB`!;?}NGmQ1+;Ap_Ycf3IFSask8kf>)#%WRl_n2qSM9@Br)J#FZOj zmq4W-MamZ@sR)D11E@g-d$VvRb&%ZVrdYVPey6Mlu=Z9 z<&zS{S5UROq0<0%_4uO@ zULZx(5JNc8!f^-~#H5oGfmP-~Risd6cwkv@L791KfkOx(pb!FJLWoJGnNOhEUteFn z_d%w68uY4}2FXfmM7#F-Yp?}vQ4KwWJS2r2?`$KQK}!gcC#C7CyDq!!y8AA?2U!}g zLPiu9fB%241J^P%|yglEWZ$Uox z#KWda?G$i9tP;%7TNY^0E7iS58%22s-*-@(4`9%P6jcy$0mQ@J0IRVbtNC$QV~reG z$#6|wwbhAn^>x@|w=BdK$k?;Uw|&f$jiNsremLTZE57)NLG#=%049)SbOA_@>ghsG zyE?0x>}7h;UmBob6%S-dVIF!0LB*GtQs98Y31p<5rhm#>ka9s_0;(*mo=}8qlzR(qsyweQh%{%`*^wCRSk>e@NSmWo?(rBYX*K?nc_t`g;V*mgkz#ip4QD7Va zJsf0{z)e1lyp+cywAFq21;78q=>u{N7!Sl0g{EX>b{;E;6-3|w%+c;i`Jt2jBGNws zsSbbzVFD`fQHzIQ0Tud?MWGIrsOgcggeE*83L#a!li)@-?n6imUGTz##BhcNso_LW zrIQ3qAUeb8k#GP|fF2rvX-yj#MG|O%47^YZQ+Qz&ZU7&rFaa(n*v@#aHar1>?pPae z01Nu!p8Xjlfm^#m7y6<^CKkkrHe|>WvRFaBsOJ#%7zZ63q>yd>?Pgc#!#h5M2|cE; zkcK=YA`@wn6`CX_GpR{J97z-S(NHFnjHD$YDG@Jp;9ooeQWRPs9G(auS)oMY@17ta zTs>d|V)+jdzSBgCc?th4u3U_O%9b%x=FW=;dq7nn;FyD4AqB1kCZ43AN(ivhl_GnH zEzg3MT<-1$$gohLB=HWyZKgB)*as#QDb8_{vz+FX*NaA@1t7d&2MbweJKxDpcPa#) z^qhzhbdUh!3?KmoC_#Fn;DqD=008|gK@Y;Cxr%531#me_2+Z`Btx2nGPN@P4NC3>K zgg`-Dgp(H=5CXVxfQ|{70tvX{05%q+p%3LiL?wbzjaq=C3R2P(A|sAE-0UDP%Zx&Z zpbK*}qj2XuDpHfGRKqb+XGMaDgN(q-(mZt`^Gis7B8VDv^$8!%7ziPo@CvWCfgu51 zDq7R3*0n~escipE%>ivA1w42|AH}!=Labm4evpF@SA8pB11s3Uvc|21eMwwBas?8E z0H$bZY-3@1HHciH5arMZFfyfAe$c}bQ#~wbLo3?RcEqrxB}rmCV#|{u>~@57$QAe? z4@<-<65R0YYfr1&-SU>SrtPiHq|k=!wV@1u1uk-vt6X#bcDXyNs&k_&-RT;#xzxQb zcC)M9?Ow#X-2EY1| zNZ>jK39FB`ecW^OQ&VG}+S*%-@9 z=+}+}!@r^>g*o<;PQ(f_1gG2?U%T7$nFB580jF8UDr)lpgOn5_{-;#r13(Xi0OtY} z1tF8J^q@1XY5W%Ykcf`LlmXTkB<#~)E9eW8I2D0tIfY4eUiGG9Eo=35+K``)f{hCx z1*0hi(&w^QsX0~YrjUA+I6m&IldbIJ*1C|lW>Ua})TALu;n6GDPI$af0v!>}P6g36 zxAp(E>~WKuyy<+&n!&d{@kJrs2yu6b+Pod|?dZ`TAj_j8?FCFnK~&|#(!LMmZvech z+y+0mZ_k|)9Kuiuxsf3zAO6PwINXJ79tsZ!z$p_LLD~a|Km$*jyp1|Pk&XSuqvHIlphM{a4^sW(Rx2&eGk-eN z(OUC|*i)YItfz#res!QrN5uaG6d9Z_0S=dM00oG^wVj~P6q9(^P8h+H%U)t+N4@TL zH{sMz*yiv+*Q^EUd*2Ikpupd~@P{;yk*weoD0-n9?bFX{);(qtO|Gjy8 z5B%X1AG^XYKJt@~+~X_1`Obg5=NlE_tv|o|*8dsvNiyS}U%&g_cUSgDqGY;%zx?K3 zQTRs!@AI?2{Vhtr_T4Z4`Kt*2^uIs;>1F@?^S}R;+Q0t=pa8~*@U34#7+|}tTlXm7 z05+fldPMIrTqQ|d#93U!QQ!k+paxP;^fl7YRT9tb+$A+!2yUPWo*+XAU)5>dJt4#j zCd3P(pbW|&0M4Kd-XH{aAP(*z51L2~_8<@j;X?eN5Eh{j5}^?$;SM695;p%K4eFp1 zMxhjP1QSl76$)S#vW6FYm^oE}7JvX=LBtnu0v*uM9C(6JU7;F+9~26W#tDE{HAIMU zSU+JHVz3CvAPn4mL~1R>6T|{$DM}OILLjc8A+jNDEKd65P_LapZk>n}5CKey0NIdV zMD!t6y#ik?!huxQWo6bOhGM#8VQMU5L}*z8I9UQjnWH$1M_}SXOhG5`RU+hxCt{N* z=AsIINvR--yvV*Mt(3AzN6ciD`11L}i-~b6o%@suf6J*q_Jb(t=RDtZF z3p!LlCe$Tbg-ANU*B}HNltZ^v1|O6IJk&u#{GmPwL^-Y_PM!uSR>bz*gpLRWC3mRhklHO#{h_cP|#Q24_7D%cPRfRSym7jiP3x1iC9@k z8ot|xbOn;eKS&PkmZXHu$3c$kr=^vR#VmK_laLKq67tmbTibOz5tZ+`PgaxkPjzL(9uK-GGhQkPXHd=yR$~?O;vYIEXQJDc#f!k+vz18Uf$< z4d7&`;BZrv{wYdKX-PniD_Dg(aTtUkn}osXfa5Bt~<&N1qP z)DJ`8k7e%91pN=|?9u>9SsW#387UA0^@jrit3||Bmn0-cX3z#XL@uLKLJ#D43vc(6iEgdeo_QLLX<=)&}0$hgv``N!IVc0EJ&%1NRd?4K1)hHfI|kT zNa}4wWMoBd07rTxPU#d+EyPc`B|-=lQJMcI*oLm4ngrn~L{xp&Xq2wtjp~-5>S(ZA z6m->BrQ=;LY!6T?k>sJ#(ja~YRpw#-Bn)gRbP21?~1P2niud|#9~3#V>y;& zO@t?6mS*i$XQ6KLZf`=2t~oVtMXVNUL4a#fuWU8r0ooRBaWDI(;PTp?a1j?VwlDp5 z@B22M>eg@jdawTW?}G8K{{}FD0k8lUaDNf70VlA1A+Q2B@Ov?^14l4=L9hf@aClL$ z1!u5!VX#5GcrEI(@O^?~hUhc#V z2iejck&q!T3nALniM!%PWalT^bsSsS)}+qNn3-@$UYm9j0v*dSwsta`D!@iMLYat%N6yy?gS z*c-m>vA*%!zyX{xCk4T|@hwks1Xi57rt8F!tIwHhFatowZCuBxTF8xD$(2vbmFTCO zoS3;>w?#8MJH#~8u+PEs46pwz%F;852<1=?opB6B(IK7E@dS_j^HM-_JHxRGs%_aK z^x0BhLNoL|cb##3osx(h*%{Q?rQO=4u-n0%Gq)W<7xWuj@`~);--Yar0N!g2UdVzp zJBu_OKQT+kbPVURIAxwp-}Fhibc)zCPWSYI(X>wo^>XR7P!~1!0ku&l^=T2cQa5$H zFtt-hHM>ByR9E$+P_3)~faZb7X(d zFH?47Uv>^Iwg+l+HlzP*xq5bIS2Jdhb|nwC&B^myvn=Wb0cVkaCLu?^w zjbRzy&>4a*bt9j3Cj=6_q3bHd9Oh#J*x^dcMjjpvNR|do0)rqDqI`e1^@TS=WTQ4V z;(E(QBu-)_o`zU$VkcUb_1gFI;rBu4cSE3}Dzajuc%w?lA}!jYWnnFXC!c~F1cO@@ zFqRTo7{UBT_>aD>GBV>rJYzKWcZS2>hQH?zMZ`9;_=!k3N{AylGQ=S213J=pi>n@5 z)3!s%?L3y6J&OMwQtZ7|TpZ!L=G%DV4&6ZGE(sni2`<4Mf@>g12yP*G<22B?1b26L zm*CJqf+st&@2#r;E6=mHlNvims2{DELZ(E* zj0q~#H8_+;)t0G;BV>19X%@=WLwB966{6x}(U{>4H{=phKu2b*1tqdd=J4;w%EP-V zsQIc7-ap599bTfVv8!raz1E~Z(0pPj6KZ2(Ef*|dkz+oE@2Hf@{X>DOBZHAW zvl{!ON=ZmF@p~=oF3K;dI&Bqup*h;$E1uT!i!~%iFJ-CUfoX+yYs`c@0WcpLq>{VlTCbL zFC$ns1-w62^-6v#kz!j$yy(z3I+ZmH_oWG1Qqf!9oJu!+9%p+W*S+8SvVCEpi@7H= zlvg(AL3I-^&a2`%tl>6_=wR2FBVS%kX7ss?2=d?~yX;2bW^$EKbKV&fUXxE9EN`Y0s1M07|1~nJ=8TKfm7sB=0(>rR-kZ zOKQ~qI{Nueq3*LLfPo8z=dkD5A?iILnry|n(ctt8d`x>HEFU6ZYzgprz_I$epqube z+e_h|mv-Wz1Q?PX$E=+=#Kf~)ooKRTL1@?{jg0-$;k2klDu$q<@khRkk+ldu4TZC) z3k!;ezr*-Fs14lE}D(2qgA4caop56SDdvPA#PJG>Z!ZX99<0 zve^raMRnfMWV6U%IhWbKBL)ZS-*)bJ5hCYx%B{vyg#0L+UNUIXBsVV(c469NJ0I2R)#*z`e6qpiz z6upA4qCW#U%aeb}qKD!V(mK`x7^BD-iOBew=iXz?k~NEx#b-+?jwNyEH-7=3^rPWU z3su_joQ}KT&QTkDg!p+Lt=X4>!fL+;V_>mju1luGldW(iuG?^F&v3j9xZ}mXLeljD zFlb=CsQ3~@edy%Iu)ZK10To`JKyM4Ia^}LBerzt9gMR!)erYH=9>6>h9fv)ym^}*? zhzd|t$ql6))ZzgwtjRJ zwAJ)UVnE%>dsWery7t(wPOz3}98ax`!u>Kn9cQ@f_m3*sWt8Rcy`kvIykq2JOTMt>3(K7JayR$*~%k znkzgWL9q+hHBJAPa(#*TmUcO=^F70<8~h`SI@It-&W8u^UTzp|#*>`L7gYO&$)*|m z0kO_h2c<BjCintdXu;y#4_nwbyv3iB635Q;pFBq~1^XI==g5_}Uan7xJO*W~>J z&BaU=Ce7>wqxb@JPJ+7koP|?A`!dK*=R?qv?b9mHS&xluuRNufWPFcJuYEu`n(A}F zC>uQw0Hu;C!RW+g1OP6{u_H_(6n@=M z`9J4*xRD03O#C?EQb`tMFns_AjfMvup2eBtXyvgv?iM5O^P5XtwOH!hQxx-u4Z~Z)CrE_*u zWKccNom~bbP4)|2Llls;r)}*b{Zv#m5LLD}Rt-q}gI8aYEhf-6b)NZ!=;|!vfxw$z zq_|U~()@|r?<|%@%M=V(wJ?Lp-YC28tJoiCrmd?K3oY)esrqPVeuI}tk(X0`5Ys83 zB`;gxe2dgdU)3pj^0rJp`9P=4x9OuUd4=I(d0u|IZcPArWohSIgR$3o4eLb}Z*&h| zmu2WRcavAkdpH>Hzt&IvyjbnCSRr(FOV@|(UK2#ly>jadawVXsdmPxbKsz7b=Afug zu%WU-8&DiGrD*7;=BlPjH=GX8vxQ?E*(r$&&!$i`l~t|U>#n_CW<6?5F)3sa$0c56 z{MB5uc;uYiZrIaFlG@wE!UZ)_UZ8s(S2RR#7^S+W0Au2k431 zpFRkn>=ZwD@*XoX{ZzNqxvzUtGt*&u-L0>86n5gjugv~?iL&Q+(?Q6^ni;a&r4vbh z8u$k3VTMWv>T}_92?l>LZ{SAs5vSBj(o0z2vbFY8F4YDKbXpMpXc?fR;PaP}up}*M z8D!Jr^VaUPq-1LudYV$_Y$ahuUD7=KY>Cg#t<#Fmta;=mg{x(Vgf-Ji)2M$o=YQr#wGp zr4Q&YAioc$LD;15vr7|2!9sjLr!2^Uatc?Z?eC4Lmflae69=4zP$48SU&8vZ0Ya%r zL;lz7`jRrB!P;Wbgq6l_{mGkRtty=~5mtyvxxIJC#utjO@Rmy4d>@lL3mqf#JJVmAd zLWRj!dSt+?M#Rs2igRs>1`=5~QRSoLVS4&|rpUH2xWJg6%d45V&^+8IQ*2upaHTN` zz@*>nTYJ<^n11;}?e%R3w=~-=BGuL@gwz__&O4~#>^hA;tDRoxDUhoBdAI-#^3SIi zPa_>_s6ui^8mrV`N_xYgWQoApRnAoN8JbF4Pf315=R1Td4F8`UHjJs&8 z=@R19_g^S^LdIa6hhtkSwZze5(j~LDn3?)vCg%9K|0C_0J>B!yRAxj>a4FoOZt;<8 zr{f{4=T{B--Jiui-VLU~gUR{6Wjfd$I-y(ZJi7F={-;P@d+d zIl^G7!ud)>aDDn3wdzSPj1Y}j|BnU{Zk;V~mWY)2&zuS|HQzg;Sz;zWD|$^5u33A~ zEQ!FZBfBQ4c%6~REU64}?~O87^MSu$;A8HcQoT##5_F z-J!|TI!irJ$K9(*GcjwXI7>56Y&5C~nM#1HY0_*E({^dn{?Md8oTaTIraRZ9x=x_W zo~1)-g3+|-V-x6cwLpX<3?-P9N$zfd}6j$Xpo;Few_h zi1`(X%`M3ONCryVV8@tqhWDaN1ZF5sFukm|9|ut62a%c%5$g0&um)j=^#dg&QMU8( zqUPA+NjQ?UI8qZiGUqs=UA4yP35pXrE9N+BNVpobxT=2XVoGzikZ=!ZahKI|&2Mu& zPN~Pben(ic+9%qN29oA5fWs|7Iw&lLQ`ZEN7>HVH&=xV-)>9-JDKC~bFK!Yq;XE%X zDIcXaAKe!5Pk_dFAhE0zIp;k8Q&Ir|ZGmS=0zBGetPK2Y4FU>Dg31m2FSR#@^YMQ6 zK84H+nUVfus&-xo%s>zoPW3ghsBjZ!b(5G?3rNC_qa%qDH7^oBFG2_qO=v(3ZJ_^2 zPnjqsR+1!EF)vm_D&C+i-sA>yb>-X!P$Z&IGK zd$_W`&Z{2yVNP%Ne z9I;tL;79O-3y1KL5ugWZF6&H1lboJeu^wTXQ&q#RY zbW{s$u?Yaw>w;Xc04qHDI5toi%W z>_-szAy9G%2NHc*rqhdex<%b+VJNU*c&!7b-y-0bQjg!FtS8rikg2Jf8xRPYQF