diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..70971c53b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,8 @@ +# When making commits that are strictly formatting/style changes, add the +# commit hash here, so git blame can ignore the change. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Example entries: +# # initial black-format +# # rename something internal diff --git a/.gitignore b/.gitignore index f3141570400..1fc0e22a320 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ docs/source/api/generated docs/source/config/options docs/source/config/shortcuts/*.csv docs/source/interactive/magics-generated.txt -docs/source/config/shortcuts/*.csv docs/gh-pages jupyter_notebook/notebook/static/mathjax jupyter_notebook/static/style/*.map @@ -27,3 +26,5 @@ __pycache__ .vscode .pytest_cache .python-version +venv*/ +.idea/ diff --git a/.mailmap b/.mailmap index 4123a50f6d5..8d4757e6865 100644 --- a/.mailmap +++ b/.mailmap @@ -1,4 +1,5 @@ A. J. Holyoake ajholyoake +Alok Singh Alok Singh <8325708+alok@users.noreply.github.com> Aaron Culich Aaron Culich Aron Ahmadia ahmadia Benjamin Ragan-Kelley @@ -82,6 +83,10 @@ Julia Evans Julia Evans Kester Tong KesterTong Kyle Kelley Kyle Kelley Kyle Kelley rgbkrk +kd2718 +Kory Donati kory donati +Kory Donati Kory Donati +Kory Donati koryd Laurent Dufréchou Laurent Dufréchou Laurent Dufréchou laurent dufrechou <> @@ -89,6 +94,7 @@ Laurent Dufréchou laurent.dufrechou <> Laurent Dufréchou Laurent Dufrechou <> Laurent Dufréchou laurent.dufrechou@gmail.com <> Laurent Dufréchou ldufrechou +Luciana da Costa Marques luciana Lorena Pantano Lorena Luis Pedro Coelho Luis Pedro Coelho Marc Molla marcmolla @@ -97,6 +103,7 @@ Matthias Bussonnier Matthias BUSSONNIER Bussonnier Matthias Matthias Bussonnier Matthias BUSSONNIER Matthias Bussonnier Matthias Bussonnier +Matthias Bussonnier Matthias Bussonnier Michael Droettboom Michael Droettboom Nicholas Bollweg Nicholas Bollweg (Nick) Nicolas Rougier diff --git a/.meeseeksdev.yml b/.meeseeksdev.yml new file mode 100644 index 00000000000..b52022dde07 --- /dev/null +++ b/.meeseeksdev.yml @@ -0,0 +1,22 @@ +users: + LucianaMarques: + can: + - tag +special: + everyone: + can: + - say + - tag + - untag + - close + config: + tag: + only: + - good first issue + - async/await + - backported + - help wanted + - documentation + - notebook + - tab-completion + - windows diff --git a/.travis.yml b/.travis.yml index 09e16146476..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,59 +1,118 @@ # http://travis-ci.org/#!/ipython/ipython language: python +os: linux + +addons: + apt: + packages: + - graphviz + python: - 3.6 - - 3.5 + sudo: false + env: global: - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH + group: edge + before_install: - - 'if [[ $GROUP != js* ]]; then COVERAGE=""; fi' + - | + # install Python on macOS + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + env | sort + if ! which python$TRAVIS_PYTHON_VERSION; then + HOMEBREW_NO_AUTO_UPDATE=1 brew tap minrk/homebrew-python-frameworks + HOMEBREW_NO_AUTO_UPDATE=1 brew cask install python-framework-${TRAVIS_PYTHON_VERSION/./} + fi + python3 -m pip install virtualenv + python3 -m virtualenv -p $(which python$TRAVIS_PYTHON_VERSION) ~/travis-env + source ~/travis-env/bin/activate + fi + - python --version + install: - - pip install pip --upgrade - - pip install setuptools --upgrade - - pip install -e file://$PWD#egg=ipython[test] --upgrade - - pip install trio curio - - pip install codecov check-manifest --upgrade - - sudo apt-get install graphviz + - pip install pip --upgrade + - pip install setuptools --upgrade + - pip install -e file://$PWD#egg=ipython[test] --upgrade + - pip install trio curio --upgrade --upgrade-strategy eager + - pip install pytest 'matplotlib !=3.2.0' mypy + - pip install codecov check-manifest --upgrade + script: - - check-manifest - - cd /tmp && iptest --coverage xml && cd - - # On the latest Python only, make sure that the docs build. - - | - if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]]; then - pip install -r docs/requirements.txt - python tools/fixup_whats_new_pr.py - make -C docs/ html SPHINXOPTS="-W" - fi + - check-manifest + - | + if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then + # on nightly fake parso known the grammar + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt + fi + - cd /tmp && iptest --coverage xml && cd - + - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils + # On the latest Python (on Linux) only, make sure that the docs build. + - | + if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + pip install -r docs/requirements.txt + python tools/fixup_whats_new_pr.py + make -C docs/ html SPHINXOPTS="-W" + fi + after_success: - - cp /tmp/ipy_coverage.xml ./ - - cp /tmp/.coverage ./ - - codecov + - cp /tmp/ipy_coverage.xml ./ + - cp /tmp/.coverage ./ + - codecov matrix: - include: - - { python: "3.7", dist: xenial, sudo: true } - - { python: "3.7-dev", dist: xenial, sudo: true } - - { python: "nightly", dist: xenial, sudo: true } - allow_failures: - - python: nightly + include: + - arch: amd64 + python: "3.7" + dist: xenial + sudo: true + - arch: amd64 + python: "3.8-dev" + dist: xenial + sudo: true + - arch: amd64 + python: "3.7-dev" + dist: xenial + sudo: true + - arch: amd64 + python: "nightly" + dist: xenial + sudo: true + - arch: arm64 + python: "nightly" + dist: bionic + env: ARM64=True + sudo: true + - os: osx + language: generic + python: 3.6 + env: TRAVIS_PYTHON_VERSION=3.6 + - os: osx + language: generic + python: 3.7 + env: TRAVIS_PYTHON_VERSION=3.7 + allow_failures: + - python: nightly before_deploy: - - rm -rf dist/ - - python setup.py sdist - - python setup.py bdist_wheel + - rm -rf dist/ + - python setup.py sdist + - python setup.py bdist_wheel deploy: - provider: releases - api_key: - secure: Y/Ae9tYs5aoBU8bDjN2YrwGG6tCbezj/h3Lcmtx8HQavSbBgXnhnZVRb2snOKD7auqnqjfT/7QMm4ZyKvaOEgyggGktKqEKYHC8KOZ7yp8I5/UMDtk6j9TnXpSqqBxPiud4MDV76SfRYEQiaDoG4tGGvSfPJ9KcNjKrNvSyyxns= - file: dist/* - file_glob: true - skip_cleanup: true - on: - repo: ipython/ipython - all_branches: true # Backports are released from e.g. 5.x branch - tags: true - python: 3.6 # Any version should work, but we only need one + provider: releases + api_key: + secure: Y/Ae9tYs5aoBU8bDjN2YrwGG6tCbezj/h3Lcmtx8HQavSbBgXnhnZVRb2snOKD7auqnqjfT/7QMm4ZyKvaOEgyggGktKqEKYHC8KOZ7yp8I5/UMDtk6j9TnXpSqqBxPiud4MDV76SfRYEQiaDoG4tGGvSfPJ9KcNjKrNvSyyxns= + file: dist/* + file_glob: true + skip_cleanup: true + on: + repo: ipython/ipython + all_branches: true # Backports are released from e.g. 5.x branch + tags: true + python: 3.6 # Any version should work, but we only need one + condition: $TRAVIS_OS_NAME = "linux" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 752486042b3..3aecb233319 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,32 @@ +## Triaging Issues + +On the IPython repository, we strive to trust users and give them responsibility. +By using one of our bots, any user can close issues or add/remove +labels by mentioning the bot and asking it to do things on your behalf. + +To close an issue (or PR), even if you did not create it, use the following: + +> @meeseeksdev close + +This command can be in the middle of another comment, but must start on its +own line. + +To add labels to an issue, ask the bot to `tag` with a comma-separated list of +tags to add: + +> @meeseeksdev tag windows, documentation + +Only already pre-created tags can be added. So far, the list is limited to: +`async/await`, `backported`, `help wanted`, `documentation`, `notebook`, +`tab-completion`, `windows` + +To remove a label, use the `untag` command: + +> @meeseeksdev untag windows, documentation + +We'll be adding additional capabilities for the bot and will share them here +when they are ready to be used. + ## Opening an Issue When opening a new Issue, please take the following steps: @@ -6,13 +35,13 @@ When opening a new Issue, please take the following steps: Keyword searches for your error messages are most helpful. 2. If possible, try updating to master and reproducing your issue, because we may have already fixed it. -3. Try to include a minimal reproducible test case +3. Try to include a minimal reproducible test case. 4. Include relevant system information. Start with the output of: python -c "import IPython; print(IPython.sys_info())" - And include any relevant package versions, depending on the issue, - such as matplotlib, numpy, Qt, Qt bindings (PyQt/PySide), tornado, web browser, etc. + And include any relevant package versions, depending on the issue, such as + matplotlib, numpy, Qt, Qt bindings (PyQt/PySide), tornado, web browser, etc. ## Pull Requests @@ -26,8 +55,8 @@ Some guidelines on contributing to IPython: The worst case is that the PR is closed. * Pull Requests should generally be made against master * Pull Requests should be tested, if feasible: - - bugfixes should include regression tests - - new behavior should at least get minimal exercise + - bugfixes should include regression tests. + - new behavior should at least get minimal exercise. * New features and backwards-incompatible changes should be documented by adding a new file to the [pr](docs/source/whatsnew/pr) directory, see [the README.md there](docs/source/whatsnew/pr/README.md) for details. @@ -43,3 +72,24 @@ particularly for PRs that affect `IPython.parallel` or Windows. For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow). +## Running Tests + +All the tests can by running +```shell +iptest +``` + +All the tests for a single module (for example **test_alias**) can be run by using the fully qualified path to the module. +```shell +iptest IPython.core.tests.test_alias +``` + +Only a single test (for example **test_alias_lifecycle**) within a single file can be run by adding the specific test after a `:` at the end: +```shell +iptest IPython.core.tests.test_alias:test_alias_lifecycle +``` + +For convenience, the full path to a file can often be used instead of the module path on unix systems. For example we can run all the tests by using +```shell +iptest IPython/core/tests/test_alias.py +``` diff --git a/IPython/__init__.py b/IPython/__init__.py index 7bd0ee3c464..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -27,12 +27,13 @@ #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 5): +if sys.version_info < (3, 6): raise ImportError( """ -IPython 7.0+ supports Python 3.5 and above. +IPython 7.10+ supports Python 3.6 and above. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. See IPython `README.rst` file for more information: @@ -64,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. @@ -75,7 +80,7 @@ def embed_kernel(module=None, local_ns=None, **kwargs): Parameters ---------- - module : ModuleType, optional + module : types.ModuleType, optional The module to load into IPython globals (default: caller) local_ns : dict, optional The namespace to load into IPython user namespace (default: caller) diff --git a/IPython/config.py b/IPython/config.py index cf2bacafad1..964f46f10ac 100644 --- a/IPython/config.py +++ b/IPython/config.py @@ -7,7 +7,7 @@ import sys from warnings import warn -from IPython.utils.shimmodule import ShimModule, ShimWarning +from .utils.shimmodule import ShimModule, ShimWarning warn("The `IPython.config` package has been deprecated since IPython 4.0. " "You should import from traitlets.config instead.", ShimWarning) diff --git a/IPython/conftest.py b/IPython/conftest.py new file mode 100644 index 00000000000..8b2af8c020a --- /dev/null +++ b/IPython/conftest.py @@ -0,0 +1,69 @@ +import types +import sys +import builtins +import os +import pytest +import pathlib +import shutil + +from .testing import tools + + +def get_ipython(): + from .terminal.interactiveshell import TerminalInteractiveShell + if TerminalInteractiveShell._instance: + return TerminalInteractiveShell.instance() + + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + + # Create and initialize our test-friendly IPython instance. + shell = TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture(scope='session', autouse=True) +def work_path(): + path = pathlib.Path("./tmp-ipython-pytest-profiledir") + os.environ["IPYTHONDIR"] = str(path.absolute()) + if path.exists(): + raise ValueError('IPython dir temporary path already exists ! Did previous test run exit successfully ?') + path.mkdir() + yield + shutil.rmtree(str(path.resolve())) + + +def nopage(strng, start=0, screen_lines=0, pager_cmd=None): + if isinstance(strng, dict): + strng = strng.get("text/plain", "") + print(strng) + + +def xsys(self, cmd): + """Replace the default system call with a capturing one for doctest. + """ + # We use getoutput, but we need to strip it because pexpect captures + # the trailing newline differently from commands.getoutput + print(self.getoutput(cmd, split=False, depth=1).rstrip(), end="", file=sys.stdout) + sys.stdout.flush() + + +# for things to work correctly we would need this as a session fixture; +# unfortunately this will fail on some test that get executed as _collection_ +# time (before the fixture run), in particular parametrized test that contain +# yields. so for now execute at import time. +#@pytest.fixture(autouse=True, scope='session') +def inject(): + + builtins.get_ipython = get_ipython + builtins._ip = get_ipython() + builtins.ip = get_ipython() + builtins.ip.system = types.MethodType(xsys, ip) + builtins.ip.builtin_trap.activate() + from .core import page + + page.pager_page = nopage + # yield + + +inject() diff --git a/IPython/core/alias.py b/IPython/core/alias.py index ee377d5ccf3..2ad990231a0 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -25,7 +25,7 @@ import sys from traitlets.config.configurable import Configurable -from IPython.core.error import UsageError +from .error import UsageError from traitlets import List, Instance from logging import error @@ -204,6 +204,8 @@ def __init__(self, shell=None, **kwargs): def init_aliases(self): # Load default & user aliases for name, cmd in self.default_aliases + self.user_aliases: + if cmd.startswith('ls ') and self.shell.colors == 'NoColor': + cmd = cmd.replace(' --color', '') self.soft_define_alias(name, cmd) @property diff --git a/IPython/core/application.py b/IPython/core/application.py index fea2b50f561..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. @@ -293,7 +293,7 @@ def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): printed on screen. For testing, the suppress_errors option is set to False, so errors will make tests fail. - `supress_errors` default value is to be `None` in which case the + `suppress_errors` default value is to be `None` in which case the behavior default to the one of `traitlets.Application`. The default value can be set : diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index a6ff86031d0..fb4cc193250 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -13,6 +13,7 @@ import ast import sys +import inspect from textwrap import dedent, indent @@ -97,14 +98,25 @@ class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): the implementation involves wrapping the repl in an async function, it is erroneously allowed (e.g. yield or return at the top level) """ + def __init__(self): + if sys.version_info >= (3,8): + raise ValueError('DEPRECATED in Python 3.8+') + self.depth = 0 + super().__init__() def generic_visit(self, node): func_types = (ast.FunctionDef, ast.AsyncFunctionDef) - invalid_types = (ast.Return, ast.Yield, ast.YieldFrom) - - if isinstance(node, func_types): - return # Don't recurse into functions - elif isinstance(node, invalid_types): + invalid_types_by_depth = { + 0: (ast.Return, ast.Yield, ast.YieldFrom), + 1: (ast.Nonlocal,) + } + + should_traverse = self.depth < max(invalid_types_by_depth.keys()) + if isinstance(node, func_types) and should_traverse: + self.depth += 1 + super().generic_visit(node) + self.depth -= 1 + elif isinstance(node, invalid_types_by_depth[self.depth]): raise SyntaxError() else: super().generic_visit(node) @@ -134,16 +146,20 @@ def _should_be_async(cell: str) -> bool: If it works, assume it should be async. Otherwise Return False. - Not handled yet: If the block of code has a return statement as the top + Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ - + if sys.version_info > (3, 8): + try: + code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE + except (SyntaxError, MemoryError): + return False try: # we can't limit ourself to ast.parse, as it __accepts__ to parse on # 3.7+, but just does not _compile_ - compile(cell, "<>", "exec") - return False - except SyntaxError: + code = compile(cell, "<>", "exec") + except (SyntaxError, MemoryError): try: parse_tree = _async_parse_cell(cell) @@ -151,7 +167,7 @@ def _should_be_async(cell: str) -> bool: v = _AsyncSyntaxErrorVisitor() v.visit(parse_tree) - except SyntaxError: + except (SyntaxError, MemoryError): return False return True return False diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index 6a055f93361..c4771af7303 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -35,6 +35,7 @@ import linecache import operator import time +from contextlib import contextmanager #----------------------------------------------------------------------------- # Constants @@ -134,6 +135,21 @@ def cache(self, code, number=0): linecache._ipython_cache[name] = entry return name + @contextmanager + def extra_flags(self, flags): + ## bits that we'll set to 1 + turn_on_bits = ~self.flags & flags + + + self.flags = self.flags | flags + try: + yield + finally: + # turn off only the bits we turned on so that something like + # __future__ that set flags stays. + self.flags &= ~turn_on_bits + + def check_linecache_ipython(*args): """Call linecache.checkcache() safely protecting our cached values. """ diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 9858236905d..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -67,9 +67,9 @@ Starting with IPython 6.0, this module can make use of the Jedi library to generate completions both using static analysis of the code, and dynamically -inspecting multiple namespaces. The APIs attached to this new mechanism is -unstable and will raise unless use in an :any:`provisionalcompleter` context -manager. +inspecting multiple namespaces. Jedi is an autocompletion and static analysis +for Python. The APIs attached to this new mechanism is unstable and will +raise unless use in an :any:`provisionalcompleter` context manager. You will find that the following are experimental: @@ -84,7 +84,7 @@ We welcome any feedback on these new API, and we also encourage you to try this module in debug mode (start IPython with ``--Completer.debug=True``) in order -to have extra logging information is :any:`jedi` is crashing, or if current +to have extra logging information if :any:`jedi` is crashing, or if current IPython completer pending deprecations are returning results not yet handled by :any:`jedi` @@ -126,7 +126,7 @@ from contextlib import contextmanager from importlib import import_module -from typing import Iterator, List, Tuple, Iterable, Union +from typing import Iterator, List, Tuple, Iterable from types import SimpleNamespace from traitlets.config.configurable import Configurable @@ -185,11 +185,11 @@ def provisionalcompleter(action='ignore'): """ - This contest manager has to be used in any place where unstable completer + This context manager has to be used in any place where unstable completer behavior and API may be called. >>> with provisionalcompleter(): - ... completer.do_experimetal_things() # works + ... completer.do_experimental_things() # works >>> completer.do_experimental_things() # raises. @@ -198,12 +198,11 @@ def provisionalcompleter(action='ignore'): By using this context manager you agree that the API in use may change without warning, and that you won't complain if they do so. - You also understand that if the API is not to you liking you should report - a bug to explain your use case upstream and improve the API and will loose - credibility if you complain after the API is make stable. + You also understand that, if the API is not to your liking, you should report + a bug to explain your use case upstream. - We'll be happy to get your feedback , feature request and improvement on - any of the unstable APIs ! + We'll be happy to get your feedback, feature requests, and improvements on + any of the unstable APIs! """ with warnings.catch_warnings(): warnings.filterwarnings(action, category=ProvisionalCompleterWarning) @@ -576,9 +575,9 @@ class Completer(Configurable): """ ).tag(config=True) - use_jedi = Bool(default_value=False, + use_jedi = Bool(default_value=JEDI_INSTALLED, help="Experimental: Use Jedi to generate autocompletions. " - "Off by default.").tag(config=True) + "Default to True if jedi is installed.").tag(config=True) jedi_compute_type_timeout = Int(default_value=400, help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types. @@ -627,6 +626,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -992,6 +993,8 @@ def _make_signature(completion)-> str: class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" + _names = None + @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" @@ -1121,12 +1124,14 @@ def matchers(self): if self.use_jedi: return [ + *self.custom_matchers, self.file_matches, self.magic_matches, self.dict_key_matches, ] else: return [ + *self.custom_matchers, self.python_matches, self.file_matches, self.magic_matches, @@ -1134,10 +1139,15 @@ def matchers(self): self.dict_key_matches, ] - def all_completions(self, text): + def all_completions(self, text) -> List[str]: """ - Wrapper around the complete method for the benefit of emacs. + Wrapper around the completion methods for the benefit of emacs. """ + prefix = text.rpartition('.')[0] + with provisionalcompleter(): + return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text + for c in self.completions(text, len(text))] + return self.complete(text)[1] def _clean_glob(self, text): @@ -1365,18 +1375,18 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): try_jedi = True try: - # should we check the type of the node is Error ? + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False try: - # jedi < 0.11 - from jedi.parser.tree import ErrorLeaf - except ImportError: - # jedi >= 0.11 - from parso.tree import ErrorLeaf + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - next_to_last_tree = interpreter._get_module().tree_node.children[-2] - completing_string = False - if isinstance(next_to_last_tree, ErrorLeaf): - completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"} # if we are in a string jedi is likely not the right candidate for # now. Skip it. try_jedi = not completing_string @@ -1538,24 +1548,19 @@ def python_func_kw_matches(self,text): usedNamedArgs.add(token) - # lookup the candidate callable matches either using global_matches - # or attr_matches for dotted names - if len(ids) == 1: - callableMatches = self.global_matches(ids[0]) - else: - callableMatches = self.attr_matches('.'.join(ids[::-1])) argMatches = [] - for callableMatch in callableMatches: - try: - namedArgs = self._default_arguments(eval(callableMatch, - self.namespace)) - except: - continue + try: + callableObj = '.'.join(ids[::-1]) + namedArgs = self._default_arguments(eval(callableObj, + self.namespace)) # Remove used named arguments from the list, no need to show twice for namedArg in set(namedArgs) - usedNamedArgs: if namedArg.startswith(text): argMatches.append(u"%s=" %namedArg) + except: + pass + return argMatches def dict_key_matches(self, text): @@ -1694,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1708,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): @@ -1978,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. @@ -1989,7 +1993,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () name_text = '' name_matches = [] - for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches): + # need to add self.fwd_unicode_match() function here when done + for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): name_text, name_matches = meth(base_text) if name_text: return name_text, name_matches[:MATCHES_LIMIT], \ @@ -2012,7 +2017,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # Start with a clean slate of completions matches = [] - custom_res = self.dispatch_custom_completer(text) + # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have @@ -2023,29 +2028,24 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, full_text = line_buffer completions = self._jedi_matches( cursor_pos, cursor_line, full_text) - if custom_res is not None: - # did custom completers produce something? - matches = [(m, 'custom') for m in custom_res] + + if self.merge_completions: + matches = [] + for matcher in self.matchers: + try: + matches.extend([(m, matcher.__qualname__) + for m in matcher(text)]) + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) else: - # Extend the list of completions with the results of each - # matcher, so we return results to the user from all - # namespaces. - if self.merge_completions: - matches = [] - for matcher in self.matchers: - try: - matches.extend([(m, matcher.__qualname__) - for m in matcher(text)]) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - matches = [(m, matcher.__qualname__) - for m in matcher(text)] - if matches: - break + for matcher in self.matchers: + matches = [(m, matcher.__qualname__) + for m in matcher(text)] + if matches: + break + seen = set() filtered_matches = set() for m in matches: @@ -2054,13 +2054,39 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, filtered_matches.add(m) seen.add(t) - _filtered_matches = sorted( - set(filtered_matches), key=lambda x: completions_sorting_key(x[0]))\ - [:MATCHES_LIMIT] + _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) + custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] + + _filtered_matches = custom_res or _filtered_matches + + _filtered_matches = _filtered_matches[:MATCHES_LIMIT] _matches = [m[0] for m in _filtered_matches] origins = [m[1] for m in _filtered_matches] self.matches = _matches return text, _matches, origins, completions + + def fwd_unicode_match(self, text:str) -> Tuple[str, list]: + if self._names is None: + self._names = [] + for c in range(0,0x10FFFF + 1): + try: + self._names.append(unicodedata.name(chr(c))) + except ValueError: + pass + + slashpos = text.rfind('\\') + # if text starts with slash + if slashpos > -1: + s = text[slashpos+1:] + candidates = [x for x in self._names if x.startswith(s)] + if candidates: + return s, candidates + else: + return '', () + + # if text does not start with slash + else: + return u'', () diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index eeb39a48b68..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -30,9 +30,9 @@ from zipimport import zipimporter # Our own imports -from IPython.core.completer import expand_user, compress_user -from IPython.core.error import TryNext -from IPython.utils._process_common import arg_split +from .completer import expand_user, compress_user +from .error import TryNext +from ..utils._process_common import arg_split # FIXME: this should be pulled in with the right call via the component system from IPython import get_ipython @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) @@ -185,7 +185,7 @@ def try_import(mod: str, only_modules=False) -> List[str]: #----------------------------------------------------------------------------- def quick_completer(cmd, completions): - """ Easily create a trivial completer for a command. + r""" Easily create a trivial completer for a command. Takes either a list of completions, or all completions in string (that will be split on whitespace). diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index f3abc1c6fe0..1e0b429d09a 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -29,6 +29,8 @@ from IPython.utils.sysinfo import sys_info from IPython.utils.py3compat import input +from IPython.core.release import __version__ as version + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -68,7 +70,7 @@ """ _lite_message_template = """ -If you suspect this is an IPython bug, please report it at: +If you suspect this is an IPython {version} bug, please report it at: https://github.com/ipython/ipython/issues or send an email to the mailing list at {email} @@ -179,13 +181,14 @@ def __call__(self, etype, evalue, etb): print('Could not create crash report on disk.', file=sys.stderr) return - # Inform user on stderr of what happened - print('\n'+'*'*70+'\n', file=sys.stderr) - print(self.message_template.format(**self.info), file=sys.stderr) + with report: + # Inform user on stderr of what happened + print('\n'+'*'*70+'\n', file=sys.stderr) + print(self.message_template.format(**self.info), file=sys.stderr) + + # Construct report on disk + report.write(self.make_report(traceback)) - # Construct report on disk - report.write(self.make_report(traceback)) - report.close() input("Hit to quit (your terminal may close):") def make_report(self,traceback): @@ -221,5 +224,5 @@ def crash_handler_lite(etype, evalue, tb): else: # we are not in a shell, show generic config config = "c." - print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr) + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 4ece380bf9c..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -153,10 +153,7 @@ def __init__(self, colors=None): # at least raise that limit to 80 chars, which should be enough for # most interactive uses. try: - try: - from reprlib import aRepr # Py 3 - except ImportError: - from repr import aRepr # Py 2 + from reprlib import aRepr aRepr.maxstring = 80 except: # This is only a user-facing convenience, so any error we encounter @@ -195,22 +192,6 @@ def wrapper(*args, **kw): return wrapper -def _file_lines(fname): - """Return the contents of a named file as a list of lines. - - This function never raises an IOError exception: if the file can't be - read, it simply returns an empty list.""" - - try: - outfile = open(fname) - except IOError: - return [] - else: - out = outfile.readlines() - outfile.close() - return out - - class Pdb(OldPdb): """Modified Pdb class, does not load readline. @@ -220,7 +201,19 @@ class Pdb(OldPdb): """ def __init__(self, color_scheme=None, completekey=None, - stdin=None, stdout=None, context=5): + stdin=None, stdout=None, context=5, **kwargs): + """Create a new IPython debugger. + + :param color_scheme: Deprecated, do not use. + :param completekey: Passed to pdb.Pdb. + :param stdin: Passed to pdb.Pdb. + :param stdout: Passed to pdb.Pdb. + :param context: Number of lines of source code context to show when + displaying stacktrace information. + :param kwargs: Passed to pdb.Pdb. + The possibilities are python version dependent, see the python + docs for more info. + """ # Parent constructor: try: @@ -230,7 +223,8 @@ def __init__(self, color_scheme=None, completekey=None, except (TypeError, ValueError): raise ValueError("Context must be a positive integer") - OldPdb.__init__(self, completekey, stdin, stdout) + # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. + OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) # IPython changes... self.shell = get_ipython() @@ -286,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - sys.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -326,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -335,12 +336,25 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass - def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ', + def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', context=None): if context is None: context = self.context @@ -350,7 +364,7 @@ def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ', raise ValueError("Context must be a positive integer") except (TypeError, ValueError): raise ValueError("Context must be a positive integer") - print(self.format_stack_entry(frame_lineno, '', context)) + print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) # vds: >> frame, lineno = frame_lineno @@ -364,9 +378,9 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): try: context=int(context) if context <= 0: - print("Context must be a positive integer") + print("Context must be a positive integer", file=self.stdout) except (TypeError, ValueError): - print("Context must be a positive integer") + print("Context must be a positive integer", file=self.stdout) try: import reprlib # Py 3 except ImportError: @@ -488,11 +502,21 @@ def print_list_lines(self, filename, first, last): src.append(line) self.lineno = lineno - print(''.join(src)) + print(''.join(src), file=self.stdout) except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -511,7 +535,7 @@ def do_list(self, arg): else: first = max(1, int(x) - 5) except: - print('*** Error in argument:', repr(arg)) + print('*** Error in argument:', repr(arg), file=self.stdout) return elif self.lineno is None: first = max(1, self.curframe.f_lineno - 5) @@ -628,13 +652,148 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + def set_trace(frame=None): """ diff --git a/IPython/core/display.py b/IPython/core/display.py index ca5c3aa5b86..424414a662f 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -13,6 +13,8 @@ import sys import warnings from copy import deepcopy +from os.path import splitext +from pathlib import Path, PurePath from IPython.utils.py3compat import cast_unicode from IPython.testing.skipdoctest import skip_doctest @@ -238,16 +240,22 @@ def display(*objs, include=None, exclude=None, metadata=None, transient=None, di want to use. Here is a list of the names of the special methods and the values they must return: - - `_repr_html_`: return raw HTML as a string - - `_repr_json_`: return a JSONable dict - - `_repr_jpeg_`: return raw JPEG data - - `_repr_png_`: return raw PNG data - - `_repr_svg_`: return raw SVG data as a string - - `_repr_latex_`: return LaTeX commands in a string surrounded by "$". + - `_repr_html_`: return raw HTML as a string, or a tuple (see below). + - `_repr_json_`: return a JSONable dict, or a tuple (see below). + - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). + - `_repr_png_`: return raw PNG data, or a tuple (see below). + - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). + - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", + or a tuple (see below). - `_repr_mimebundle_`: return a full mimebundle containing the mapping from all mimetypes to data. Use this for any mime-type not listed above. + The above functions may also return the object's metadata alonside the + data. If the metadata is available, the functions will return a tuple + containing the data and metadata, in that order. If there is no metadata + available, then the functions will return the data only. + When you are directly writing your own classes, you can adapt them for display in IPython by following the above approach. But in practice, you often need to work with existing classes that you can't easily modify. @@ -288,6 +296,13 @@ def display(*objs, include=None, exclude=None, metadata=None, transient=None, di if transient: kwargs['transient'] = transient + if not objs and display_id: + # if given no objects, but still a request for a display_id, + # we assume the user wants to insert an empty output that + # can be updated later + objs = [{}] + raw = True + if not raw: format = InteractiveShell.instance().display_formatter.format @@ -587,6 +602,9 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): metadata : dict Dict of metadata associated to be the object when displayed """ + if isinstance(data, (Path, PurePath)): + data = str(data) + if data is not None and isinstance(data, str): if data.startswith('http') and url is None: url = data @@ -597,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -634,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -666,6 +700,23 @@ def _repr_pretty_(self, pp, cycle): class HTML(TextDisplayObject): + def __init__(self, data=None, url=None, filename=None, metadata=None): + def warn(): + if not data: + return False + + # + # Avoid calling lower() on the entire data, because it could be a + # long string and we're only interested in its beginning and end. + # + prefix = data[:10].lower() + suffix = data[-10:].lower() + return prefix.startswith("') + m_warn.assert_not_called() + + display.HTML('') + m_warn.assert_called_with('Consider using IPython.display.IFrame instead') + + m_warn.reset_mock() + display.HTML('') + m_warn.assert_called_with('Consider using IPython.display.IFrame instead') + def test_progress(): p = display.ProgressBar(10) nt.assert_in('0/10',repr(p)) diff --git a/IPython/core/tests/test_displayhook.py b/IPython/core/tests/test_displayhook.py index 12f4a8d1fa5..6ad89793442 100644 --- a/IPython/core/tests/test_displayhook.py +++ b/IPython/core/tests/test_displayhook.py @@ -3,8 +3,6 @@ from IPython.core.displayhook import CapturingDisplayHook from IPython.utils.capture import CapturedIO -ip = get_ipython() - def test_output_displayed(): """Checking to make sure that output is displayed""" @@ -84,7 +82,7 @@ def test_interactivehooks_ast_modes(): finally: ip.ast_node_interactivity = saved_mode -def test_interactivehooks_ast_modes_semi_supress(): +def test_interactivehooks_ast_modes_semi_suppress(): """ Test that ast nodes can be triggered with different modes and suppressed by semicolon diff --git a/IPython/core/tests/test_events.py b/IPython/core/tests/test_events.py index 37edd919b5d..a4211ecea4c 100644 --- a/IPython/core/tests/test_events.py +++ b/IPython/core/tests/test_events.py @@ -1,4 +1,3 @@ -from backcall import callback_prototype import unittest from unittest.mock import Mock import nose.tools as nt @@ -51,6 +50,12 @@ def test_cb_error(self): with tt.AssertPrints("Error in callback"): self.em.trigger('ping_received') + def test_cb_keyboard_interrupt(self): + cb = Mock(side_effect=KeyboardInterrupt) + self.em.register('ping_received', cb) + with tt.AssertPrints("Error in callback"): + self.em.trigger('ping_received') + def test_unregister_during_callback(self): invoked = [False] * 3 diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index 45409d93120..19248177295 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -10,15 +10,12 @@ # our own packages from IPython.core import autocall from IPython.testing import tools as tt -from IPython.testing.globalipapp import get_ipython -from IPython.utils import py3compat #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- # Get the public instance of IPython -ip = get_ipython() failures = [] num_tests = 0 @@ -51,11 +48,11 @@ def test_handlers(): # For many of the below, we're also checking that leading whitespace # turns off the esc char, which it should unless there is a continuation # line. - run([(i,py3compat.u_format(o)) for i,o in \ + run( [('"no change"', '"no change"'), # normal (u"lsmagic", "get_ipython().run_line_magic('lsmagic', '')"), # magic #("a = b # PYTHON-MODE", '_i'), # emacs -- avoids _in cache - ]]) + ]) # Objects which are instances of IPyAutocall are *always* autocalled autocallable = Autocallable() diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index fcc22f1c00a..f4f080dc83f 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -11,6 +11,7 @@ import sys import tempfile from datetime import datetime +import sqlite3 # third party import nose.tools as nt @@ -19,10 +20,12 @@ from traitlets.config.loader import Config from IPython.utils.tempdir import TemporaryDirectory from IPython.core.history import HistoryManager, extract_hist_ranges +from IPython.testing.decorators import skipif -def setUp(): +def test_proper_default_encoding(): nt.assert_equal(sys.getdefaultencoding(), "utf-8") +@skipif(sqlite3.sqlite_version_info > (3,24,0)) def test_history(): ip = get_ipython() with TemporaryDirectory() as tmpdir: @@ -40,7 +43,7 @@ def test_history(): ip.history_manager.store_output(3) nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist) - + # Detailed tests for _get_range_session grs = ip.history_manager._get_range_session nt.assert_equal(list(grs(start=2,stop=-1)), list(zip([0], [2], hist[1:-1]))) @@ -50,7 +53,7 @@ def test_history(): # Check whether specifying a range beyond the end of the current # session results in an error (gh-804) ip.magic('%hist 2-500') - + # Check that we can write non-ascii characters to a file ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1")) ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2")) @@ -85,6 +88,7 @@ def test_history(): nt.assert_equal(list(gothist), expected) # Check get_hist_search + gothist = ip.history_manager.search("*test*") nt.assert_equal(list(gothist), [(1,2,hist[1])] ) diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 0b3e47d35b5..a39943aed80 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -14,8 +14,6 @@ from IPython.core.inputtransformer import InputTransformer from IPython.core.tests.test_inputtransformer import syntax, syntax_ml from IPython.testing import tools as tt -from IPython.utils import py3compat -from IPython.utils.py3compat import input #----------------------------------------------------------------------------- # Semi-complete examples (also used as tests) @@ -568,8 +566,8 @@ class CellMagicsCommon(object): def test_whole_cell(self): src = "%%cellm line\nbody\n" out = self.sp.transform_cell(src) - ref = u"get_ipython().run_cell_magic('cellm', 'line', 'body')\n" - nt.assert_equal(out, py3compat.u_format(ref)) + ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n" + nt.assert_equal(out, ref) def test_cellmagic_help(self): self.sp.push('%%cellm?') diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 90a1d5afd1e..0d97fd4d6b1 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -113,6 +113,7 @@ def transform_checker(tests, transformer, **kwargs): (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"), (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"), (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"), + (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"), (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"), (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"), (u'a = abc?', "get_ipython().set_next_input('a = abc');" diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index f78a0b37158..b29a0196d3f 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -8,7 +8,9 @@ import string from IPython.core import inputtransformer2 as ipt2 -from IPython.core.inputtransformer2 import make_tokens_by_line +from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op + +from textwrap import dedent MULTILINE_MAGIC = ("""\ a = f() @@ -51,6 +53,22 @@ g() """.splitlines(keepends=True)) +##### + +MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\ +def test(): + for i in range(1): + print(i) + res =! ls +""".splitlines(keepends=True), (4, 7), '''\ +def test(): + for i in range(1): + print(i) + res =get_ipython().getoutput(\' ls\') +'''.splitlines(keepends=True)) + +###### + AUTOCALL_QUOTE = ( [",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'] @@ -101,6 +119,18 @@ [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] ) +HELP_UNICODE = ( + ["π.foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"] +) + + +def null_cleanup_transformer(lines): + """ + A cleanup transform that returns an empty list. + """ + return [] + def check_make_token_by_line_never_ends_empty(): """ Check that not sequence of single or double characters ends up leading to en empty list of tokens @@ -136,18 +166,21 @@ def test_continued_line(): def test_find_assign_magic(): check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False) + check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False) def test_transform_assign_magic(): check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) def test_find_assign_system(): check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) + check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None)) check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None)) check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False) def test_transform_assign_system(): check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) + check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) def test_find_magic_escape(): check_find(ipt2.EscapedCommand, MULTILINE_MAGIC) @@ -195,10 +228,25 @@ def test_transform_help(): tf = ipt2.HelpEnd((1, 0), (2, 8)) nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2]) + tf = ipt2.HelpEnd((1, 0), (1, 0)) + nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2]) + +def test_find_assign_op_dedent(): + """ + be careful that empty token like dedent are not counted as parens + """ + class Tk: + def __init__(self, s): + self.string = s + + nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2) + nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6) + def test_check_complete(): cc = ipt2.TransformerManager().check_complete nt.assert_equal(cc("a = 1"), ('complete', None)) nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4)) + nt.assert_equal(cc("for a in range(5):\n if a > 0:"), ('incomplete', 8)) nt.assert_equal(cc("raise = 2"), ('invalid', None)) nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0)) nt.assert_equal(cc(")"), ('incomplete', 0)) @@ -206,6 +254,16 @@ def test_check_complete(): nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3)) nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None)) nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash + nt.assert_equal(cc("1\\\n+2"), ('complete', None)) + nt.assert_equal(cc("exit"), ('complete', None)) + + example = dedent(""" + if True: + a=1""" ) + + nt.assert_equal(cc(example), ('incomplete', 4)) + nt.assert_equal(cc(example+'\n'), ('complete', None)) + nt.assert_equal(cc(example+'\n '), ('complete', None)) # no need to loop on all the letters/numbers. short = '12abAB'+string.printable[62:] @@ -215,3 +273,20 @@ def test_check_complete(): for k in short: cc(c+k) + nt.assert_equal(cc("def f():\n x=0\n \\\n "), ('incomplete', 2)) + +def test_check_complete_II(): + """ + Test that multiple line strings are properly handled. + + Separate test function for convenience + + """ + cc = ipt2.TransformerManager().check_complete + nt.assert_equal(cc('''def foo():\n """'''), ('incomplete', 4)) + + +def test_null_cleanup_transformer(): + manager = ipt2.TransformerManager() + manager.cleanup_transforms.insert(0, null_cleanup_transformer) + nt.assert_is(manager.transform_cell(""), "") diff --git a/IPython/core/tests/test_inputtransformer2_line.py b/IPython/core/tests/test_inputtransformer2_line.py index 13a18d2d537..41b6ed2935c 100644 --- a/IPython/core/tests/test_inputtransformer2_line.py +++ b/IPython/core/tests/test_inputtransformer2_line.py @@ -86,3 +86,31 @@ def test_leading_indent(): for sample, expected in [INDENT_SPACES, INDENT_TABS]: nt.assert_equal(ipt2.leading_indent(sample.splitlines(keepends=True)), expected.splitlines(keepends=True)) + +LEADING_EMPTY_LINES = ("""\ + \t + +if True: + a = 3 + +b = 4 +""", """\ +if True: + a = 3 + +b = 4 +""") + +ONLY_EMPTY_LINES = ("""\ + \t + +""", """\ + \t + +""") + +def test_leading_empty_lines(): + for sample, expected in [LEADING_EMPTY_LINES, ONLY_EMPTY_LINES]: + nt.assert_equal( + ipt2.leading_empty_lines(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 39fa41bd4df..496e3bd02bc 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -36,7 +36,6 @@ # Globals #----------------------------------------------------------------------------- # This is used by every single test, no point repeating it ad nauseam -ip = get_ipython() #----------------------------------------------------------------------------- # Tests @@ -127,8 +126,8 @@ def test_gh_597(self): """Pretty-printing lists of objects with non-ascii reprs may cause problems.""" class Spam(object): - def __repr__(self): - return "\xe9"*50 + def __repr__(self): + return "\xe9"*50 import IPython.core.formatters f = IPython.core.formatters.PlainTextFormatter() f([Spam(),Spam()]) @@ -495,6 +494,16 @@ def test_last_execution_result(self): self.assertFalse(ip.last_execution_result.success) self.assertIsInstance(ip.last_execution_result.error_in_exec, NameError) + def test_reset_aliasing(self): + """ Check that standard posix aliases work after %reset. """ + if os.name != 'posix': + return + + ip.reset() + for cmd in ('clear', 'more', 'less', 'man'): + res = ip.run_cell('%' + cmd) + self.assertEqual(res.success, True) + class TestSafeExecfileNonAsciiPath(unittest.TestCase): @@ -520,6 +529,10 @@ def test_1(self): ip.safe_execfile(self.fname, {}, raise_exceptions=True) class ExitCodeChecks(tt.TempFileMixin): + + def setUp(self): + self.system = ip.system_raw + def test_exit_code_ok(self): self.system('exit 0') self.assertEqual(ip.user_ns['_exit_code'], 0) @@ -549,8 +562,11 @@ def test_exit_code_signal_csh(self): del os.environ['SHELL'] -class TestSystemRaw(ExitCodeChecks, unittest.TestCase): - system = ip.system_raw +class TestSystemRaw(ExitCodeChecks): + + def setUp(self): + super().setUp() + self.system = ip.system_raw @onlyif_unicode_paths def test_1(self): @@ -570,8 +586,11 @@ def test_control_c(self, *mocks): self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT) # TODO: Exit codes are currently ignored on Windows. -class TestSystemPipedExitCode(ExitCodeChecks, unittest.TestCase): - system = ip.system_piped +class TestSystemPipedExitCode(ExitCodeChecks): + + def setUp(self): + super().setUp() + self.system = ip.system_piped @skip_win32 def test_exit_code_ok(self): @@ -585,7 +604,7 @@ def test_exit_code_error(self): def test_exit_code_signal(self): ExitCodeChecks.test_exit_code_signal(self) -class TestModules(tt.TempFileMixin, unittest.TestCase): +class TestModules(tt.TempFileMixin): def test_extraneous_loads(self): """Test we're not loading modules on startup that we shouldn't. """ @@ -599,10 +618,18 @@ def test_extraneous_loads(self): class Negator(ast.NodeTransformer): """Negates all number literals in an AST.""" + + # for python 3.7 and earlier def visit_Num(self, node): node.n = -node.n return node + # for python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + class TestAstTransform(unittest.TestCase): def setUp(self): self.negator = Negator() @@ -664,12 +691,23 @@ def test_macro(self): class IntegerWrapper(ast.NodeTransformer): """Wraps all integers in a call to Integer()""" + + # for Python 3.7 and earlier + + # for Python 3.7 and earlier def visit_Num(self, node): if isinstance(node.n, int): return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()), args=[node], keywords=[]) return node + # For Python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + + class TestAstTransform2(unittest.TestCase): def setUp(self): self.intwrapper = IntegerWrapper() @@ -710,15 +748,24 @@ def f(x): class ErrorTransformer(ast.NodeTransformer): """Throws an error when it sees a number.""" + + # for Python 3.7 and earlier def visit_Num(self, node): raise ValueError("test") + # for Python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + + class TestAstTransformError(unittest.TestCase): def test_unregistering(self): err_transformer = ErrorTransformer() ip.ast_transformers.append(err_transformer) - with tt.AssertPrints("unregister", channel='stderr'): + with self.assertWarnsRegex(UserWarning, "It will be unregistered"): ip.run_cell("1 + 2") # This should have been removed. @@ -731,10 +778,17 @@ class StringRejector(ast.NodeTransformer): Used to verify that NodeTransformers can signal that a piece of code should not be executed by throwing an InputRejected. """ - + + #for python 3.7 and earlier def visit_Str(self, node): raise InputRejected("test") + # 3.8 only + def visit_Constant(self, node): + if isinstance(node.value, str): + raise InputRejected("test") + return node + class TestAstTransformInputRejection(unittest.TestCase): @@ -827,9 +881,6 @@ def test_user_expression(): # back to text only ip.display_formatter.active_types = ['text/plain'] - - - class TestSyntaxErrorTransformer(unittest.TestCase): @@ -863,25 +914,25 @@ def test_syntaxerror_input_transformer(self): ip.run_cell('3456') - -def test_warning_suppression(): - ip.run_cell("import warnings") - try: - with tt.AssertPrints("UserWarning: asdf", channel="stderr"): - ip.run_cell("warnings.warn('asdf')") - # Here's the real test -- if we run that again, we should get the - # warning again. Traditionally, each warning was only issued once per - # IPython session (approximately), even if the user typed in new and - # different code that should have also triggered the warning, leading - # to much confusion. - with tt.AssertPrints("UserWarning: asdf", channel="stderr"): - ip.run_cell("warnings.warn('asdf')") - finally: - ip.run_cell("del warnings") +class TestWarningSuppression(unittest.TestCase): + def test_warning_suppression(self): + ip.run_cell("import warnings") + try: + with self.assertWarnsRegex(UserWarning, "asdf"): + ip.run_cell("warnings.warn('asdf')") + # Here's the real test -- if we run that again, we should get the + # warning again. Traditionally, each warning was only issued once per + # IPython session (approximately), even if the user typed in new and + # different code that should have also triggered the warning, leading + # to much confusion. + with self.assertWarnsRegex(UserWarning, "asdf"): + ip.run_cell("warnings.warn('asdf')") + finally: + ip.run_cell("del warnings") -def test_deprecation_warning(): - ip.run_cell(""" + def test_deprecation_warning(self): + ip.run_cell(""" import warnings def wrn(): warnings.warn( @@ -889,17 +940,17 @@ def wrn(): DeprecationWarning ) """) - try: - with tt.AssertPrints("I AM A WARNING", channel="stderr"): - ip.run_cell("wrn()") - finally: - ip.run_cell("del warnings") - ip.run_cell("del wrn") + try: + with self.assertWarnsRegex(DeprecationWarning, "I AM A WARNING"): + ip.run_cell("wrn()") + finally: + ip.run_cell("del warnings") + ip.run_cell("del wrn") class TestImportNoDeprecate(tt.TempFileMixin): - def setup(self): + def setUp(self): """Make a valid python temp file.""" self.mktmp(""" import warnings @@ -909,6 +960,7 @@ def wrn(): DeprecationWarning ) """) + super().setUp() def test_no_dep(self): """ @@ -946,3 +998,21 @@ def test_should_run_async(): assert not ip.should_run_async("a = 5") assert ip.should_run_async("await x") assert ip.should_run_async("import asyncio; await asyncio.sleep(1)") + + +def test_set_custom_completer(): + num_completers = len(ip.Completer.matchers) + + def foo(*args, **kwargs): + return "I'm a completer!" + + ip.set_custom_completer(foo, 0) + + # check that we've really added a new completer + assert len(ip.Completer.matchers) == num_completers + 1 + + # check that the first completer is the function we defined + assert ip.Completer.matchers[0]() == "I'm a completer!" + + # clean up + ip.Completer.custom_matchers.pop() diff --git a/IPython/core/tests/test_iplib.py b/IPython/core/tests/test_iplib.py index b5305d447ed..adadae56ab9 100644 --- a/IPython/core/tests/test_iplib.py +++ b/IPython/core/tests/test_iplib.py @@ -8,14 +8,6 @@ import nose.tools as nt # our own packages -from IPython.testing.globalipapp import get_ipython - -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- - -# Get the public instance of IPython -ip = get_ipython() #----------------------------------------------------------------------------- # Test functions diff --git a/IPython/core/tests/test_logger.py b/IPython/core/tests/test_logger.py index 4d61ff2433d..ebebac16cfe 100644 --- a/IPython/core/tests/test_logger.py +++ b/IPython/core/tests/test_logger.py @@ -6,8 +6,6 @@ import nose.tools as nt from IPython.utils.tempdir import TemporaryDirectory -_ip = get_ipython() - def test_logstart_inaccessible_file(): try: _ip.logger.logstart(logfname="/") # Opening that filename will fail. diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index dfeabca1f21..877326ccfc3 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -9,7 +9,9 @@ import re import sys import warnings +from textwrap import dedent from unittest import TestCase +from unittest import mock from importlib import invalidate_caches from io import StringIO @@ -23,17 +25,15 @@ from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic, register_line_magic, register_cell_magic) -from IPython.core.magics import execution, script, code, logging +from IPython.core.magics import execution, script, code, logging, osm from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils.io import capture_output -from IPython.utils.tempdir import TemporaryDirectory +from IPython.utils.tempdir import (TemporaryDirectory, + TemporaryWorkingDirectory) from IPython.utils.process import find_cmd - -_ip = get_ipython() - @magic.magics_class class DummyMagics(magic.Magics): pass @@ -148,6 +148,7 @@ def test_rehashx(): # rehashx must fill up syscmdlist scoms = _ip.db['syscmdlist'] nt.assert_true(len(scoms) > 10) + def test_magic_parse_options(): @@ -369,6 +370,31 @@ def test_reset_in_length(): _ip.run_cell("reset -f in") nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1) +class TestResetErrors(TestCase): + + def test_reset_redefine(self): + + @magics_class + class KernelMagics(Magics): + @line_magic + def less(self, shell): pass + + _ip.register_magics(KernelMagics) + + with self.assertLogs() as cm: + # hack, we want to just capture logs, but assertLogs fails if not + # logs get produce. + # so log one things we ignore. + import logging as log_mod + log = log_mod.getLogger() + log.info('Nothing') + # end hack. + _ip.run_cell("reset -f") + + assert len(cm.output) == 1 + for out in cm.output: + assert "Invalid alias" not in out + def test_tb_syntaxerror(): """test %tb after a SyntaxError""" ip = get_ipython() @@ -400,6 +426,15 @@ def test_time(): with tt.AssertPrints("hihi", suppress=False): ip.run_cell("f('hi')") +def test_time_last_not_expression(): + ip.run_cell("%%time\n" + "var_1 = 1\n" + "var_2 = 2\n") + assert ip.user_ns['var_1'] == 1 + del ip.user_ns['var_1'] + assert ip.user_ns['var_2'] == 2 + del ip.user_ns['var_2'] + @dec.skip_win32 def test_time2(): @@ -418,6 +453,29 @@ def test_time3(): "run = 0\n" "run += 1") +def test_multiline_time(): + """Make sure last statement from time return a value.""" + ip = get_ipython() + ip.user_ns.pop('run', None) + + ip.run_cell(dedent("""\ + %%time + a = "ho" + b = "hey" + a+b + """)) + nt.assert_equal(ip.user_ns_hidden['_'], 'hohey') + +def test_time_local_ns(): + """ + Test that local_ns is actually global_ns when running a cell magic + """ + ip = get_ipython() + ip.run_cell("%%time\n" + "myvar = 1") + nt.assert_equal(ip.user_ns['myvar'], 1) + del ip.user_ns['myvar'] + def test_doctest_mode(): "Toggle doctest_mode twice, it should be a no-op and run without error" _ip.magic('doctest_mode') @@ -432,7 +490,7 @@ def test_parse_options(): nt.assert_equal(m.parse_options('foo', '')[1], 'foo') nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo') - + def test_dirops(): """Test various directory handling operations.""" # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/') @@ -452,10 +510,27 @@ def test_dirops(): os.chdir(startdir) +def test_cd_force_quiet(): + """Test OSMagics.cd_force_quiet option""" + _ip.config.OSMagics.cd_force_quiet = True + osmagics = osm.OSMagics(shell=_ip) + + startdir = os.getcwd() + ipdir = os.path.realpath(_ip.ipython_dir) + + try: + with tt.AssertNotPrints(ipdir): + osmagics.cd('"%s"' % ipdir) + with tt.AssertNotPrints(startdir): + osmagics.cd('-') + finally: + os.chdir(startdir) + + def test_xmode(): # Calling xmode three times should be a no-op xmode = _ip.InteractiveTB.mode - for i in range(3): + for i in range(4): _ip.magic("xmode") nt.assert_equal(_ip.InteractiveTB.mode, xmode) @@ -550,6 +625,8 @@ def doctest_precision(): def test_psearch(): with tt.AssertPrints("dict.fromkeys"): _ip.run_cell("dict.fr*?") + with tt.AssertPrints("π.is_integer"): + _ip.run_cell("π = 3.14;\nπ.is_integ*?") def test_timeit_shlex(): """test shlex issues with timeit (#1109)""" @@ -659,6 +736,24 @@ def test_env(self): env = _ip.magic("env") self.assertTrue(isinstance(env, dict)) + def test_env_secret(self): + env = _ip.magic("env") + hidden = "" + with mock.patch.dict( + os.environ, + { + "API_KEY": "abc123", + "SECRET_THING": "ssshhh", + "JUPYTER_TOKEN": "", + "VAR": "abc" + } + ): + env = _ip.magic("env") + assert env["API_KEY"] == hidden + assert env["SECRET_THING"] == hidden + assert env["JUPYTER_TOKEN"] == hidden + assert env["VAR"] == "abc" + def test_env_get_set_simple(self): env = _ip.magic("env var val1") self.assertEqual(env, None) @@ -738,11 +833,41 @@ def cellm33(self, line, cell): nt.assert_equal(c33, None) def test_file(): - """Basic %%file""" + """Basic %%writefile""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) + +@dec.skip_win32 +def test_file_single_quote(): + """Basic %%writefile with embedded single quotes""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, '\'file1\'') + ip.run_cell_magic("writefile", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) + +@dec.skip_win32 +def test_file_double_quote(): + """Basic %%writefile with embedded double quotes""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, '"file1"') + ip.run_cell_magic("writefile", fname, u'\n'.join([ 'line1', 'line2', ])) @@ -752,12 +877,12 @@ def test_file(): nt.assert_in('line2', s) def test_file_var_expand(): - """%%file $filename""" + """%%writefile $filename""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') ip.user_ns['filename'] = fname - ip.run_cell_magic("file", '$filename', u'\n'.join([ + ip.run_cell_magic("writefile", '$filename', u'\n'.join([ 'line1', 'line2', ])) @@ -767,11 +892,11 @@ def test_file_var_expand(): nt.assert_in('line2', s) def test_file_unicode(): - """%%file with unicode cell""" + """%%writefile with unicode cell""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ u'liné1', u'liné2', ])) @@ -781,15 +906,15 @@ def test_file_unicode(): nt.assert_in(u'liné2', s) def test_file_amend(): - """%%file -a amends files""" + """%%writefile -a amends files""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file2') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ 'line1', 'line2', ])) - ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([ + ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([ 'line3', 'line4', ])) @@ -797,7 +922,20 @@ def test_file_amend(): s = f.read() nt.assert_in('line1\n', s) nt.assert_in('line3\n', s) - + +def test_file_spaces(): + """%%file with spaces in filename""" + ip = get_ipython() + with TemporaryWorkingDirectory() as td: + fname = "file name" + ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) def test_script_config(): ip = get_ipython() @@ -1056,7 +1194,8 @@ def test_logging_magic_quiet_from_config(): lm.logstart(os.path.join(td, "quiet_from_config.log")) finally: _ip.logger.logstop() - + + def test_logging_magic_not_quiet(): _ip.config.LoggingMagics.quiet = False lm = logging.LoggingMagics(shell=_ip) @@ -1067,14 +1206,20 @@ def test_logging_magic_not_quiet(): finally: _ip.logger.logstop() -## + +def test_time_no_var_expand(): + _ip.user_ns['a'] = 5 + _ip.user_ns['b'] = [] + _ip.magic('time b.append("{a}")') + assert _ip.user_ns['b'] == ['{a}'] + + # this is slow, put at the end for local testing. -## def test_timeit_arguments(): "Test valid timeit arguments, should not cause SyntaxError (GH #1269)" if sys.version_info < (3,7): - _ip.magic("timeit ('#')") + _ip.magic("timeit -n1 -r1 ('#')") else: # 3.7 optimize no-op statement like above out, and complain there is # nothing in the for loop. - _ip.magic("timeit a=('#')") + _ip.magic("timeit -n1 -r1 a=('#')") diff --git a/IPython/core/tests/test_magic_terminal.py b/IPython/core/tests/test_magic_terminal.py index a9dd7ac937f..79e2d3ed4a5 100644 --- a/IPython/core/tests/test_magic_terminal.py +++ b/IPython/core/tests/test_magic_terminal.py @@ -15,11 +15,6 @@ from IPython.testing import tools as tt -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- -ip = get_ipython() - #----------------------------------------------------------------------------- # Test functions begin #----------------------------------------------------------------------------- @@ -170,7 +165,7 @@ def test_paste_echo(self): ip.write = writer nt.assert_equal(ip.user_ns['a'], 100) nt.assert_equal(ip.user_ns['b'], 200) - nt.assert_equal(out, code+"\n## -- End pasted text --\n") + assert out == code+"\n## -- End pasted text --\n" def test_paste_leading_commas(self): "Test multiline strings with leading commas" diff --git a/IPython/core/tests/test_oinspect.py b/IPython/core/tests/test_oinspect.py index 06d6d5aaa1f..19c6db7c4f8 100644 --- a/IPython/core/tests/test_oinspect.py +++ b/IPython/core/tests/test_oinspect.py @@ -5,20 +5,16 @@ # Distributed under the terms of the Modified BSD License. -from inspect import Signature, Parameter +from inspect import signature, Signature, Parameter import os import re -import sys import nose.tools as nt from .. import oinspect -from IPython.core.magic import (Magics, magics_class, line_magic, - cell_magic, line_cell_magic, - register_line_magic, register_cell_magic, - register_line_cell_magic) + from decorator import decorator -from IPython import get_ipython + from IPython.testing.tools import AssertPrints, AssertNotPrints from IPython.utils.path import compress_user @@ -27,8 +23,12 @@ # Globals and constants #----------------------------------------------------------------------------- -inspector = oinspect.Inspector() -ip = get_ipython() +inspector = None + +def setup_module(): + global inspector + inspector = oinspect.Inspector() + #----------------------------------------------------------------------------- # Local utilities @@ -127,49 +127,6 @@ def method(self, x, z=2): """Some method's docstring""" -class OldStyle: - """An old-style class for testing.""" - pass - - -def f(x, y=2, *a, **kw): - """A simple function.""" - - -def g(y, z=3, *a, **kw): - pass # no docstring - - -@register_line_magic -def lmagic(line): - "A line magic" - - -@register_cell_magic -def cmagic(line, cell): - "A cell magic" - - -@register_line_cell_magic -def lcmagic(line, cell=None): - "A line/cell magic" - - -@magics_class -class SimpleMagics(Magics): - @line_magic - def Clmagic(self, cline): - "A class-based line magic" - - @cell_magic - def Ccmagic(self, cline, ccell): - "A class-based cell magic" - - @line_cell_magic - def Clcmagic(self, cline, ccell=None): - "A class-based line/cell magic" - - class Awkward(object): def __getattr__(self, name): raise Exception(name) @@ -261,10 +218,12 @@ def test_info_serialliar(): # infinite loops: https://github.com/ipython/ipython/issues/9122 nt.assert_less(fib_tracker[0], 9000) +def support_function_one(x, y=2, *a, **kw): + """A simple function.""" + def test_calldef_none(): # We should ignore __call__ for all of these. - for obj in [f, SimpleClass().method, any, str.upper]: - print(obj) + for obj in [support_function_one, SimpleClass().method, any, str.upper]: i = inspector.info(obj) nt.assert_is(i['call_def'], None) @@ -363,6 +322,12 @@ def test_pinfo_nonascii(): ip.user_ns['nonascii2'] = nonascii2 ip._inspect('pinfo', 'nonascii2', detail_level=1) +def test_pinfo_type(): + """ + type can fail in various edge case, for example `type.__subclass__()` + """ + ip._inspect('pinfo', 'type') + def test_pinfo_docstring_no_source(): """Docstring should be included with detail_level=1 if there is no source""" @@ -432,3 +397,51 @@ def test_builtin_init(): init_def = info['init_definition'] nt.assert_is_not_none(init_def) + +def test_render_signature_short(): + def short_fun(a=1): pass + sig = oinspect._render_signature( + signature(short_fun), + short_fun.__name__, + ) + nt.assert_equal(sig, 'short_fun(a=1)') + + +def test_render_signature_long(): + from typing import Optional + + def long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, + ) -> bool: pass + + sig = oinspect._render_signature( + signature(long_function), + long_function.__name__, + ) + nt.assert_in(sig, [ + # Python >=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', + # Python >=3.7 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Union[str, NoneType] = None, +) -> bool\ +''', # Python <=3.6 + '''\ +long_function( + a_really_long_parameter:int, + and_another_long_one:bool=False, + let_us_make_sure_this_is_looong:Union[str, NoneType]=None, +) -> bool\ +''', + ]) diff --git a/IPython/core/tests/test_paths.py b/IPython/core/tests/test_paths.py index 8f09001d9f7..ab1c4132a8e 100644 --- a/IPython/core/tests/test_paths.py +++ b/IPython/core/tests/test_paths.py @@ -19,7 +19,7 @@ XDG_CACHE_DIR = os.path.join(HOME_TEST_DIR, "xdg_cache_dir") IP_TEST_DIR = os.path.join(HOME_TEST_DIR,'.ipython') -def setup(): +def setup_module(): """Setup testenvironment for the module: - Adds dummy home dir tree @@ -31,7 +31,7 @@ def setup(): os.makedirs(os.path.join(XDG_CACHE_DIR, 'ipython')) -def teardown(): +def teardown_module(): """Teardown testenvironment for the module: - Remove dummy home dir tree diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 83d8e908427..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -6,12 +6,10 @@ import nose.tools as nt from IPython.core.prefilter import AutocallChecker -from IPython.testing.globalipapp import get_ipython #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- -ip = get_ipython() def test_prefilter(): """Test user input conversions""" @@ -117,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] diff --git a/IPython/core/tests/test_profile.py b/IPython/core/tests/test_profile.py index 021b31c3f17..e63fb3ef047 100644 --- a/IPython/core/tests/test_profile.py +++ b/IPython/core/tests/test_profile.py @@ -48,7 +48,7 @@ # Setup/teardown functions/decorators # -def setup(): +def setup_module(): """Setup test environment for the module: - Adds dummy home dir tree @@ -58,7 +58,7 @@ def setup(): os.makedirs(IP_TEST_DIR) -def teardown(): +def teardown_module(): """Teardown test environment for the module: - Remove dummy home dir tree diff --git a/IPython/core/tests/test_prompts.py b/IPython/core/tests/test_prompts.py index 4082b1408ef..95e6163b213 100644 --- a/IPython/core/tests/test_prompts.py +++ b/IPython/core/tests/test_prompts.py @@ -4,10 +4,6 @@ import unittest from IPython.core.prompts import LazyEvaluate -from IPython.testing.globalipapp import get_ipython - -ip = get_ipython() - class PromptTests(unittest.TestCase): def test_lazy_eval_unicode(self): diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 181e99f9b84..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -61,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): @@ -248,3 +248,9 @@ def test_qt_gtk(self): def test_no_gui_backends(): for k in ['agg', 'svg', 'pdf', 'ps']: assert k not in pt.backend2gui + + +def test_figure_no_canvas(): + fig = Figure() + fig.canvas = None + pt.print_figure(fig) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 2afa5ba7c51..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -35,7 +35,6 @@ from IPython.utils.tempdir import TemporaryDirectory from IPython.core import debugger - def doctest_refbug(): """Very nasty problem with references held by multiple runs of a script. See: https://github.com/ipython/ipython/issues/141 @@ -166,7 +165,7 @@ def doctest_reset_del(): class TestMagicRunPass(tt.TempFileMixin): - def setup(self): + def setUp(self): content = "a = [1,2,3]\nb = 1" self.mktmp(content) @@ -403,6 +402,25 @@ def test_run_nb(self): nt.assert_equal(_ip.user_ns['answer'], 42) + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + def test_file_options(self): src = ('import sys\n' 'a = " ".join(sys.argv[1:])\n') @@ -537,6 +555,37 @@ def test_run_tb(): nt.assert_not_in("execfile", out) nt.assert_in("RuntimeError", out) nt.assert_equal(out.count("---->"), 3) + del ip.user_ns['bar'] + del ip.user_ns['foo'] + + +def test_multiprocessing_run(): + """Set we can run mutiprocesgin without messing up up main namespace + + Note that import `nose.tools as nt` mdify the value s + sys.module['__mp_main__'] so wee need to temporarily set it to None to test + the issue. + """ + with TemporaryDirectory() as td: + mpm = sys.modules.get('__mp_main__') + assert mpm is not None + sys.modules['__mp_main__'] = None + try: + path = pjoin(td, 'test.py') + with open(path, 'w') as f: + f.write("import multiprocessing\nprint('hoy')") + with capture_output() as io: + _ip.run_line_magic('run', path) + _ip.run_cell("i_m_undefined") + out = io.stdout + nt.assert_in("hoy", out) + nt.assert_not_in("AttributeError", out) + nt.assert_in("NameError", out) + nt.assert_equal(out.count("---->"), 1) + except: + raise + finally: + sys.modules['__mp_main__'] = mpm @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows") def test_script_tb(): diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index fb6898f8103..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -10,7 +10,8 @@ import unittest from unittest import mock -from ..ultratb import ColorTB, VerboseTB, find_recursion +import IPython.core.ultratb as ultratb +from IPython.core.ultratb import ColorTB, VerboseTB, find_recursion from IPython.testing import tools as tt @@ -18,8 +19,6 @@ from IPython.utils.syspathcontext import prepended_to_syspath from IPython.utils.tempdir import TemporaryDirectory -ip = get_ipython() - file_1 = """1 2 3 @@ -31,6 +30,30 @@ def f(): 1/0 """ + +def recursionlimit(frames): + """ + decorator to set the recursion limit temporarily + """ + + def inner(test_function): + def wrapper(*args, **kwargs): + _orig_rec_limit = ultratb._FRAME_RECURSION_LIMIT + ultratb._FRAME_RECURSION_LIMIT = 50 + + rl = sys.getrecursionlimit() + sys.setrecursionlimit(frames) + try: + return test_function(*args, **kwargs) + finally: + sys.setrecursionlimit(rl) + ultratb._FRAME_RECURSION_LIMIT = _orig_rec_limit + + return wrapper + + return inner + + class ChangedPyFileTest(unittest.TestCase): def test_changing_py_file(self): """Traceback produced if the line where the error occurred is missing? @@ -111,6 +134,12 @@ def test_nonascii_msg(self): with tt.AssertPrints(expected): ip.run_cell(cell) + ip.run_cell("%xmode minimal") + with tt.AssertPrints(u"Exception: é"): + ip.run_cell(cell) + + # Put this back into Context mode for later tests. + ip.run_cell("%xmode context") class NestedGenExprTestCase(unittest.TestCase): """ @@ -194,6 +223,8 @@ def bar(): # Assert syntax error during runtime generate stacktrace with tt.AssertPrints(["foo()", "bar()"]): ip.run_cell(syntax_error_at_runtime) + del ip.user_ns['bar'] + del ip.user_ns['foo'] def test_changing_py_file(self): with TemporaryDirectory() as td: @@ -221,6 +252,17 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) + class Python3ChainedExceptionsTest(unittest.TestCase): DIRECT_CAUSE_ERROR_CODE = """ @@ -265,6 +307,25 @@ def test_suppress_exception_chaining(self): tt.AssertPrints("ValueError", suppress=False): ip.run_cell(self.SUPPRESS_CHAINING_CODE) + def test_plain_direct_cause_error(self): + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_exception_during_handling_error(self): + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_suppress_exception_chaining(self): + with tt.AssertNotPrints("ZeroDivisionError"), \ + tt.AssertPrints("ValueError", suppress=False): + ip.run_cell("%xmode Plain") + ip.run_cell(self.SUPPRESS_CHAINING_CODE) + ip.run_cell("%xmode Verbose") + class RecursionTest(unittest.TestCase): DEFINITIONS = """ @@ -296,14 +357,17 @@ def test_no_recursion(self): with tt.AssertNotPrints("frames repeated"): ip.run_cell("non_recurs()") + @recursionlimit(150) def test_recursion_one_frame(self): with tt.AssertPrints("1 frames repeated"): ip.run_cell("r1()") + @recursionlimit(150) def test_recursion_three_frames(self): with tt.AssertPrints("3 frames repeated"): ip.run_cell("r3o2()") + @recursionlimit(150) def test_find_recursion(self): captured = [] def capture_exc(*args, **kwargs): @@ -373,10 +437,16 @@ def eggs(f, g, z=globals()): handler(*sys.exc_info()) buff.write('') +from IPython.testing.decorators import skipif class TokenizeFailureTest(unittest.TestCase): """Tests related to https://github.com/ipython/ipython/issues/6864.""" + # that appear to test that we are handling an exception that can be thrown + # by the tokenizer due to a bug that seem to have been fixed in 3.8, though + # I'm unsure if other sequences can make it raise this error. Let's just + # skip in 3.8 for now + @skipif(sys.version_info > (3,8)) def testLogging(self): message = "An unexpected error occurred while tokenizing input" cell = 'raise ValueError("""a\nb""")' diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 3b97a82dd8d..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -41,7 +41,7 @@ .. note:: The verbose mode print all variables in the stack, which means it can - potentially leak sensitive information like access keys, or unencryted + potentially leak sensitive information like access keys, or unencrypted password. Installation instructions for VerboseTB:: @@ -101,10 +101,7 @@ import tokenize import traceback -try: # Python 2 - generate_tokens = tokenize.generate_tokens -except AttributeError: # Python 3 - generate_tokens = tokenize.tokenize +from tokenize import generate_tokens # For purposes of monkeypatching inspect to fix a bug in it. from inspect import getsourcefile, getfile, getmodule, \ @@ -137,6 +134,12 @@ # to users of ultratb who are NOT running inside ipython. DEFAULT_SCHEME = 'NoColor' + +# Number of frame above which we are likely to have a recursion and will +# **attempt** to detect it. Made modifiable mostly to speedup test suite +# as detecting recursion is one of our slowest test +_FRAME_RECURSION_LIMIT = 500 + # --------------------------------------------------------------------------- # Code begins @@ -431,7 +434,7 @@ def is_recursion_error(etype, value, records): # a recursion error. return (etype is recursion_error_type) \ and "recursion" in str(value).lower() \ - and len(records) > 500 + and len(records) > _FRAME_RECURSION_LIMIT def find_recursion(etype, value, records): """Identify the repeating stack frames from a RecursionError traceback @@ -524,6 +527,30 @@ def _set_ostream(self, val): ostream = property(_get_ostream, _set_ostream) + def get_parts_of_chained_exception(self, evalue): + def get_chained_exception(exception_value): + cause = getattr(exception_value, '__cause__', None) + if cause: + return cause + if getattr(exception_value, '__suppress_context__', False): + return None + return getattr(exception_value, '__context__', None) + + chained_evalue = get_chained_exception(evalue) + + if chained_evalue: + return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + + def prepare_chained_exception_message(self, cause): + direct_cause = "\nThe above exception was the direct cause of the following exception:\n" + exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" + + if cause: + message = [[direct_cause]] + else: + message = [[exception_during_handling]] + return message + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" @@ -597,7 +624,13 @@ def __call__(self, etype, value, elist): self.ostream.write(self.text(etype, value, elist)) self.ostream.write('\n') - def structured_traceback(self, etype, value, elist, tb_offset=None, + def _extract_tb(self, tb): + if tb: + return traceback.extract_tb(tb) + else: + return None + + def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, context=5): """Return a color formatted string with the traceback info. @@ -606,15 +639,16 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, etype : exception type Type of the exception raised. - value : object + evalue : object Data stored in the exception - elist : list - List of frames, see class docstring for details. + etb : object + If list: List of frames, see class docstring for details. + If Traceback: Traceback of the exception. tb_offset : int, optional Number of frames in the traceback to skip. If not given, the - instance value is used (set in constructor). + instance evalue is used (set in constructor). context : int, optional Number of lines of context information to print. @@ -623,6 +657,19 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, ------- String with formatted exception. """ + # This is a workaround to get chained_exc_ids in recursive calls + # etb should not be a tuple if structured_traceback is not recursive + if isinstance(etb, tuple): + etb, chained_exc_ids = etb + else: + chained_exc_ids = set() + + if isinstance(etb, list): + elist = etb + elif etb is not None: + elist = self._extract_tb(etb) + else: + elist = [] tb_offset = self.tb_offset if tb_offset is None else tb_offset Colors = self.Colors out_list = [] @@ -635,9 +682,25 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, (Colors.normalEm, Colors.Normal) + '\n') out_list.extend(self._format_list(elist)) # The exception info should be a single entry in the list. - lines = ''.join(self._format_exception_only(etype, value)) + lines = ''.join(self._format_exception_only(etype, evalue)) out_list.append(lines) + exception = self.get_parts_of_chained_exception(evalue) + + if exception and not id(exception[1]) in chained_exc_ids: + chained_exception_message = self.prepare_chained_exception_message( + evalue.__cause__)[0] + etype, evalue, etb = exception + # Trace exception to avoid infinite 'cause' loop + chained_exc_ids.add(id(exception[1])) + chained_exceptions_tb_offset = 0 + out_list = ( + self.structured_traceback( + etype, evalue, (etb, chained_exc_ids), + chained_exceptions_tb_offset, context) + + chained_exception_message + + out_list) + return out_list def _format_list(self, extracted_list): @@ -757,7 +820,7 @@ def get_exception_only(self, etype, value): etype : exception type value : exception value """ - return ListTB.structured_traceback(self, etype, value, []) + return ListTB.structured_traceback(self, etype, value) def show_exception_only(self, etype, evalue): """Only print the exception type and message, without a traceback. @@ -816,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -1007,16 +1093,6 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline): _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format))) - def prepare_chained_exception_message(self, cause): - direct_cause = "\nThe above exception was the direct cause of the following exception:\n" - exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" - - if cause: - message = [[direct_cause]] - else: - message = [[exception_during_handling]] - return message - def prepare_header(self, etype, long_version=False): colors = self.Colors # just a shorthand + quicker name lookup colorsnormal = colors.Normal # used a lot @@ -1070,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) @@ -1111,20 +1185,6 @@ def get_records(self, etb, number_of_lines_of_context, tb_offset): info('\nUnfortunately, your original traceback can not be constructed.\n') return None - def get_parts_of_chained_exception(self, evalue): - def get_chained_exception(exception_value): - cause = getattr(exception_value, '__cause__', None) - if cause: - return cause - if getattr(exception_value, '__suppress_context__', False): - return None - return getattr(exception_value, '__context__', None) - - chained_evalue = get_chained_exception(evalue) - - if chained_evalue: - return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ - def structured_traceback(self, etype, evalue, etb, tb_offset=None, number_of_lines_of_context=5): """Return a nice text document describing the traceback.""" @@ -1203,7 +1263,7 @@ def debugger(self, force=False): if etb and etb.tb_next: etb = etb.tb_next self.pdb.botframe = etb.tb_frame - self.pdb.interaction(self.tb.tb_frame, self.tb) + self.pdb.interaction(None, etb) if hasattr(self, 'tb'): del self.tb @@ -1251,7 +1311,7 @@ def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, parent=None, config=None): # NEVER change the order of this list. Put new modes at the end: - self.valid_modes = ['Plain', 'Context', 'Verbose'] + self.valid_modes = ['Plain', 'Context', 'Verbose', 'Minimal'] self.verbose_modes = self.valid_modes[1:3] VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, @@ -1262,16 +1322,11 @@ def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, # Different types of tracebacks are joined with different separators to # form a single string. They are taken from this dict - self._join_chars = dict(Plain='', Context='\n', Verbose='\n') + self._join_chars = dict(Plain='', Context='\n', Verbose='\n', + Minimal='') # set_mode also sets the tb_join_char attribute self.set_mode(mode) - def _extract_tb(self, tb): - if tb: - return traceback.extract_tb(tb) - else: - return None - def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset mode = self.mode @@ -1280,14 +1335,15 @@ def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines return VerboseTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context ) + elif mode == 'Minimal': + return ListTB.get_exception_only(self, etype, value) else: # We must check the source cache because otherwise we can print # out-of-date source code. self.check_cache() # Now we can extract and format the exception - elist = self._extract_tb(tb) return ListTB.structured_traceback( - self, etype, value, elist, tb_offset, number_of_lines_of_context + self, etype, value, tb, tb_offset, number_of_lines_of_context ) def stb2text(self, stb): @@ -1324,6 +1380,9 @@ def context(self): def verbose(self): self.set_mode(self.valid_modes[2]) + def minimal(self): + self.set_mode(self.valid_modes[3]) + #---------------------------------------------------------------------------- class AutoFormattedTB(FormattedTB): @@ -1368,7 +1427,11 @@ def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): if etype is None: etype, value, tb = sys.exc_info() - self.tb = tb + if isinstance(tb, tuple): + # tb is a tuple if this is a chained exception. + self.tb = tb[0] + else: + self.tb = tb return FormattedTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context) diff --git a/IPython/extensions/autoreload.py b/IPython/extensions/autoreload.py index 306bb8bd26a..ada680fcf08 100644 --- a/IPython/extensions/autoreload.py +++ b/IPython/extensions/autoreload.py @@ -115,6 +115,7 @@ import traceback import types import weakref +import gc from importlib import import_module from importlib.util import source_from_cache from imp import reload @@ -267,6 +268,18 @@ def update_function(old, new): pass +def update_instances(old, new): + """Use garbage collector to find all instances that refer to the old + class definition and update their __class__ to point to the new class + definition""" + + refs = gc.get_referrers(old) + + for ref in refs: + if type(ref) is old: + ref.__class__ = new + + def update_class(old, new): """Replace stuff in the __dict__ of a class, and upgrade method code objects, and add new methods, if any""" @@ -274,7 +287,9 @@ def update_class(old, new): old_obj = getattr(old, key) try: new_obj = getattr(new, key) - if old_obj == new_obj: + # explicitly checking that comparison returns True to handle + # cases where `==` doesn't return a boolean. + if (old_obj == new_obj) is True: continue except AttributeError: # obsolete attribute: remove it @@ -298,6 +313,9 @@ def update_class(old, new): except (AttributeError, TypeError): pass # skip non-writable attributes + # update all instances of class + update_instances(old, new) + def update_property(old, new): """Replace get/set/del functions of a property""" @@ -338,7 +356,7 @@ def __call__(self): return self.obj -def superreload(module, reload=reload, old_objects={}): +def superreload(module, reload=reload, old_objects=None): """Enhanced version of the builtin reload function. superreload remembers objects previously in the module, and @@ -348,6 +366,8 @@ def superreload(module, reload=reload, old_objects={}): - clears the module's namespace before reloading """ + if old_objects is None: + old_objects = {} # collect old objects in the module for name, obj in list(module.__dict__.items()): diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index 9a203ff5e4a..51b79ad314e 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -20,12 +20,15 @@ from traitlets import Bool -def restore_aliases(ip): +def restore_aliases(ip, alias=None): staliases = ip.db.get('stored_aliases', {}) - for k,v in staliases.items(): - #print "restore alias",k,v # dbg - #self.alias_table[k] = v - ip.alias_manager.define_alias(k,v) + if alias is None: + for k,v in staliases.items(): + #print "restore alias",k,v # dbg + #self.alias_table[k] = v + ip.alias_manager.define_alias(k,v) + else: + ip.alias_manager.define_alias(alias, staliases[alias]) def refresh_variables(ip): @@ -58,13 +61,13 @@ class StoreMagics(Magics): """Lightweight persistence for python variables. Provides the %store magic.""" - + autorestore = Bool(False, help= """If True, any %store-d variables will be automatically restored when IPython starts. """ ).tag(config=True) - + def __init__(self, shell): super(StoreMagics, self).__init__(shell=shell) self.shell.configurables.append(self) @@ -94,13 +97,13 @@ def store(self, parameter_s=''): * ``%store`` - Show list of all variables and their current values - * ``%store spam`` - Store the *current* value of the variable spam - to disk + * ``%store spam bar`` - Store the *current* value of the variables spam + and bar to disk * ``%store -d spam`` - Remove the variable and its value from storage * ``%store -z`` - Remove all variables from storage - * ``%store -r`` - Refresh all variables from store (overwrite - current vals) - * ``%store -r spam bar`` - Refresh specified variables from store + * ``%store -r`` - Refresh all variables, aliases and directory history + from store (overwrite current vals) + * ``%store -r spam bar`` - Refresh specified variables and aliases from store (delete current val) * ``%store foo >a.txt`` - Store value of foo to new file a.txt * ``%store foo >>a.txt`` - Append value of foo to file a.txt @@ -112,10 +115,11 @@ def store(self, parameter_s=''): python types can be safely %store'd. Also aliases can be %store'd across sessions. + To remove an alias from the storage, use the %unalias magic. """ opts,argsl = self.parse_options(parameter_s,'drz',mode='string') - args = argsl.split(None,1) + args = argsl.split() ip = self.shell db = ip.db # delete @@ -140,7 +144,10 @@ def store(self, parameter_s=''): try: obj = db['autorestore/' + arg] except KeyError: - print("no stored variable %s" % arg) + try: + restore_aliases(ip, alias=arg) + except KeyError: + print("no stored variable or alias %s" % arg) else: ip.user_ns[arg] = obj else: @@ -172,55 +179,55 @@ def store(self, parameter_s=''): fil = open(fnam, 'a') else: fil = open(fnam, 'w') - obj = ip.ev(args[0]) - print("Writing '%s' (%s) to file '%s'." % (args[0], - obj.__class__.__name__, fnam)) - - - if not isinstance (obj, str): - from pprint import pprint - pprint(obj, fil) - else: - fil.write(obj) - if not obj.endswith('\n'): - fil.write('\n') + with fil: + obj = ip.ev(args[0]) + print("Writing '%s' (%s) to file '%s'." % (args[0], + obj.__class__.__name__, fnam)) + + if not isinstance (obj, str): + from pprint import pprint + pprint(obj, fil) + else: + fil.write(obj) + if not obj.endswith('\n'): + fil.write('\n') - fil.close() return # %store foo - try: - obj = ip.user_ns[args[0]] - except KeyError: - # it might be an alias - name = args[0] + for arg in args: try: - cmd = ip.alias_manager.retrieve_alias(name) - except ValueError: - raise UsageError("Unknown variable '%s'" % name) - - staliases = db.get('stored_aliases',{}) - staliases[name] = cmd - db['stored_aliases'] = staliases - print("Alias stored: %s (%s)" % (name, cmd)) - return - - else: - modname = getattr(inspect.getmodule(obj), '__name__', '') - if modname == '__main__': - print(textwrap.dedent("""\ - Warning:%s is %s - Proper storage of interactively declared classes (or instances - of those classes) is not possible! Only instances - of classes in real modules on file system can be %%store'd. - """ % (args[0], obj) )) + obj = ip.user_ns[arg] + except KeyError: + # it might be an alias + name = arg + try: + cmd = ip.alias_manager.retrieve_alias(name) + except ValueError: + raise UsageError("Unknown variable '%s'" % name) + + staliases = db.get('stored_aliases',{}) + staliases[name] = cmd + db['stored_aliases'] = staliases + print("Alias stored: %s (%s)" % (name, cmd)) return - #pickled = pickle.dumps(obj) - db[ 'autorestore/' + args[0] ] = obj - print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__)) + + else: + modname = getattr(inspect.getmodule(obj), '__name__', '') + if modname == '__main__': + print(textwrap.dedent("""\ + Warning:%s is %s + Proper storage of interactively declared classes (or instances + of those classes) is not possible! Only instances + of classes in real modules on file system can be %%store'd. + """ % (arg, obj) )) + return + #pickled = pickle.dumps(obj) + db[ 'autorestore/' + arg ] = obj + print("Stored '%s' (%s)" % (arg, obj.__class__.__name__)) def load_ipython_extension(ip): """Load the extension in IPython.""" ip.register_magics(StoreMagics) - + diff --git a/IPython/extensions/sympyprinting.py b/IPython/extensions/sympyprinting.py index 7f9fb2ef98a..e6a83cd34b6 100644 --- a/IPython/extensions/sympyprinting.py +++ b/IPython/extensions/sympyprinting.py @@ -14,7 +14,7 @@ As of SymPy 0.7.2, maintenance of this extension has moved to SymPy under sympy.interactive.ipythonprinting, any modifications to account for changes to SymPy should be submitted to SymPy rather than changed here. This module is -maintained here for backwards compatablitiy with old SymPy versions. +maintained here for backwards compatibility with old SymPy versions. """ #----------------------------------------------------------------------------- diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index a942c5ebc15..e81bf221515 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -24,7 +24,7 @@ import nose.tools as nt import IPython.testing.tools as tt -from IPython.testing.decorators import skipif +from unittest import TestCase from IPython.extensions.autoreload import AutoreloadMagics from IPython.core.events import EventManager, pre_run_cell @@ -35,10 +35,12 @@ noop = lambda *a, **kw: None -class FakeShell(object): +class FakeShell: def __init__(self): self.ns = {} + self.user_ns = self.ns + self.user_ns_hidden = {} self.events = EventManager(self, {'pre_run_cell', pre_run_cell}) self.auto_magics = AutoreloadMagics(shell=self) self.events.register('pre_run_cell', self.auto_magics.pre_run_cell) @@ -47,7 +49,7 @@ def __init__(self): def run_code(self, code): self.events.trigger('pre_run_cell') - exec(code, self.ns) + exec(code, self.user_ns) self.auto_magics.post_execute_hook() def push(self, items): @@ -61,7 +63,7 @@ def magic_aimport(self, parameter, stream=None): self.auto_magics.post_execute_hook() -class Fixture(object): +class Fixture(TestCase): """Fixture for creating test module files""" test_dir = None @@ -104,35 +106,39 @@ def write_file(self, filename, content): (because that is stored in the file). The only reliable way to achieve this seems to be to sleep. """ - + content = textwrap.dedent(content) # Sleep one second + eps time.sleep(1.05) # Write - f = open(filename, 'w') - try: + with open(filename, 'w') as f: f.write(content) - finally: - f.close() def new_module(self, code): + code = textwrap.dedent(code) mod_name, mod_fn = self.get_module() - f = open(mod_fn, 'w') - try: + with open(mod_fn, 'w') as f: f.write(code) - finally: - f.close() return mod_name, mod_fn #----------------------------------------------------------------------------- # Test automatic reloading #----------------------------------------------------------------------------- +def pickle_get_current_class(obj): + """ + Original issue comes from pickle; hence the name. + """ + name = obj.__class__.__name__ + module_name = getattr(obj, "__module__", None) + obj2 = sys.modules[module_name] + for subpath in name.split("."): + obj2 = getattr(obj2, subpath) + return obj2 + class TestAutoreload(Fixture): - @skipif(sys.version_info < (3, 6)) def test_reload_enums(self): - import enum mod_name, mod_fn = self.new_module(textwrap.dedent(""" from enum import Enum class MyEnum(Enum): @@ -151,6 +157,42 @@ class MyEnum(Enum): with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'): self.shell.run_code("pass") # trigger another reload + def test_reload_class_type(self): + self.shell.magic_autoreload("2") + mod_name, mod_fn = self.new_module( + """ + class Test(): + def meth(self): + return "old" + """ + ) + assert "test" not in self.shell.ns + assert "result" not in self.shell.ns + + self.shell.run_code("from %s import Test" % mod_name) + self.shell.run_code("test = Test()") + + self.write_file( + mod_fn, + """ + class Test(): + def meth(self): + return "new" + """, + ) + + test_object = self.shell.ns["test"] + + # important to trigger autoreload logic ! + self.shell.run_code("pass") + + test_class = pickle_get_current_class(test_object) + assert isinstance(test_object, test_class) + + # extra check. + self.shell.run_code("import pickle") + self.shell.run_code("p = pickle.dumps(test)") + def test_reload_class_attributes(self): self.shell.magic_autoreload("2") mod_name, mod_fn = self.new_module(textwrap.dedent(""" @@ -402,3 +444,4 @@ def test_smoketest_autoreload(self): + diff --git a/IPython/extensions/tests/test_storemagic.py b/IPython/extensions/tests/test_storemagic.py index 373a7169261..6f8371d336f 100644 --- a/IPython/extensions/tests/test_storemagic.py +++ b/IPython/extensions/tests/test_storemagic.py @@ -3,33 +3,49 @@ from traitlets.config.loader import Config import nose.tools as nt -ip = get_ipython() -ip.magic('load_ext storemagic') + +def setup_module(): + ip.magic('load_ext storemagic') def test_store_restore(): + assert 'bar' not in ip.user_ns, "Error: some other test leaked `bar` in user_ns" + assert 'foo' not in ip.user_ns, "Error: some other test leaked `foo` in user_ns" + assert 'foobar' not in ip.user_ns, "Error: some other test leaked `foobar` in user_ns" + assert 'foobaz' not in ip.user_ns, "Error: some other test leaked `foobaz` in user_ns" ip.user_ns['foo'] = 78 ip.magic('alias bar echo "hello"') + ip.user_ns['foobar'] = 79 + ip.user_ns['foobaz'] = '80' tmpd = tempfile.mkdtemp() ip.magic('cd ' + tmpd) ip.magic('store foo') ip.magic('store bar') - + ip.magic('store foobar foobaz') + # Check storing nt.assert_equal(ip.db['autorestore/foo'], 78) nt.assert_in('bar', ip.db['stored_aliases']) - + nt.assert_equal(ip.db['autorestore/foobar'], 79) + nt.assert_equal(ip.db['autorestore/foobaz'], '80') + # Remove those items ip.user_ns.pop('foo', None) + ip.user_ns.pop('foobar', None) + ip.user_ns.pop('foobaz', None) ip.alias_manager.undefine_alias('bar') ip.magic('cd -') ip.user_ns['_dh'][:] = [] - + # Check restoring - ip.magic('store -r') + ip.magic('store -r foo bar foobar foobaz') nt.assert_equal(ip.user_ns['foo'], 78) assert ip.alias_manager.is_alias('bar') + nt.assert_equal(ip.user_ns['foobar'], 79) + nt.assert_equal(ip.user_ns['foobaz'], '80') + + ip.magic('store -r') # restores _dh too nt.assert_in(os.path.realpath(tmpd), ip.user_ns['_dh']) - + os.rmdir(tmpd) def test_autorestore(): diff --git a/IPython/external/__init__.py b/IPython/external/__init__.py index 3104c194622..1c8c546f118 100644 --- a/IPython/external/__init__.py +++ b/IPython/external/__init__.py @@ -2,4 +2,4 @@ This package contains all third-party modules bundled with IPython. """ -__all__ = ["simplegeneric"] +__all__ = [] diff --git a/IPython/external/decorators/__init__.py b/IPython/external/decorators/__init__.py index dd8f52b711a..1db80edd357 100644 --- a/IPython/external/decorators/__init__.py +++ b/IPython/external/decorators/__init__.py @@ -1,8 +1,7 @@ try: - from numpy.testing.decorators import * - from numpy.testing.noseclasses import KnownFailure + from numpy.testing import KnownFailure, knownfailureif except ImportError: - from ._decorators import * + from ._decorators import knownfailureif try: from ._numpy_testing_noseclasses import KnownFailure except ImportError: diff --git a/IPython/external/qt_loaders.py b/IPython/external/qt_loaders.py index 2e822e229cd..46cd9c35cb9 100644 --- a/IPython/external/qt_loaders.py +++ b/IPython/external/qt_loaders.py @@ -68,7 +68,7 @@ def commit_api(api): ID.forbid('PySide') ID.forbid('PyQt4') ID.forbid('PyQt5') - if api == QT_API_PYSIDE: + elif api == QT_API_PYSIDE: ID.forbid('PySide2') ID.forbid('PyQt4') ID.forbid('PyQt5') @@ -217,10 +217,9 @@ def import_pyqt5(): ImportErrors rasied within this function are non-recoverable """ - import sip from PyQt5 import QtCore, QtSvg, QtWidgets, QtGui - + # Alias PyQt-specific functions for PySide compatibility. QtCore.Signal = QtCore.pyqtSignal QtCore.Slot = QtCore.pyqtSlot @@ -283,7 +282,7 @@ def load_qt(api_options): Raises ------ ImportError, if it isn't possible to import any requested - bindings (either becaues they aren't installed, or because + bindings (either because they aren't installed, or because an incompatible library has already been installed) """ loaders = { diff --git a/IPython/lib/backgroundjobs.py b/IPython/lib/backgroundjobs.py index ebd70d3db39..31997e13f28 100644 --- a/IPython/lib/backgroundjobs.py +++ b/IPython/lib/backgroundjobs.py @@ -86,6 +86,7 @@ def __init__(self): self._s_running = BackgroundJobBase.stat_running_c self._s_completed = BackgroundJobBase.stat_completed_c self._s_dead = BackgroundJobBase.stat_dead_c + self._current_job_id = 0 @property def running(self): @@ -187,7 +188,8 @@ def new(self, func_or_exp, *args, **kwargs): if kwargs.get('daemon', False): job.daemon = True - job.num = len(self.all)+1 if self.all else 0 + job.num = self._current_job_id + self._current_job_id += 1 self.running.append(job) self.all[job.num] = job debug('Starting job # %s in a separate thread.' % job.num) diff --git a/IPython/lib/clipboard.py b/IPython/lib/clipboard.py index 1b8e756b5c6..316a8ab1f8a 100644 --- a/IPython/lib/clipboard.py +++ b/IPython/lib/clipboard.py @@ -32,15 +32,15 @@ def win32_clipboard_get(): win32clipboard.CloseClipboard() return text -def osx_clipboard_get(): +def osx_clipboard_get() -> str: """ Get the clipboard's text on OS X. """ p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'], stdout=subprocess.PIPE) - text, stderr = p.communicate() + bytes_, stderr = p.communicate() # Text comes in with old Mac \r line endings. Change them to \n. - text = text.replace(b'\r', b'\n') - text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING) + bytes_ = bytes_.replace(b'\r', b'\n') + text = py3compat.decode(bytes_) return text def tkinter_clipboard_get(): diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py index 586b2be8b1e..bd8c01b2a75 100644 --- a/IPython/lib/deepreload.py +++ b/IPython/lib/deepreload.py @@ -7,13 +7,7 @@ imported from that module, which is useful when you're changing files deep inside a package. -To use this as your default reload function, type this for Python 2:: - - import __builtin__ - from IPython.lib import deepreload - __builtin__.reload = deepreload.reload - -Or this for Python 3:: +To use this as your default reload function, type this:: import builtins from IPython.lib import deepreload @@ -271,7 +265,7 @@ def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1): def deep_reload_hook(m): """Replacement for reload().""" - # Hardcode this one as it would raise a NotImplemeentedError from the + # Hardcode this one as it would raise a NotImplementedError from the # bowels of Python and screw up the import machinery after. # unlike other imports the `exclude` list already in place is not enough. diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index 26e4fadd36a..0b19c413c37 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -197,7 +197,7 @@ def re_mark(mark): class Demo(object): - re_stop = re_mark('-*\s?stop\s?-*') + re_stop = re_mark(r'-*\s?stop\s?-*') re_silent = re_mark('silent') re_auto = re_mark('auto') re_auto_all = re_mark('auto_all') @@ -258,13 +258,17 @@ def __init__(self,src,title='',arg_str='',auto_all=None, format_rst=False, self.auto_all = auto_all self.src = src - self.inside_ipython = "get_ipython" in globals() + try: + ip = get_ipython() # this is in builtins whenever IPython is running + self.inside_ipython = True + except NameError: + self.inside_ipython = False + if self.inside_ipython: # get a few things from ipython. While it's a bit ugly design-wise, # it ensures that things like color scheme and the like are always in # sync with the ipython mode being used. This class is only meant to # be used inside ipython anyways, so it's OK. - ip = get_ipython() # this is in builtins whenever IPython is running self.ip_ns = ip.user_ns self.ip_colorize = ip.pycolorize self.ip_showtb = ip.showtraceback diff --git a/IPython/lib/display.py b/IPython/lib/display.py index 490d76fac6d..de31788ab97 100644 --- a/IPython/lib/display.py +++ b/IPython/lib/display.py @@ -33,9 +33,9 @@ class Audio(DisplayObject): * Bytestring containing raw PCM data or * URL pointing to a file on the web. - If the array option is used the waveform will be normalized. + If the array option is used, the waveform will be normalized. - If a filename or url is used the format support will be browser + If a filename or url is used, the format support will be browser dependent. url : unicode A URL to download the data from. @@ -54,6 +54,12 @@ class Audio(DisplayObject): autoplay : bool Set to True if the audio should immediately start playing. Default is `False`. + normalize : bool + Whether audio should be normalized (rescaled) to the maximum possible + range. Default is `True`. When set to `False`, `data` must be between + -1 and 1 (inclusive), otherwise an error is raised. + Applies only when `data` is a list or array of samples; other types of + audio are never normalized. Examples -------- @@ -63,7 +69,7 @@ class Audio(DisplayObject): import numpy as np framerate = 44100 t = np.linspace(0,5,framerate*5) - data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)) + data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t) Audio(data,rate=framerate) # Can also do stereo or more channels @@ -80,12 +86,18 @@ class Audio(DisplayObject): Audio(b'RAW_WAV_DATA..) # From bytes Audio(data=b'RAW_WAV_DATA..) + See Also + -------- + + See also the ``Audio`` widgets form the ``ipywidget`` package for more flexibility and options. + """ _read_flags = 'rb' - def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False): + def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *, + element_id=None): if filename is None and url is None and data is None: - raise ValueError("No image data found. Expecting filename, url, or data.") + raise ValueError("No audio data found. Expecting filename, url, or data.") if embed is False and url is None: raise ValueError("No url found. Expecting url when embed=False") @@ -94,10 +106,13 @@ def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, au else: self.embed = True self.autoplay = autoplay + self.element_id = element_id super(Audio, self).__init__(data=data, url=url, filename=filename) if self.data is not None and not isinstance(self.data, bytes): - self.data = self._make_wav(data,rate) + if rate is None: + raise ValueError("rate must be specified when data is a numpy array or list of audio samples.") + self.data = Audio._make_wav(data, rate, normalize) def reload(self): """Reload the raw data from file or URL.""" @@ -112,41 +127,16 @@ def reload(self): else: self.mimetype = "audio/wav" - def _make_wav(self, data, rate): + @staticmethod + def _make_wav(data, rate, normalize): """ Transform a numpy array to a PCM bytestring """ - import struct from io import BytesIO import wave try: - import numpy as np - - data = np.array(data, dtype=float) - if len(data.shape) == 1: - nchan = 1 - elif len(data.shape) == 2: - # In wave files,channels are interleaved. E.g., - # "L1R1L2R2..." for stereo. See - # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx - # for channel ordering - nchan = data.shape[0] - data = data.T.ravel() - else: - raise ValueError('Array audio input must be a 1D or 2D array') - scaled = np.int16(data/np.max(np.abs(data))*32767).tolist() + scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize) except ImportError: - # check that it is a "1D" list - idata = iter(data) # fails if not an iterable - try: - iter(idata.next()) - raise TypeError('Only lists of mono audio are ' - 'supported if numpy is not installed') - except TypeError: - # this means it's not a nested list, which is what we want - pass - maxabsvalue = float(max([abs(x) for x in data])) - scaled = [int(x/maxabsvalue*32767) for x in data] - nchan = 1 + scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize) fp = BytesIO() waveobj = wave.open(fp,mode='wb') @@ -154,12 +144,61 @@ def _make_wav(self, data, rate): waveobj.setframerate(rate) waveobj.setsampwidth(2) waveobj.setcomptype('NONE','NONE') - waveobj.writeframes(b''.join([struct.pack(' 1: + raise ValueError('Audio data must be between -1 and 1 when normalize=False.') + return max_abs_value if normalize else 1 + def _data_and_metadata(self): """shortcut for returning metadata with url information, if defined""" md = {} @@ -172,12 +211,13 @@ def _data_and_metadata(self): def _repr_html_(self): src = """ -